[INFO] fetching crate sharedstate 0.4.2... [INFO] testing sharedstate-0.4.2 against master#cdb45c87e2cd43495379f7e867e3cc15dcee9f93 for pr-145838-1 [INFO] extracting crate sharedstate 0.4.2 into /workspace/builds/worker-4-tc1/source [INFO] started tweaking crates.io crate sharedstate 0.4.2 [INFO] finished tweaking crates.io crate sharedstate 0.4.2 [INFO] tweaked toml for crates.io crate sharedstate 0.4.2 written to /workspace/builds/worker-4-tc1/source/Cargo.toml [INFO] validating manifest of crates.io crate sharedstate 0.4.2 on toolchain cdb45c87e2cd43495379f7e867e3cc15dcee9f93 [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+cdb45c87e2cd43495379f7e867e3cc15dcee9f93" "metadata" "--manifest-path" "Cargo.toml" "--no-deps", kill_on_drop: false }` [INFO] crate crates.io crate sharedstate 0.4.2 already has a lockfile, it will not be regenerated [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+cdb45c87e2cd43495379f7e867e3cc15dcee9f93" "fetch" "--manifest-path" "Cargo.toml", kill_on_drop: false }` [INFO] [stderr] Updating crates.io index [INFO] [stderr] Downloading crates ... [INFO] [stderr] Downloaded arc-metrics v0.1.2 [INFO] [stderr] Downloaded sequenced-broadcast v0.2.6 [INFO] [stderr] Downloaded pkg-details v0.1.2 [INFO] [stderr] Downloaded message-encoding v0.2.7 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7ad1b28ee6f5f7f699f6cf7015098d6ccdd96d6f2d78dd06228f5b4c9faf309c" "/opt/rustwide/cargo-home/bin/cargo" "+cdb45c87e2cd43495379f7e867e3cc15dcee9f93" "metadata" "--no-deps" "--format-version=1", kill_on_drop: false }` [INFO] [stdout] c8a92b02e8fb181f6240620b44d03929e934c34b2b4d8e00bc25cfa999dc9bee [INFO] running `Command { std: "docker" "start" "-a" "c8a92b02e8fb181f6240620b44d03929e934c34b2b4d8e00bc25cfa999dc9bee", kill_on_drop: false }` [INFO] running `Command { std: "docker" "inspect" "c8a92b02e8fb181f6240620b44d03929e934c34b2b4d8e00bc25cfa999dc9bee", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "c8a92b02e8fb181f6240620b44d03929e934c34b2b4d8e00bc25cfa999dc9bee", kill_on_drop: false }` [INFO] [stdout] c8a92b02e8fb181f6240620b44d03929e934c34b2b4d8e00bc25cfa999dc9bee [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7ad1b28ee6f5f7f699f6cf7015098d6ccdd96d6f2d78dd06228f5b4c9faf309c" "/opt/rustwide/cargo-home/bin/cargo" "+cdb45c87e2cd43495379f7e867e3cc15dcee9f93" "build" "--frozen" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] 8b4211b8323e1588c20ef141caf986d47839b2e7ae658b00fc5cfb1f976f6ab8 [INFO] running `Command { std: "docker" "start" "-a" "8b4211b8323e1588c20ef141caf986d47839b2e7ae658b00fc5cfb1f976f6ab8", kill_on_drop: false }` [INFO] [stderr] Compiling smallvec v1.15.0 [INFO] [stderr] Compiling lock_api v0.4.13 [INFO] [stderr] Compiling syn v2.0.101 [INFO] [stderr] Compiling signal-hook-registry v1.4.5 [INFO] [stderr] Compiling socket2 v0.5.10 [INFO] [stderr] Compiling mio v1.0.4 [INFO] [stderr] Compiling pkg-details v0.1.2 [INFO] [stderr] Compiling ppv-lite86 v0.2.21 [INFO] [stderr] Compiling arc-metrics v0.1.2 [INFO] [stderr] Compiling parking_lot_core v0.9.11 [INFO] [stderr] Compiling rand_core v0.6.4 [INFO] [stderr] Compiling parking_lot v0.12.4 [INFO] [stderr] Compiling message-encoding v0.2.7 [INFO] [stderr] Compiling rand_chacha v0.3.1 [INFO] [stderr] Compiling tokio-macros v2.5.0 [INFO] [stderr] Compiling tracing-attributes v0.1.28 [INFO] [stderr] Compiling futures-macro v0.3.31 [INFO] [stderr] Compiling tokio v1.45.1 [INFO] [stderr] Compiling futures-util v0.3.31 [INFO] [stderr] Compiling tracing v0.1.41 [INFO] [stderr] Compiling tokio-util v0.7.15 [INFO] [stderr] Compiling sequenced-broadcast v0.2.6 [INFO] [stderr] Compiling sharedstate v0.4.2 (/opt/rustwide/workdir) [INFO] [stdout] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:1:32 [INFO] [stdout] | [INFO] [stdout] 1 | use std::{hash::{Hash, Hasher, SipHasher}, time::{Duration, SystemTime, UNIX_EPOCH}}; [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(deprecated)]` on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:491:20 [INFO] [stdout] | [INFO] [stdout] 491 | let mut hash = SipHasher::new(); [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated associated function `std::hash::SipHasher::new`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:491:31 [INFO] [stdout] | [INFO] [stdout] 491 | let mut hash = SipHasher::new(); [INFO] [stdout] | ^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: field `last_client_msg_epoch` is never read [INFO] [stdout] --> src/worker/sync_manager.rs:1370:5 [INFO] [stdout] | [INFO] [stdout] 1367 | struct DiscoveredPeer { [INFO] [stdout] | -------------- field in this struct [INFO] [stdout] ... [INFO] [stdout] 1370 | last_client_msg_epoch: u64, [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: method `log` is never used [INFO] [stdout] --> src/utils.rs:4:8 [INFO] [stdout] | [INFO] [stdout] 3 | pub trait LogHelper { [INFO] [stdout] | --------- method in this trait [INFO] [stdout] 4 | fn log(self) -> Self; [INFO] [stdout] | ^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: trait `TimeoutPanicHelper` is never used [INFO] [stdout] --> src/utils.rs:110:11 [INFO] [stdout] | [INFO] [stdout] 110 | pub trait TimeoutPanicHelper: Sized { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:136:17 [INFO] [stdout] | [INFO] [stdout] 136 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] = note: `#[warn(mismatched_lifetime_syntaxes)]` on by default [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 136 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:142:17 [INFO] [stdout] | [INFO] [stdout] 142 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 142 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:266:23 [INFO] [stdout] | [INFO] [stdout] 266 | pub fn read_state(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 266 | pub fn read_state(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:384:17 [INFO] [stdout] | [INFO] [stdout] 384 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 384 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stderr] Finished `dev` profile [unoptimized + debuginfo] target(s) in 25.68s [INFO] running `Command { std: "docker" "inspect" "8b4211b8323e1588c20ef141caf986d47839b2e7ae658b00fc5cfb1f976f6ab8", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "8b4211b8323e1588c20ef141caf986d47839b2e7ae658b00fc5cfb1f976f6ab8", kill_on_drop: false }` [INFO] [stdout] 8b4211b8323e1588c20ef141caf986d47839b2e7ae658b00fc5cfb1f976f6ab8 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7ad1b28ee6f5f7f699f6cf7015098d6ccdd96d6f2d78dd06228f5b4c9faf309c" "/opt/rustwide/cargo-home/bin/cargo" "+cdb45c87e2cd43495379f7e867e3cc15dcee9f93" "test" "--frozen" "--no-run" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] acd4d382db10f5f483451384c1213aa5d3f993fcf150374cad7d72c97921e322 [INFO] running `Command { std: "docker" "start" "-a" "acd4d382db10f5f483451384c1213aa5d3f993fcf150374cad7d72c97921e322", kill_on_drop: false }` [INFO] [stderr] Compiling tracing v0.1.41 [INFO] [stderr] Compiling tracing-subscriber v0.3.19 [INFO] [stderr] Compiling sequenced-broadcast v0.2.6 [INFO] [stderr] Compiling sharedstate v0.4.2 (/opt/rustwide/workdir) [INFO] [stdout] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:1:32 [INFO] [stdout] | [INFO] [stdout] 1 | use std::{hash::{Hash, Hasher, SipHasher}, time::{Duration, SystemTime, UNIX_EPOCH}}; [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(deprecated)]` on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:491:20 [INFO] [stdout] | [INFO] [stdout] 491 | let mut hash = SipHasher::new(); [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated associated function `std::hash::SipHasher::new`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:491:31 [INFO] [stdout] | [INFO] [stdout] 491 | let mut hash = SipHasher::new(); [INFO] [stdout] | ^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: field `last_client_msg_epoch` is never read [INFO] [stdout] --> src/worker/sync_manager.rs:1370:5 [INFO] [stdout] | [INFO] [stdout] 1367 | struct DiscoveredPeer { [INFO] [stdout] | -------------- field in this struct [INFO] [stdout] ... [INFO] [stdout] 1370 | last_client_msg_epoch: u64, [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: method `log` is never used [INFO] [stdout] --> src/utils.rs:4:8 [INFO] [stdout] | [INFO] [stdout] 3 | pub trait LogHelper { [INFO] [stdout] | --------- method in this trait [INFO] [stdout] 4 | fn log(self) -> Self; [INFO] [stdout] | ^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: trait `TimeoutPanicHelper` is never used [INFO] [stdout] --> src/utils.rs:110:11 [INFO] [stdout] | [INFO] [stdout] 110 | pub trait TimeoutPanicHelper: Sized { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:136:17 [INFO] [stdout] | [INFO] [stdout] 136 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] = note: `#[warn(mismatched_lifetime_syntaxes)]` on by default [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 136 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:142:17 [INFO] [stdout] | [INFO] [stdout] 142 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 142 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:266:23 [INFO] [stdout] | [INFO] [stdout] 266 | pub fn read_state(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 266 | pub fn read_state(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:384:17 [INFO] [stdout] | [INFO] [stdout] 384 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 384 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: unused import: `worker::sync_updater::FollowTarget` [INFO] [stdout] --> src/worker/sync_updater.rs:502:133 [INFO] [stdout] | [INFO] [stdout] 502 | ...TestState, TestStateAction}}, utils::PanicHelper, worker::sync_updater::FollowTarget}; [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: unused import: `tokio::sync::mpsc::channel` [INFO] [stdout] --> src/testing/fuzzy_test.rs:4:5 [INFO] [stdout] | [INFO] [stdout] 4 | use tokio::sync::mpsc::channel; [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:1:32 [INFO] [stdout] | [INFO] [stdout] 1 | use std::{hash::{Hash, Hasher, SipHasher}, time::{Duration, SystemTime, UNIX_EPOCH}}; [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(deprecated)]` on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:491:20 [INFO] [stdout] | [INFO] [stdout] 491 | let mut hash = SipHasher::new(); [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated associated function `std::hash::SipHasher::new`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:491:31 [INFO] [stdout] | [INFO] [stdout] 491 | let mut hash = SipHasher::new(); [INFO] [stdout] | ^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: field `last_client_msg_epoch` is never read [INFO] [stdout] --> src/worker/sync_manager.rs:1370:5 [INFO] [stdout] | [INFO] [stdout] 1367 | struct DiscoveredPeer { [INFO] [stdout] | -------------- field in this struct [INFO] [stdout] ... [INFO] [stdout] 1370 | last_client_msg_epoch: u64, [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: method `log` is never used [INFO] [stdout] --> src/utils.rs:4:8 [INFO] [stdout] | [INFO] [stdout] 3 | pub trait LogHelper { [INFO] [stdout] | --------- method in this trait [INFO] [stdout] 4 | fn log(self) -> Self; [INFO] [stdout] | ^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: trait `TimeoutPanicHelper` is never used [INFO] [stdout] --> src/utils.rs:110:11 [INFO] [stdout] | [INFO] [stdout] 110 | pub trait TimeoutPanicHelper: Sized { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: field `rate_limit_bps` is never read [INFO] [stdout] --> src/testing/test_sync_io.rs:99:5 [INFO] [stdout] | [INFO] [stdout] 96 | struct ConnManage { [INFO] [stdout] | ---------- field in this struct [INFO] [stdout] ... [INFO] [stdout] 99 | rate_limit_bps: Arc, [INFO] [stdout] | ^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:136:17 [INFO] [stdout] | [INFO] [stdout] 136 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] = note: `#[warn(mismatched_lifetime_syntaxes)]` on by default [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 136 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:142:17 [INFO] [stdout] | [INFO] [stdout] 142 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 142 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:266:23 [INFO] [stdout] | [INFO] [stdout] 266 | pub fn read_state(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 266 | pub fn read_state(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:384:17 [INFO] [stdout] | [INFO] [stdout] 384 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 384 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stderr] Finished `test` profile [unoptimized + debuginfo] target(s) in 14.39s [INFO] running `Command { std: "docker" "inspect" "acd4d382db10f5f483451384c1213aa5d3f993fcf150374cad7d72c97921e322", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "acd4d382db10f5f483451384c1213aa5d3f993fcf150374cad7d72c97921e322", kill_on_drop: false }` [INFO] [stdout] acd4d382db10f5f483451384c1213aa5d3f993fcf150374cad7d72c97921e322 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7ad1b28ee6f5f7f699f6cf7015098d6ccdd96d6f2d78dd06228f5b4c9faf309c" "/opt/rustwide/cargo-home/bin/cargo" "+cdb45c87e2cd43495379f7e867e3cc15dcee9f93" "test" "--frozen", kill_on_drop: false }` [INFO] [stdout] 08a46c804b7ab9f4457424298e45448a67a968771879dee1db9c4cab6c0a232c [INFO] running `Command { std: "docker" "start" "-a" "08a46c804b7ab9f4457424298e45448a67a968771879dee1db9c4cab6c0a232c", kill_on_drop: false }` [INFO] [stderr] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stderr] --> src/worker/sync_updater.rs:1:32 [INFO] [stderr] | [INFO] [stderr] 1 | use std::{hash::{Hash, Hasher, SipHasher}, time::{Duration, SystemTime, UNIX_EPOCH}}; [INFO] [stderr] | ^^^^^^^^^ [INFO] [stderr] | [INFO] [stderr] = note: `#[warn(deprecated)]` on by default [INFO] [stderr] [INFO] [stderr] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stderr] --> src/worker/sync_updater.rs:491:20 [INFO] [stderr] | [INFO] [stderr] 491 | let mut hash = SipHasher::new(); [INFO] [stderr] | ^^^^^^^^^ [INFO] [stderr] [INFO] [stderr] warning: use of deprecated associated function `std::hash::SipHasher::new`: use `std::hash::DefaultHasher` instead [INFO] [stderr] --> src/worker/sync_updater.rs:491:31 [INFO] [stderr] | [INFO] [stderr] 491 | let mut hash = SipHasher::new(); [INFO] [stderr] | ^^^ [INFO] [stderr] [INFO] [stderr] warning: field `last_client_msg_epoch` is never read [INFO] [stderr] --> src/worker/sync_manager.rs:1370:5 [INFO] [stderr] | [INFO] [stderr] 1367 | struct DiscoveredPeer { [INFO] [stderr] | -------------- field in this struct [INFO] [stderr] ... [INFO] [stderr] 1370 | last_client_msg_epoch: u64, [INFO] [stderr] | ^^^^^^^^^^^^^^^^^^^^^ [INFO] [stderr] | [INFO] [stderr] = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default [INFO] [stderr] [INFO] [stderr] warning: method `log` is never used [INFO] [stderr] --> src/utils.rs:4:8 [INFO] [stderr] | [INFO] [stderr] 3 | pub trait LogHelper { [INFO] [stderr] | --------- method in this trait [INFO] [stderr] 4 | fn log(self) -> Self; [INFO] [stderr] | ^^^ [INFO] [stderr] [INFO] [stderr] warning: trait `TimeoutPanicHelper` is never used [INFO] [stderr] --> src/utils.rs:110:11 [INFO] [stderr] | [INFO] [stderr] 110 | pub trait TimeoutPanicHelper: Sized { [INFO] [stderr] | ^^^^^^^^^^^^^^^^^^ [INFO] [stderr] [INFO] [stderr] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stderr] --> src/state.rs:136:17 [INFO] [stderr] | [INFO] [stderr] 136 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stderr] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stderr] | | [INFO] [stderr] | the lifetime is elided here [INFO] [stderr] | [INFO] [stderr] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stderr] = note: `#[warn(mismatched_lifetime_syntaxes)]` on by default [INFO] [stderr] help: use `'_` for type paths [INFO] [stderr] | [INFO] [stderr] 136 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stderr] | +++ [INFO] [stderr] [INFO] [stderr] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stderr] --> src/state.rs:142:17 [INFO] [stderr] | [INFO] [stderr] 142 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stderr] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stderr] | | [INFO] [stderr] | the lifetime is elided here [INFO] [stderr] | [INFO] [stderr] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stderr] help: use `'_` for type paths [INFO] [stderr] | [INFO] [stderr] 142 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stderr] | +++ [INFO] [stderr] [INFO] [stderr] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stderr] --> src/state.rs:266:23 [INFO] [stderr] | [INFO] [stderr] 266 | pub fn read_state(&self) -> RwLockReadGuard { [INFO] [stderr] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stderr] | | [INFO] [stderr] | the lifetime is elided here [INFO] [stderr] | [INFO] [stderr] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stderr] help: use `'_` for type paths [INFO] [stderr] | [INFO] [stderr] 266 | pub fn read_state(&self) -> RwLockReadGuard<'_, D> { [INFO] [stderr] | +++ [INFO] [stderr] [INFO] [stderr] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stderr] --> src/state.rs:384:17 [INFO] [stderr] | [INFO] [stderr] 384 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stderr] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stderr] | | [INFO] [stderr] | the lifetime is elided here [INFO] [stderr] | [INFO] [stderr] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stderr] help: use `'_` for type paths [INFO] [stderr] | [INFO] [stderr] 384 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stderr] | +++ [INFO] [stderr] [INFO] [stderr] warning: `sharedstate` (lib) generated 10 warnings (run `cargo fix --lib -p sharedstate` to apply 4 suggestions) [INFO] [stderr] warning: unused import: `worker::sync_updater::FollowTarget` [INFO] [stderr] --> src/worker/sync_updater.rs:502:133 [INFO] [stderr] | [INFO] [stderr] 502 | ...TestState, TestStateAction}}, utils::PanicHelper, worker::sync_updater::FollowTarget}; [INFO] [stderr] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stderr] | [INFO] [stderr] = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default [INFO] [stderr] [INFO] [stderr] warning: unused import: `tokio::sync::mpsc::channel` [INFO] [stderr] --> src/testing/fuzzy_test.rs:4:5 [INFO] [stderr] | [INFO] [stderr] 4 | use tokio::sync::mpsc::channel; [INFO] [stderr] | ^^^^^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stderr] [INFO] [stderr] warning: field `rate_limit_bps` is never read [INFO] [stderr] --> src/testing/test_sync_io.rs:99:5 [INFO] [stderr] | [INFO] [stderr] 96 | struct ConnManage { [INFO] [stderr] | ---------- field in this struct [INFO] [stderr] ... [INFO] [stderr] 99 | rate_limit_bps: Arc, [INFO] [stderr] | ^^^^^^^^^^^^^^ [INFO] [stderr] [INFO] [stderr] warning: `sharedstate` (lib test) generated 13 warnings (10 duplicates) (run `cargo fix --lib -p sharedstate --tests` to apply 2 suggestions) [INFO] [stderr] Finished `test` profile [unoptimized + debuginfo] target(s) in 0.18s [INFO] [stderr] Running unittests src/lib.rs (/opt/rustwide/target/debug/deps/sharedstate-ee8aadbf8182d277) [INFO] [stdout] [INFO] [stdout] running 16 tests [INFO] [stdout] test recoverable_state::test::state_recovery_check_test ... ok [INFO] [stdout] test recoverable_state::test::recoverable_state_encoding_test ... ok [INFO] [stdout] test state::test::state_flush_tests ... ok [INFO] [stdout] test testing::message_io_tests::message_io_send_receive ... ok [INFO] [stdout] test testing::message_io_tests::message_io_zero_send ... ok [INFO] [stdout] test testing::state_tests::test_state_action_encoding_test ... ok [INFO] [stdout] test net::message_io::test::message_io_test ... ok [INFO] [stdout] test worker::sync_updater::test::sync_updater_recover_state_fork_same_seq_reject_test ... ok [INFO] [stdout] test worker::sync_updater::test::sync_updater_recover_state_fork_reject_test ... ok [INFO] [stdout] test worker::sync_updater::test::sync_updater_recover_state_test ... ok [INFO] [stdout] test worker::sync_updater::test::sync_updater_simple_follower_test ... ok [INFO] [stdout] test worker::sync_updater::test::sync_updater_simple_leader_test ... ok [INFO] [stdout] test worker::sync_manager::test::sync_manager_connects_through_peer_test ... ok [INFO] [stdout] test testing::state_tests::slow_state_deadlock_test ... FAILED [INFO] [stdout] test worker::sync_manager::test::sync_manager_doesnt_allow_peer_with_different_leader_test ... ok [INFO] [stdout] test testing::fuzzy_test::fuzzy_test has been running for over 60 seconds [INFO] [stdout] test testing::fuzzy_test::fuzzy_test ... FAILED [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] [INFO] [stdout] ---- testing::state_tests::slow_state_deadlock_test stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'testing::state_tests::slow_state_deadlock_test' (24) panicked at src/testing/state_tests.rs:215:5: [INFO] [stdout] assertion failed: !failed.load(Ordering::SeqCst) [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x5e97b9557d52 - std::backtrace_rs::backtrace::libunwind::trace::h7cddb8376417e7cc [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x5e97b9557d52 - std::backtrace_rs::backtrace::trace_unsynchronized::hb4e41acf4b349ff1 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x5e97b9557d52 - std::sys::backtrace::_print_fmt::h1222b80910ba6eb5 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x5e97b9557d52 - ::fmt::h5bb8a979ba5db788 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x5e97b956977f - core::fmt::rt::Argument::fmt::h2ee2c138a50a7796 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x5e97b956977f - core::fmt::write::h1e0dbf07fe3990bd [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/fmt/mod.rs:1468:25 [INFO] [stdout] 6: 0x5e97b9524ee3 - std::io::default_write_fmt::h7d7ad5ed6a883d81 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x5e97b9524ee3 - std::io::Write::write_fmt::hc477d9325b345ece [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x5e97b9530e62 - std::sys::backtrace::BacktraceLock::print::h788d486777205086 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x5e97b9535b7f - std::panicking::default_hook::{{closure}}::ha7bdfeb5949fc0fa [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x5e97b9535a11 - std::panicking::default_hook::h6dad75ec721846f4 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x5e97b940225e - as core::ops::function::Fn>::call::h33799d04b0b96146 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 12: 0x5e97b940225e - test::test_main_with_exit_callback::{{closure}}::h880e31c829a2eb5d [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x5e97b95362ce - as core::ops::function::Fn>::call::h5e1b8c652ea49180 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 14: 0x5e97b95362ce - std::panicking::panic_with_hook::hba00e869ada17676 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x5e97b9535fb6 - std::panicking::panic_handler::{{closure}}::h3d21149c258e5ceb [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:700:13 [INFO] [stdout] 16: 0x5e97b9530f99 - std::sys::backtrace::__rust_end_short_backtrace::h4f5d9b9dfb3e6ec1 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x5e97b95190ed - __rustc[9a7a9f9af7564de1]::rust_begin_unwind [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x5e97b9571700 - core::panicking::panic_fmt::h78e817a90331d98b [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/panicking.rs:75:14 [INFO] [stdout] 19: 0x5e97b95716dc - core::panicking::panic::h33d2d00634ffed6f [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/panicking.rs:145:5 [INFO] [stdout] 20: 0x5e97b932162b - sharedstate::testing::state_tests::slow_state_deadlock_test::h23f8e617f5cc152e [INFO] [stdout] at /opt/rustwide/workdir/src/testing/state_tests.rs:215:5 [INFO] [stdout] 21: 0x5e97b9321d37 - sharedstate::testing::state_tests::slow_state_deadlock_test::{{closure}}::hf740f04be3eec1e7 [INFO] [stdout] at /opt/rustwide/workdir/src/testing/state_tests.rs:150:30 [INFO] [stdout] 22: 0x5e97b91f13a6 - core::ops::function::FnOnce::call_once::hcfcc7c108ff33561 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/ops/function.rs:253:5 [INFO] [stdout] 23: 0x5e97b940205b - core::ops::function::FnOnce::call_once::h7f4b4fba903e39d5 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/ops/function.rs:253:5 [INFO] [stdout] 24: 0x5e97b940205b - test::__rust_begin_short_backtrace::h9277cb6a2ccfc000 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/test/src/lib.rs:663:18 [INFO] [stdout] 25: 0x5e97b9417b05 - test::run_test_in_process::{{closure}}::h9aea5ca90d1f4423 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/test/src/lib.rs:686:74 [INFO] [stdout] 26: 0x5e97b9417b05 - as core::ops::function::FnOnce<()>>::call_once::h364f5fe6cc8afa85 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/panic/unwind_safe.rs:272:9 [INFO] [stdout] 27: 0x5e97b9417b05 - std::panicking::catch_unwind::do_call::h2b2bec3317fe53ec [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:590:40 [INFO] [stdout] 28: 0x5e97b9417b05 - std::panicking::catch_unwind::hc3763734156da4af [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:553:19 [INFO] [stdout] 29: 0x5e97b9417b05 - std::panic::catch_unwind::h28038391e867eabc [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panic.rs:359:14 [INFO] [stdout] 30: 0x5e97b9417b05 - test::run_test_in_process::had0273166695a036 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/test/src/lib.rs:686:27 [INFO] [stdout] 31: 0x5e97b9417b05 - test::run_test::{{closure}}::h743d09d4bb476605 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/test/src/lib.rs:607:43 [INFO] [stdout] 32: 0x5e97b93ee524 - test::run_test::{{closure}}::he950b8f9118d37e2 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/test/src/lib.rs:637:41 [INFO] [stdout] 33: 0x5e97b93ee524 - std::sys::backtrace::__rust_begin_short_backtrace::hbfaffa6539f6abb7 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 34: 0x5e97b93f1e0a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hcce3c2c65b9c3b20 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 35: 0x5e97b93f1e0a - as core::ops::function::FnOnce<()>>::call_once::hb66b30b6d37985a5 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/panic/unwind_safe.rs:272:9 [INFO] [stdout] 36: 0x5e97b93f1e0a - std::panicking::catch_unwind::do_call::h992bbe2c32dc1d79 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:590:40 [INFO] [stdout] 37: 0x5e97b93f1e0a - std::panicking::catch_unwind::h412991d5237de610 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:553:19 [INFO] [stdout] 38: 0x5e97b93f1e0a - std::panic::catch_unwind::ha82b139b3eb5840a [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panic.rs:359:14 [INFO] [stdout] 39: 0x5e97b93f1e0a - std::thread::Builder::spawn_unchecked_::{{closure}}::ha823b36f5114938e [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 40: 0x5e97b93f1e0a - core::ops::function::FnOnce::call_once{{vtable.shim}}::hf120e7b1db22ac07 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/ops/function.rs:253:5 [INFO] [stdout] 41: 0x5e97b952b21f - as core::ops::function::FnOnce>::call_once::h3e049222c99298ac [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 42: 0x5e97b952b21f - std::sys::pal::unix::thread::Thread::new::thread_start::h942e336943ad5963 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/sys/pal/unix/thread.rs:118:17 [INFO] [stdout] 43: 0x76916dec8aa4 - [INFO] [stdout] 44: 0x76916df55a34 - clone [INFO] [stdout] 45: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- testing::fuzzy_test::fuzzy_test stdout ---- [INFO] [stdout] 2025-09-01T11:52:10.317060Z INFO Node{id=1}: sequenced_broadcast: 15|SequencedBroadcastWorker Started id=15 [INFO] [stdout] 2025-09-01T11:52:10.317150Z INFO Node{id=2}: sequenced_broadcast: 16|SequencedBroadcastWorker Started id=16 [INFO] [stdout] 2025-09-01T11:52:10.317216Z INFO Node{id=3}: sequenced_broadcast: 17|SequencedBroadcastWorker Started id=17 [INFO] [stdout] 2025-09-01T11:52:10.317273Z INFO Node{id=4}: sequenced_broadcast: 18|SequencedBroadcastWorker Started id=18 [INFO] [stdout] 2025-09-01T11:52:10.317343Z INFO Node{id=5}: sequenced_broadcast: 19|SequencedBroadcastWorker Started id=19 [INFO] [stdout] 2025-09-01T11:52:10.317413Z INFO Node{id=6}: sequenced_broadcast: 20|SequencedBroadcastWorker Started id=20 [INFO] [stdout] 2025-09-01T11:52:10.317473Z INFO Node{id=7}: sequenced_broadcast: 21|SequencedBroadcastWorker Started id=21 [INFO] [stdout] 2025-09-01T11:52:10.317529Z INFO Node{id=8}: sequenced_broadcast: 22|SequencedBroadcastWorker Started id=22 [INFO] [stdout] 2025-09-01T11:52:10.317612Z INFO Node{id=9}: sequenced_broadcast: 23|SequencedBroadcastWorker Started id=23 [INFO] [stdout] 2025-09-01T11:52:10.317694Z INFO Node{id=10}: sequenced_broadcast: 24|SequencedBroadcastWorker Started id=24 [INFO] [stdout] 2025-09-01T11:52:10.317777Z INFO Node{id=11}: sequenced_broadcast: 25|SequencedBroadcastWorker Started id=25 [INFO] [stdout] 2025-09-01T11:52:10.317834Z INFO Node{id=12}: sequenced_broadcast: 26|SequencedBroadcastWorker Started id=26 [INFO] [stdout] 2025-09-01T11:52:10.317881Z INFO Node{id=13}: sequenced_broadcast: 27|SequencedBroadcastWorker Started id=27 [INFO] [stdout] 2025-09-01T11:52:10.317932Z INFO Node{id=14}: sequenced_broadcast: 28|SequencedBroadcastWorker Started id=28 [INFO] [stdout] 2025-09-01T11:52:10.317984Z INFO Node{id=15}: sequenced_broadcast: 29|SequencedBroadcastWorker Started id=29 [INFO] [stdout] 2025-09-01T11:52:10.318036Z INFO Node{id=16}: sequenced_broadcast: 30|SequencedBroadcastWorker Started id=30 [INFO] [stdout] 2025-09-01T11:52:10.318115Z INFO Node{id=17}: sequenced_broadcast: 31|SequencedBroadcastWorker Started id=31 [INFO] [stdout] 2025-09-01T11:52:10.318174Z INFO Node{id=18}: sequenced_broadcast: 32|SequencedBroadcastWorker Started id=32 [INFO] [stdout] 2025-09-01T11:52:10.318229Z INFO Node{id=19}: sequenced_broadcast: 33|SequencedBroadcastWorker Started id=33 [INFO] [stdout] 2025-09-01T11:52:10.318296Z INFO Node{id=20}: sequenced_broadcast: 34|SequencedBroadcastWorker Started id=34 [INFO] [stdout] 2025-09-01T11:52:10.318356Z INFO Node{id=21}: sequenced_broadcast: 35|SequencedBroadcastWorker Started id=35 [INFO] [stdout] 2025-09-01T11:52:10.318413Z INFO Node{id=22}: sequenced_broadcast: 36|SequencedBroadcastWorker Started id=36 [INFO] [stdout] 2025-09-01T11:52:10.318465Z INFO Node{id=23}: sequenced_broadcast: 37|SequencedBroadcastWorker Started id=37 [INFO] [stdout] 2025-09-01T11:52:10.318543Z INFO Node{id=24}: sequenced_broadcast: 38|SequencedBroadcastWorker Started id=38 [INFO] [stdout] 2025-09-01T11:52:10.318604Z INFO Node{id=25}: sequenced_broadcast: 39|SequencedBroadcastWorker Started id=39 [INFO] [stdout] 2025-09-01T11:52:10.318658Z INFO Node{id=26}: sequenced_broadcast: 40|SequencedBroadcastWorker Started id=40 [INFO] [stdout] 2025-09-01T11:52:10.318712Z INFO Node{id=27}: sequenced_broadcast: 41|SequencedBroadcastWorker Started id=41 [INFO] [stdout] 2025-09-01T11:52:10.318789Z INFO Node{id=28}: sequenced_broadcast: 42|SequencedBroadcastWorker Started id=42 [INFO] [stdout] 2025-09-01T11:52:10.318850Z INFO Node{id=29}: sequenced_broadcast: 43|SequencedBroadcastWorker Started id=43 [INFO] [stdout] 2025-09-01T11:52:10.318906Z INFO Node{id=30}: sequenced_broadcast: 44|SequencedBroadcastWorker Started id=44 [INFO] [stdout] 2025-09-01T11:52:10.318961Z INFO Node{id=31}: sequenced_broadcast: 45|SequencedBroadcastWorker Started id=45 [INFO] [stdout] 2025-09-01T11:52:10.319044Z INFO Node{id=32}: sequenced_broadcast: 46|SequencedBroadcastWorker Started id=46 [INFO] [stdout] 2025-09-01T11:52:10.319160Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.319261Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.319351Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.319425Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.319498Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.319574Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.319644Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.319667Z INFO Node{id=1}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:52:10.319742Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.320375Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.320449Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.320521Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.320592Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.320671Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.320735Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.320828Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.320877Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.320945Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321015Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321086Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321157Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321225Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321305Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321374Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321442Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321502Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321576Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321647Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321724Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321820Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.321895Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.322039Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.322126Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:52:10.325402Z INFO Node{id=7}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325442Z INFO Node{id=6}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325469Z INFO Node{id=5}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325494Z INFO Node{id=4}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325518Z INFO Node{id=3}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325542Z INFO Node{id=2}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325563Z INFO Node{id=15}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325583Z INFO Node{id=14}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325604Z INFO Node{id=13}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325630Z INFO Node{id=12}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325655Z INFO Node{id=11}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325833Z INFO Node{id=10}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325863Z INFO Node{id=9}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325881Z INFO Node{id=8}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325898Z INFO Node{id=23}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325916Z INFO Node{id=22}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325933Z INFO Node{id=21}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325951Z INFO Node{id=20}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325968Z INFO Node{id=19}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.325984Z INFO Node{id=18}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.326002Z INFO Node{id=17}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.326019Z INFO Node{id=16}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.326036Z INFO Node{id=30}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.326053Z INFO Node{id=29}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.326071Z INFO Node{id=28}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.326087Z INFO Node{id=27}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.326103Z INFO Node{id=26}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.326120Z INFO Node{id=25}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.326137Z INFO Node{id=24}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.326154Z INFO Node{id=32}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.326266Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.326290Z INFO Node{id=11}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.326300Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.326360Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.326371Z INFO Node{id=12}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.326379Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.326413Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.326424Z INFO Node{id=13}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.326431Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.326459Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.326468Z INFO Node{id=14}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.326475Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.326507Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.326517Z INFO Node{id=15}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.326524Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.326560Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.326571Z INFO Node{id=2}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.326578Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.326611Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.326621Z INFO Node{id=3}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.326628Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.326660Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.326671Z INFO Node{id=4}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.326678Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.326713Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.326724Z INFO Node{id=5}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.326732Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.326816Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.326831Z INFO Node{id=6}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.326839Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.326905Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.326917Z INFO Node{id=7}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.326928Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.326974Z INFO Node{id=31}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:52:10.327024Z INFO Node{id=11}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.327091Z INFO Node{id=11}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.327138Z INFO Node{id=12}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.327204Z INFO Node{id=12}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.327245Z INFO Node{id=13}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.327315Z INFO Node{id=13}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.327361Z INFO Node{id=14}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.327423Z INFO Node{id=14}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.327466Z INFO Node{id=15}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.327527Z INFO Node{id=15}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.327570Z INFO Node{id=2}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.327631Z INFO Node{id=2}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.327672Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.327733Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.327797Z INFO Node{id=4}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.327862Z INFO Node{id=4}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.327903Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.327966Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.328072Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328085Z INFO Node{id=32}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328092Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328134Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328145Z INFO Node{id=24}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328153Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328188Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328199Z INFO Node{id=25}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328206Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328241Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328253Z INFO Node{id=26}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328260Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328315Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328328Z INFO Node{id=27}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328336Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328373Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328384Z INFO Node{id=28}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328392Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328424Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328434Z INFO Node{id=29}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328442Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328472Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328482Z INFO Node{id=30}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328490Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328526Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328539Z INFO Node{id=16}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328547Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328586Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328598Z INFO Node{id=17}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328605Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328639Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328650Z INFO Node{id=18}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328658Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328692Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328703Z INFO Node{id=19}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328711Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328750Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328781Z INFO Node{id=20}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328789Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328836Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328849Z INFO Node{id=21}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328856Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328894Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328912Z INFO Node{id=22}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328921Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.328965Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.328978Z INFO Node{id=23}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.328986Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.329027Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.329040Z INFO Node{id=8}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.329048Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.329092Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.329106Z INFO Node{id=9}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.329114Z INFO Node{id=9}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.329165Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.329180Z INFO Node{id=10}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.329188Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.329253Z INFO Node{id=6}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.329338Z INFO Node{id=6}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.329387Z INFO Node{id=7}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.329457Z INFO Node{id=7}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.329503Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:52:10.329686Z INFO Node{id=32}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.329810Z INFO Node{id=32}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.329860Z INFO Node{id=24}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.329927Z INFO Node{id=24}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.329972Z INFO Node{id=25}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.330042Z INFO Node{id=25}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.330098Z INFO Node{id=26}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.330175Z INFO Node{id=26}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.330223Z INFO Node{id=27}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.330308Z INFO Node{id=27}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.330359Z INFO Node{id=28}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.330436Z INFO Node{id=28}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.330493Z INFO Node{id=29}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.330608Z INFO Node{id=29}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.330662Z INFO Node{id=30}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.330744Z INFO Node{id=30}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.330817Z INFO Node{id=16}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.330891Z INFO Node{id=16}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.330936Z INFO Node{id=17}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.331015Z INFO Node{id=17}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.331061Z INFO Node{id=18}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.331133Z INFO Node{id=18}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.331187Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.331270Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.331329Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.331408Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.331455Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.331528Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.331583Z INFO Node{id=22}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.331666Z INFO Node{id=22}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.331720Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.331850Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.331905Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.331988Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.332070Z INFO Node{id=9}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.332168Z INFO Node{id=9}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.332221Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.332316Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.332699Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:52:10.332994Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:10.333014Z INFO Node{id=31}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:52:10.333024Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:10.333815Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:52:10.334137Z INFO Node{id=31}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:10.334231Z INFO Node{id=31}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:10.335452Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:52:10.336182Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:52:10.336326Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727530327) } [INFO] [stdout] 2025-09-01T11:52:10.336353Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=11 [INFO] [stdout] 2025-09-01T11:52:10.336584Z INFO Node{id=11}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:52:10.336611Z INFO Node{id=11}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.336817Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.336907Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:52:10.337367Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.337599Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:52:10.337665Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727530327) } [INFO] [stdout] 2025-09-01T11:52:10.337681Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=12 [INFO] [stdout] 2025-09-01T11:52:10.337959Z INFO Node{id=12}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=10 [INFO] [stdout] 2025-09-01T11:52:10.337984Z INFO Node{id=12}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.338210Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.338273Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:52:10.338530Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.338675Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:52:10.338722Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727530327) } [INFO] [stdout] 2025-09-01T11:52:10.338737Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=13 [INFO] [stdout] 2025-09-01T11:52:10.338995Z INFO Node{id=13}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:52:10.339021Z INFO Node{id=13}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.339501Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.339556Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:52:10.339935Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:52:10.339987Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeRecovery(RecoverableStateDetails { id: 12489568692394841378, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.340234Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.340478Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727530327) } [INFO] [stdout] 2025-09-01T11:52:10.340502Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=15 [INFO] [stdout] 2025-09-01T11:52:10.340542Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:52:10.340994Z INFO Node{id=15}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:52:10.341020Z INFO Node{id=15}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.341270Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:52:10.341339Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727530327) } [INFO] [stdout] 2025-09-01T11:52:10.341365Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=14 [INFO] [stdout] 2025-09-01T11:52:10.341683Z INFO Node{id=14}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=14 [INFO] [stdout] 2025-09-01T11:52:10.341710Z INFO Node{id=14}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.342037Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727530327) } [INFO] [stdout] 2025-09-01T11:52:10.342058Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=2 [INFO] [stdout] 2025-09-01T11:52:10.342099Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:52:10.342372Z INFO Node{id=2}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:52:10.342399Z INFO Node{id=2}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.342606Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:52:10.342656Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.342988Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.343221Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727530327) } [INFO] [stdout] 2025-09-01T11:52:10.343242Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=3 [INFO] [stdout] 2025-09-01T11:52:10.343291Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:52:10.343584Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:52:10.343611Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.343851Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:52:10.343897Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:10.344182Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(1): Added, allow_drop: true, next_sequence: 4, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:10.344251Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:52:10.344576Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4) [INFO] [stdout] 2025-09-01T11:52:10.344693Z INFO Node{id=11}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:10.344724Z INFO Node{id=11}: sequenced_broadcast: 49|SequencedBroadcastWorker Started id=49 [INFO] [stdout] 2025-09-01T11:52:10.344832Z INFO Node{id=11}: sequenced_broadcast: 25|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:10.344850Z INFO Node{id=11}: sequenced_broadcast: 25|SequencedBroadcastWorker Stopped id=25 elapsed=27.056633ms iter=2 [INFO] [stdout] 2025-09-01T11:52:10.345037Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.345108Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:52:10.345387Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.345836Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:52:10.345880Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727530327) } [INFO] [stdout] 2025-09-01T11:52:10.345898Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=5 [INFO] [stdout] 2025-09-01T11:52:10.346268Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727530327) } [INFO] [stdout] 2025-09-01T11:52:10.346297Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=4 [INFO] [stdout] 2025-09-01T11:52:10.346337Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:52:10.346527Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=19 [INFO] [stdout] 2025-09-01T11:52:10.346552Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.346644Z INFO Node{id=4}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=19 [INFO] [stdout] 2025-09-01T11:52:10.346667Z INFO Node{id=4}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.346958Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:52:10.347015Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.347463Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.347733Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeRecovery(RecoverableStateDetails { id: 118277545382215759, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.347813Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:52:10.348483Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:52:10.348538Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.348959Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.349268Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727530329) } [INFO] [stdout] 2025-09-01T11:52:10.349303Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=7 [INFO] [stdout] 2025-09-01T11:52:10.349351Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:52:10.349709Z INFO Node{id=7}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:52:10.349736Z INFO Node{id=7}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.350055Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:52:10.350106Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727530329) } [INFO] [stdout] 2025-09-01T11:52:10.350123Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=6 [INFO] [stdout] 2025-09-01T11:52:10.350406Z INFO Node{id=6}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=21 [INFO] [stdout] 2025-09-01T11:52:10.350441Z INFO Node{id=6}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.350746Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.350817Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:52:10.351149Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.351602Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:52:10.351643Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727530329) } [INFO] [stdout] 2025-09-01T11:52:10.351657Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=32 [INFO] [stdout] 2025-09-01T11:52:10.352042Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.352094Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=9 [INFO] [stdout] 2025-09-01T11:52:10.352296Z INFO Node{id=32}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=23 [INFO] [stdout] 2025-09-01T11:52:10.352320Z INFO Node{id=32}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.352413Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.352663Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:52:10.352710Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727530329) } [INFO] [stdout] 2025-09-01T11:52:10.352725Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=24 [INFO] [stdout] 2025-09-01T11:52:10.353112Z INFO Node{id=24}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=24 [INFO] [stdout] 2025-09-01T11:52:10.353137Z INFO Node{id=24}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.353351Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.353405Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:52:10.353780Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.354063Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeRecovery(RecoverableStateDetails { id: 82584403990137593, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.354333Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727530330) } [INFO] [stdout] 2025-09-01T11:52:10.354353Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=25 [INFO] [stdout] 2025-09-01T11:52:10.354511Z INFO Node{id=25}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=24 [INFO] [stdout] 2025-09-01T11:52:10.354535Z INFO Node{id=25}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.354792Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727530330) } [INFO] [stdout] 2025-09-01T11:52:10.354829Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=26 [INFO] [stdout] 2025-09-01T11:52:10.354919Z INFO Node{id=26}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=24 [INFO] [stdout] 2025-09-01T11:52:10.354942Z INFO Node{id=26}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.355089Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727530330) } [INFO] [stdout] 2025-09-01T11:52:10.355109Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=27 [INFO] [stdout] 2025-09-01T11:52:10.355182Z INFO Node{id=27}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=25 [INFO] [stdout] 2025-09-01T11:52:10.355203Z INFO Node{id=27}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.355386Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727530330) } [INFO] [stdout] 2025-09-01T11:52:10.355405Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=28 [INFO] [stdout] 2025-09-01T11:52:10.355479Z INFO Node{id=28}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=25 [INFO] [stdout] 2025-09-01T11:52:10.355500Z INFO Node{id=28}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.355645Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.355800Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.356064Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727530330) } [INFO] [stdout] 2025-09-01T11:52:10.356085Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=30 [INFO] [stdout] 2025-09-01T11:52:10.356191Z INFO Node{id=30}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=26 [INFO] [stdout] 2025-09-01T11:52:10.356214Z INFO Node{id=30}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.356383Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727530330) } [INFO] [stdout] 2025-09-01T11:52:10.356403Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=29 [INFO] [stdout] 2025-09-01T11:52:10.356481Z INFO Node{id=29}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=26 [INFO] [stdout] 2025-09-01T11:52:10.356503Z INFO Node{id=29}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.356661Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.356754Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.357038Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:10.357404Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(2): Added, allow_drop: true, next_sequence: 7, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:10.357715Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 7) [INFO] [stdout] 2025-09-01T11:52:10.357824Z INFO Node{id=12}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:10.357850Z INFO Node{id=12}: sequenced_broadcast: 66|SequencedBroadcastWorker Started id=66 [INFO] [stdout] 2025-09-01T11:52:10.357928Z INFO Node{id=12}: sequenced_broadcast: 26|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:10.357943Z INFO Node{id=12}: sequenced_broadcast: 26|SequencedBroadcastWorker Stopped id=26 elapsed=40.096759ms iter=2 [INFO] [stdout] 2025-09-01T11:52:10.358097Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.358390Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.359019Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeRecovery(RecoverableStateDetails { id: 17916318048569937296, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.359189Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727530331) } [INFO] [stdout] 2025-09-01T11:52:10.359208Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=17 [INFO] [stdout] 2025-09-01T11:52:10.359432Z INFO Node{id=17}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=28 [INFO] [stdout] 2025-09-01T11:52:10.359456Z INFO Node{id=17}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.359797Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727530330) } [INFO] [stdout] 2025-09-01T11:52:10.359819Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=16 [INFO] [stdout] 2025-09-01T11:52:10.359935Z INFO Node{id=16}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=29 [INFO] [stdout] 2025-09-01T11:52:10.359957Z INFO Node{id=16}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.360141Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.360394Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.360744Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727530331) } [INFO] [stdout] 2025-09-01T11:52:10.360788Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=18 [INFO] [stdout] 2025-09-01T11:52:10.360895Z INFO Node{id=18}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=29 [INFO] [stdout] 2025-09-01T11:52:10.360917Z INFO Node{id=18}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.361066Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.361196Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.361379Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeRecovery(RecoverableStateDetails { id: 10965062950730445307, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.361515Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727530331) } [INFO] [stdout] 2025-09-01T11:52:10.361539Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=19 [INFO] [stdout] 2025-09-01T11:52:10.361810Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=30 [INFO] [stdout] 2025-09-01T11:52:10.361835Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.362081Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.362218Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.362395Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727530331) } [INFO] [stdout] 2025-09-01T11:52:10.362415Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=20 [INFO] [stdout] 2025-09-01T11:52:10.362510Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=31 [INFO] [stdout] 2025-09-01T11:52:10.362531Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.362679Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.362840Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.362923Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727530331) } [INFO] [stdout] 2025-09-01T11:52:10.362939Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=21 [INFO] [stdout] 2025-09-01T11:52:10.363009Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=32 [INFO] [stdout] 2025-09-01T11:52:10.363029Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.363410Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.363599Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727530331) } [INFO] [stdout] 2025-09-01T11:52:10.363615Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=23 [INFO] [stdout] 2025-09-01T11:52:10.363746Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.363803Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=32 [INFO] [stdout] 2025-09-01T11:52:10.363820Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.364104Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeRecovery(RecoverableStateDetails { id: 17249428599497467871, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.364276Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727530331) } [INFO] [stdout] 2025-09-01T11:52:10.364300Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=22 [INFO] [stdout] 2025-09-01T11:52:10.364441Z INFO Node{id=22}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=33 [INFO] [stdout] 2025-09-01T11:52:10.364507Z INFO Node{id=22}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.364689Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.364828Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.364983Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.365151Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.365416Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: Ping(1756727530332) } [INFO] [stdout] 2025-09-01T11:52:10.365434Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=9 [INFO] [stdout] 2025-09-01T11:52:10.365523Z INFO Node{id=9}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=33 [INFO] [stdout] 2025-09-01T11:52:10.365544Z INFO Node{id=9}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.365683Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727530331) } [INFO] [stdout] 2025-09-01T11:52:10.365715Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=8 [INFO] [stdout] 2025-09-01T11:52:10.365808Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=34 [INFO] [stdout] 2025-09-01T11:52:10.365829Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.365984Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:10.366053Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(3): Added, allow_drop: true, next_sequence: 13, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:10.366295Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 13) [INFO] [stdout] 2025-09-01T11:52:10.366380Z INFO Node{id=13}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:10.366416Z INFO Node{id=13}: sequenced_broadcast: 67|SequencedBroadcastWorker Started id=67 [INFO] [stdout] 2025-09-01T11:52:10.366493Z INFO Node{id=13}: sequenced_broadcast: 27|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:10.366508Z INFO Node{id=13}: sequenced_broadcast: 27|SequencedBroadcastWorker Stopped id=27 elapsed=48.613347ms iter=2 [INFO] [stdout] 2025-09-01T11:52:10.366725Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeRecovery(RecoverableStateDetails { id: 5566833424272702303, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.366865Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727530332) } [INFO] [stdout] 2025-09-01T11:52:10.366890Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=10 [INFO] [stdout] 2025-09-01T11:52:10.367086Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=35 [INFO] [stdout] 2025-09-01T11:52:10.367109Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.367310Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.367485Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.367688Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727530334) } [INFO] [stdout] 2025-09-01T11:52:10.367708Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=31 [INFO] [stdout] 2025-09-01T11:52:10.367871Z INFO Node{id=31}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=33 [INFO] [stdout] 2025-09-01T11:52:10.367895Z INFO Node{id=31}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:10.368094Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.368249Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.368436Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.368626Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.368861Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.369032Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.369490Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.369751Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeRecovery(RecoverableStateDetails { id: 12240540419735701133, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.369872Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.369987Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.370230Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.370440Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.370693Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.370984Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.371186Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.371807Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeRecovery(RecoverableStateDetails { id: 4141255820887668551, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.371897Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:10.371973Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(4): Added, allow_drop: true, next_sequence: 16, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:10.372259Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 16) [INFO] [stdout] 2025-09-01T11:52:10.372354Z INFO Node{id=15}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:10.372394Z INFO Node{id=15}: sequenced_broadcast: 68|SequencedBroadcastWorker Started id=68 [INFO] [stdout] 2025-09-01T11:52:10.372472Z INFO Node{id=15}: sequenced_broadcast: 29|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:10.372485Z INFO Node{id=15}: sequenced_broadcast: 29|SequencedBroadcastWorker Stopped id=29 elapsed=54.486945ms iter=2 [INFO] [stdout] 2025-09-01T11:52:10.373174Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.373381Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.373773Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.373982Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.375913Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.376161Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.376308Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.376347Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.376552Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:10.377054Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(20) < seq(21) [INFO] [stdout] 2025-09-01T11:52:10.388337Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(5): Added, allow_drop: true, next_sequence: 21, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:10.388550Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 21) [INFO] [stdout] 2025-09-01T11:52:10.388650Z INFO Node{id=14}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:10.388683Z INFO Node{id=14}: sequenced_broadcast: 72|SequencedBroadcastWorker Started id=72 [INFO] [stdout] 2025-09-01T11:52:10.388786Z INFO Node{id=14}: sequenced_broadcast: 28|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:10.388803Z INFO Node{id=14}: sequenced_broadcast: 28|SequencedBroadcastWorker Stopped id=28 elapsed=70.856881ms iter=2 [INFO] [stdout] 2025-09-01T11:52:10.388988Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeRecovery(RecoverableStateDetails { id: 8943843644095261925, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.389556Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.400007Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.400816Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.405681Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.409100Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.409701Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeRecovery(RecoverableStateDetails { id: 7453206412063911533, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.410075Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.410220Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.410484Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.412371Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.412511Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.412740Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.426230Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.427129Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:10.427853Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(6): Added, allow_drop: true, next_sequence: 25, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:10.433120Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 25) [INFO] [stdout] 2025-09-01T11:52:10.433259Z INFO Node{id=2}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:10.433305Z INFO Node{id=2}: sequenced_broadcast: 74|SequencedBroadcastWorker Started id=74 [INFO] [stdout] 2025-09-01T11:52:10.433703Z INFO Node{id=2}: sequenced_broadcast: 16|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:10.433721Z INFO Node{id=2}: sequenced_broadcast: 16|SequencedBroadcastWorker Stopped id=16 elapsed=116.552938ms iter=2 [INFO] [stdout] 2025-09-01T11:52:10.434209Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SubscribeRecovery(RecoverableStateDetails { id: 8976929815698306089, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.438887Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.440549Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.456241Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.456703Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeRecovery(RecoverableStateDetails { id: 6938180368232146558, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.457629Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.458404Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.459448Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.479158Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:10.479701Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.480560Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.480890Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(30) < seq(31) [INFO] [stdout] 2025-09-01T11:52:10.509524Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(7): Added, allow_drop: true, next_sequence: 31, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:10.516489Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.540738Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 31) [INFO] [stdout] 2025-09-01T11:52:10.540889Z INFO Node{id=3}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:10.541344Z INFO Node{id=3}: sequenced_broadcast: 75|SequencedBroadcastWorker Started id=75 [INFO] [stdout] 2025-09-01T11:52:10.541599Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.541875Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.542673Z INFO Node{id=3}: sequenced_broadcast: 17|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:10.542691Z INFO Node{id=3}: sequenced_broadcast: 17|SequencedBroadcastWorker Stopped id=17 elapsed=225.459079ms iter=2 [INFO] [stdout] 2025-09-01T11:52:10.560577Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.561441Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:10.580139Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SubscribeRecovery(RecoverableStateDetails { id: 3236247324298727552, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.580424Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:52:10.581422Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.593711Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.598266Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.610366Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeRecovery(RecoverableStateDetails { id: 8521716953515948609, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.610842Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.611085Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.636210Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.636438Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeRecovery(RecoverableStateDetails { id: 12331897904907472227, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.648176Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:10.653991Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(44) < seq(47) [INFO] [stdout] 2025-09-01T11:52:10.686539Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(46) < seq(47) [INFO] [stdout] 2025-09-01T11:52:10.687261Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 47, max: 46 } [INFO] [stdout] 2025-09-01T11:52:10.687287Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:10.711215Z ERROR Node{id=5}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:10.715187Z WARN Node{id=5}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:10.715214Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:10.715631Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 2920180286697105469, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.715998Z INFO Node{id=5}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:10.716595Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.735388Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.736282Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.740791Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.743353Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SubscribeRecovery(RecoverableStateDetails { id: 13045332854218029016, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.747506Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeRecovery(RecoverableStateDetails { id: 2950095792267309118, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.749962Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:10.751279Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(56) < seq(59) [INFO] [stdout] 2025-09-01T11:52:10.767505Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(8): Added, allow_drop: true, next_sequence: 59, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:10.770852Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.775940Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 59) [INFO] [stdout] 2025-09-01T11:52:10.776594Z INFO Node{id=4}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:10.777206Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.777309Z INFO Node{id=4}: sequenced_broadcast: 76|SequencedBroadcastWorker Started id=76 [INFO] [stdout] 2025-09-01T11:52:10.778028Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeRecovery(RecoverableStateDetails { id: 1789406345984957020, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.778635Z INFO Node{id=4}: sequenced_broadcast: 18|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:10.778654Z INFO Node{id=4}: sequenced_broadcast: 18|SequencedBroadcastWorker Stopped id=18 elapsed=461.357856ms iter=2 [INFO] [stdout] 2025-09-01T11:52:10.781990Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.782606Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.784142Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeRecovery(RecoverableStateDetails { id: 2026628811775300967, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.794272Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.804438Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.805881Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SubscribeRecovery(RecoverableStateDetails { id: 13762996665778444007, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.836231Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:10.836739Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(78) < seq(81) [INFO] [stdout] 2025-09-01T11:52:10.849385Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(9): Added, allow_drop: true, next_sequence: 81, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:10.850776Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 81) [INFO] [stdout] 2025-09-01T11:52:10.850925Z INFO Node{id=7}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:10.851332Z INFO Node{id=7}: sequenced_broadcast: 77|SequencedBroadcastWorker Started id=77 [INFO] [stdout] 2025-09-01T11:52:10.852032Z INFO Node{id=7}: sequenced_broadcast: 21|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:10.852051Z INFO Node{id=7}: sequenced_broadcast: 21|SequencedBroadcastWorker Stopped id=21 elapsed=534.562006ms iter=2 [INFO] [stdout] 2025-09-01T11:52:10.865941Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.871985Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.877958Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeRecovery(RecoverableStateDetails { id: 3017628437321749879, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:10.881883Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:10.882195Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(90) < seq(94) [INFO] [stdout] 2025-09-01T11:52:10.930064Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(10): Added, allow_drop: true, next_sequence: 94, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:10.943039Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.954138Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:10.954358Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 94) [INFO] [stdout] 2025-09-01T11:52:10.954496Z INFO Node{id=6}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:10.954553Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.969250Z INFO Node{id=6}: sequenced_broadcast: 78|SequencedBroadcastWorker Started id=78 [INFO] [stdout] 2025-09-01T11:52:10.974686Z INFO Node{id=6}: sequenced_broadcast: 20|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:10.974714Z INFO Node{id=6}: sequenced_broadcast: 20|SequencedBroadcastWorker Stopped id=20 elapsed=657.283923ms iter=2 [INFO] [stdout] 2025-09-01T11:52:10.979794Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:10.997359Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:11.008656Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeRecovery(RecoverableStateDetails { id: 8084228702317741667, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:11.021560Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:11.025335Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeRecovery(RecoverableStateDetails { id: 2604719619735459382, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:11.043188Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeRecovery(RecoverableStateDetails { id: 17456534934631183880, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:11.117139Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:11.126576Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(112) < seq(115) [INFO] [stdout] 2025-09-01T11:52:11.174702Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(11): Added, allow_drop: true, next_sequence: 115, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:11.222648Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 115) [INFO] [stdout] 2025-09-01T11:52:11.222800Z INFO Node{id=32}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:11.237496Z INFO Node{id=32}: sequenced_broadcast: 79|SequencedBroadcastWorker Started id=79 [INFO] [stdout] 2025-09-01T11:52:11.239018Z INFO Node{id=32}: sequenced_broadcast: 46|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:11.239040Z INFO Node{id=32}: sequenced_broadcast: 46|SequencedBroadcastWorker Stopped id=46 elapsed=919.964413ms iter=2 [INFO] [stdout] 2025-09-01T11:52:11.272591Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:11.299656Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(126) < seq(128) [INFO] [stdout] 2025-09-01T11:52:11.354456Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(12): Added, allow_drop: true, next_sequence: 128, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:11.368525Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:52:11.368642Z INFO Node{id=1}: sharedstate::worker::sync_manager: Broadcasted 31 peers to 31 peers [INFO] [stdout] 2025-09-01T11:52:11.402617Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.402659Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.402671Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.402679Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.402690Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.402698Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.402705Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.402713Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.402723Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.402731Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.402739Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.402746Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.402754Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.408299Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.411796Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.411834Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.411844Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.411851Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.411858Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.411866Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.411873Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.411880Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.411887Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.411899Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.411906Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.411913Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.411920Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.411928Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.411935Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.411955Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.411963Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.411971Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.412040Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.412060Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.412069Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.412077Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.412087Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.412094Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.412102Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.412110Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.412121Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.412128Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.412135Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.412143Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.412150Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.412158Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.412165Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.412177Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.412185Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.412192Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.412199Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.412206Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.412213Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.412221Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.412228Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.412236Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.412243Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.412250Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.412257Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.412265Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.412278Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.412300Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.412309Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.412317Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.412353Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.412366Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.412374Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.412382Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.412391Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.412398Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.412405Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.412413Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.412423Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.412431Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.412438Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.412446Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.412453Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.412460Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.412467Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.412479Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.412486Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.412494Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.412502Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.412509Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.412516Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.412524Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.412539Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.412548Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.412556Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.412563Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.412571Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.412580Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.412588Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.412604Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.412612Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.412620Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.423141Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 128) [INFO] [stdout] 2025-09-01T11:52:11.423308Z INFO Node{id=24}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:11.423358Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.423382Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.423394Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.423403Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.423414Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.423423Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.423431Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.423440Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.423451Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.423459Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.423467Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.423476Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.423484Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.423492Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.423500Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.423514Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.423522Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.423538Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.423547Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.423556Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.423564Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.423572Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.423580Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.423588Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.423596Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.423603Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.423611Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.423618Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.423626Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.423649Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.423658Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.423667Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.423707Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.423725Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.423734Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.423743Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.423779Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.423790Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.423798Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.423805Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.423817Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.423825Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.423832Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.423839Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.423846Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.423853Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.423861Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.423873Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.423881Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.423888Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.423896Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.423903Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.423911Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.423918Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.423926Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.423933Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.423940Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.423948Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.423956Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.423963Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.423971Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.423992Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.424000Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.424008Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.424039Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.424052Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.424060Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.424067Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.424076Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.424084Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.424091Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.424099Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.424109Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.424122Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.424130Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.424137Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.424145Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.424153Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.424160Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.424172Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.424180Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.424188Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.424195Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.424202Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.424209Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.424217Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.424224Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.424231Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.424238Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.424245Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.424252Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.424260Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.424267Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.424288Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.424298Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.424305Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.424338Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.424351Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.424359Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.424366Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.424376Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.424383Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.424390Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.424398Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.424408Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.424415Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.424423Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.424430Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.424437Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.424445Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.424452Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.424464Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.424471Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.424479Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.424486Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.424494Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.424501Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.424508Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.424515Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.424522Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.424538Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.424546Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.424554Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.424561Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.424568Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.424589Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.424597Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.424604Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.424627Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.424637Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.424645Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.424653Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.424663Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.424670Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.424678Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.424686Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.424696Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.424705Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.424713Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.424722Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.424730Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.424738Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.424894Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.424953Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.424963Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.424971Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.424978Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.424986Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.424994Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.425002Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.425009Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.425017Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.425024Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.425032Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.425040Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.425048Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.425055Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.425079Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.425088Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.425095Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.425128Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.425143Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.425151Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.425159Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.425170Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.425178Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.425185Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.425193Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.425203Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.425211Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.425218Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.425226Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.425234Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.425242Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.425250Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.425262Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.425271Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.425367Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.425380Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.425389Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.425396Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.425404Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.425412Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.425420Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.425427Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.425435Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.425442Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.425449Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.425456Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.425474Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.425482Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.425491Z INFO Node{id=10}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.425534Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.425551Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.425565Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.425573Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.425583Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.425591Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.425599Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.425606Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.425616Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.425624Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.425631Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.425639Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.425648Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.425655Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.425663Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.425715Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.425727Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.425735Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.425744Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.425752Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.425780Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.425788Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.425795Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.425802Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.425810Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.425818Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.425825Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.425833Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.425840Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.425859Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.425867Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.425875Z INFO Node{id=21}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.425910Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.425927Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.425936Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.425944Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.425953Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.425961Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.425969Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.425982Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.425993Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.426001Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.426009Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.426018Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.426026Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.426034Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.426041Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.426055Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.426063Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.426070Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.426078Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.426085Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.426093Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.426100Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.426108Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.426115Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.426123Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.426131Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.426138Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.426145Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.426153Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.426171Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.426179Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.426187Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.426218Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.426230Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.426238Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.426246Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.426255Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.426263Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.426271Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.426280Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.426291Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.426299Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.426306Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.426314Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.426328Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.426336Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.426344Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.426357Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.426365Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.426372Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.426380Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.426387Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.426395Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.426403Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.426410Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.426419Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.426427Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.426434Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.426442Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.426450Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.426458Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.426478Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.426486Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.426494Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.426538Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.426554Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.426563Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.426571Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.426580Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.426589Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.426598Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.426605Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.426615Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.426623Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.426630Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.426638Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.426646Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.426655Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.426663Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.426675Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.426683Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.426696Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.426704Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.426712Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.426719Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.426727Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.426735Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.426742Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.426750Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.434983Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.435022Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.435032Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.435041Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.435066Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.435074Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.435082Z INFO Node{id=23}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.435136Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.435157Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.435168Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.435177Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.435188Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.435197Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.435205Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.435213Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.435225Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.435233Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.435241Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.435250Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.435281Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.435290Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.435298Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.435312Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.435320Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.435328Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.435337Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.435345Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.435353Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.435362Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.435379Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.435388Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.435396Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.435405Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.435412Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.435420Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.435428Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.435448Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.435456Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.435464Z INFO Node{id=20}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.435502Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.435520Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.435571Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.435591Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.435603Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.435613Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.435621Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.435629Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.435640Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.435648Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.435656Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.435663Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.435672Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.435679Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.435688Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.435701Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.435709Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.435717Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.435725Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.435733Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.435741Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.435748Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.439790Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.439816Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.439825Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.439833Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.439840Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.439854Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.439861Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.439881Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.439888Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.439896Z INFO Node{id=8}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.439946Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.439962Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.439971Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.439979Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.439989Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.439996Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.440004Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.440012Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.440022Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.440030Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.440038Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.440045Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.440052Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.440060Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.440068Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.440080Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.440089Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.440097Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.440105Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.440113Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.440121Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.440129Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.440137Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.440144Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.440151Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.440159Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.440166Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.440174Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.440182Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.440200Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.440208Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.440215Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.440247Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.440261Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.440269Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.440277Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.440287Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.440295Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.440302Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.440310Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.440319Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.440326Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.440334Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.440340Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.440347Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.440354Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.440361Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.440373Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.440380Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.440387Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.440394Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.440401Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.440412Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.440419Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.440426Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.440433Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.440440Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.440447Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.440453Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.440461Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.440468Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.440485Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.440492Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.440499Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.440532Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.440545Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.440553Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.440559Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.440574Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.440582Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.440589Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.440596Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.440605Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.440612Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.440619Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.440626Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.440633Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.440639Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.440646Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.440657Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.440664Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.440672Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.440679Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.440686Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.440693Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.440700Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.440707Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.440714Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.440722Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.440728Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.440735Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.440742Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.440749Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.440785Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.440794Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.440802Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.440834Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.440846Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.440853Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.440861Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.440870Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.440877Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.440885Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.440893Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.440903Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.440914Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.440922Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.440929Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.440937Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.440944Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.440951Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.440963Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.440970Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.440978Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.440986Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.440993Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.441000Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.441007Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.441014Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.441021Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.441028Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.441036Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.441044Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.441052Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.441059Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.441077Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.441084Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.441092Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.441120Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.441134Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.441142Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.441149Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.441158Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.441165Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.441172Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.441180Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.441190Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.441198Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.441206Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.441214Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.441221Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.441229Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.441241Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.441253Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.441260Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.441267Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.441274Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.441282Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.441291Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.441297Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.441305Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.441313Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.441321Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.441330Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.441337Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.441344Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.441353Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.441372Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.441380Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.441388Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.441419Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.441434Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.441443Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.441451Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.441461Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.441469Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.441477Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.441485Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.441497Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.441505Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.441512Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.441519Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.441534Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.441542Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.441549Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.441560Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.441568Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.441575Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.441583Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.441594Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.441601Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.441608Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.441615Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.441623Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.441630Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.441637Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.441645Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.441652Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.441660Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.441678Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.441685Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.441692Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.453395Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.453437Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.453449Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.453457Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.453468Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.453475Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.453482Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.453489Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.453499Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.453507Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.453514Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.453521Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.453536Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.453543Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.453550Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.453563Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.453570Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.453577Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.453584Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.453591Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.453598Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.453605Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.453612Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.453619Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.453633Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.453641Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.453647Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.453654Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.453661Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.453680Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.453687Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.453694Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.459156Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.459252Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.459266Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.459274Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.459286Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.459295Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.459302Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.459309Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.459321Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.459328Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.459336Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.459343Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.459350Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.459358Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.459366Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.459380Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.459388Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.459396Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.459404Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.459412Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.459419Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.459427Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.459434Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.459442Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.459450Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.459457Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.459464Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.459472Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.459479Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.459505Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.459515Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.459522Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.459591Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.459607Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.459615Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.459624Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.459633Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.459641Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.459648Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.459656Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.459666Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.459674Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.459681Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.459689Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.459696Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.459704Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.459711Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.459724Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.459731Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.459739Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.459746Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.459754Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.459781Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.459790Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.459798Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.459805Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.459813Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.459820Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.459827Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.459835Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.459843Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.459862Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.459869Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.459877Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.459905Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.459918Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.459940Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.459948Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.459957Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.459965Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.459973Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.459981Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.459991Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.459999Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.460008Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.460016Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.460023Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.460031Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.460039Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.460050Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.460058Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.460065Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.460073Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.460080Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.460087Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.460094Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.460102Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.460109Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.460116Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.460123Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.460130Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.460137Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.460145Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.460164Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.460173Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.460181Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.460216Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.460232Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.460242Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.460250Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.460261Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.460270Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.460286Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.460295Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.460306Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.460314Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.460321Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.460329Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.460337Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.460345Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.460354Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.460367Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.460376Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.460383Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.460391Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.460399Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.460407Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.460416Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.460423Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.460431Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.460439Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.460446Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.460454Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.460461Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.460469Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.460487Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.460495Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.460502Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.469710Z INFO Node{id=24}: sequenced_broadcast: 80|SequencedBroadcastWorker Started id=80 [INFO] [stdout] 2025-09-01T11:52:11.470132Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.470163Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.470176Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.470185Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.470196Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.470204Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.470211Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.470220Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.470231Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.470239Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.470255Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.470264Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.470272Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.470280Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.470288Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.470302Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.470310Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.470318Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.470327Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.470334Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.470342Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.470350Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.470358Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.470365Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.470373Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.470380Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.470387Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.470395Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.470402Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.470420Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.470428Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.470436Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.480720Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.488970Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.489020Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.489030Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.489041Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.489049Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.489056Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.489064Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.489074Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.489081Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.489089Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.489096Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.489103Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.489111Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.489119Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.489140Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.489147Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.489155Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.489162Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.489168Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.489175Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.489182Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.489189Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.489195Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.489203Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.489210Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.489217Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.489225Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.489233Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.489251Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.489259Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.489267Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.489862Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.489886Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.489895Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.489903Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.489912Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.489919Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.489926Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.489934Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.489943Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.489951Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.489958Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.489965Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.489972Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.489979Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.489987Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.489999Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.490007Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.490015Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.490023Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.490031Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.490047Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.490056Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.490063Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.490071Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.490079Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.490087Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.490095Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.490103Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.490110Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.490128Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.490136Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.490144Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.490182Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:11.490199Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:52:11.490208Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:52:11.490216Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:52:11.490225Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:52:11.490232Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:52:11.490241Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:52:11.490249Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:52:11.490261Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:52:11.490268Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:52:11.490275Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:52:11.490284Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:52:11.490292Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:52:11.490299Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:52:11.490307Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:52:11.490319Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:52:11.490328Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:52:11.490335Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:52:11.490343Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:52:11.490350Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:52:11.490358Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:52:11.490366Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:52:11.490374Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:52:11.490383Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:52:11.490391Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:52:11.490406Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:52:11.490414Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:52:11.490421Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:52:11.490429Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:52:11.490447Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:52:11.490456Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:52:11.490464Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:52:11.490589Z INFO Node{id=24}: sequenced_broadcast: 38|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:11.490609Z INFO Node{id=24}: sequenced_broadcast: 38|SequencedBroadcastWorker Stopped id=38 elapsed=1.172048055s iter=2 [INFO] [stdout] 2025-09-01T11:52:11.528513Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:11.566796Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:11.576633Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeRecovery(RecoverableStateDetails { id: 6302458865832610014, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:11.597325Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:52:11.614601Z ERROR Node{id=10}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:11.622442Z WARN Node{id=10}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:11.622472Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:11.640936Z INFO Node{id=10}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:11.736011Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:11.736058Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=1 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:11.754487Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:11.754574Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:11.762552Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:52:11.772098Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:11.786097Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:11.845326Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:11.867641Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:11.878349Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeRecovery(RecoverableStateDetails { id: 5502544115740205408, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:11.892993Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeRecovery(RecoverableStateDetails { id: 2409269800524822834, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:12.155549Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:12.324721Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeRecovery(RecoverableStateDetails { id: 16065943122104467468, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:12.337469Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:12.349590Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(201) < seq(203) [INFO] [stdout] 2025-09-01T11:52:12.422829Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(13): Added, allow_drop: true, next_sequence: 203, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:12.455958Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 203) [INFO] [stdout] 2025-09-01T11:52:12.456080Z INFO Node{id=25}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:12.471911Z INFO Node{id=25}: sequenced_broadcast: 81|SequencedBroadcastWorker Started id=81 [INFO] [stdout] 2025-09-01T11:52:12.481755Z INFO Node{id=25}: sequenced_broadcast: 39|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:12.492847Z INFO Node{id=25}: sequenced_broadcast: 39|SequencedBroadcastWorker Stopped id=39 elapsed=2.174218275s iter=2 [INFO] [stdout] 2025-09-01T11:52:12.571510Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:12.588213Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeRecovery(RecoverableStateDetails { id: 16397677385429040004, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:12.592062Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:12.628026Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:12.628071Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:52:12.636608Z INFO Node{id=10}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:12.636711Z INFO Node{id=10}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:12.647210Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:52:12.668331Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:12.688053Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(231) < seq(233) [INFO] [stdout] 2025-09-01T11:52:12.706813Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727532636) } [INFO] [stdout] 2025-09-01T11:52:12.723100Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(14): Added, allow_drop: true, next_sequence: 233, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:12.734742Z INFO Node{id=10}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=98 [INFO] [stdout] 2025-09-01T11:52:12.734806Z INFO Node{id=10}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:12.764264Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 233) [INFO] [stdout] 2025-09-01T11:52:12.764397Z INFO Node{id=26}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:12.771876Z INFO Node{id=26}: sequenced_broadcast: 82|SequencedBroadcastWorker Started id=82 [INFO] [stdout] 2025-09-01T11:52:12.778377Z INFO Node{id=26}: sequenced_broadcast: 40|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:12.778409Z INFO Node{id=26}: sequenced_broadcast: 40|SequencedBroadcastWorker Stopped id=40 elapsed=2.459735098s iter=2 [INFO] [stdout] 2025-09-01T11:52:12.786229Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:12.810209Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:52:12.842183Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:12.844741Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(247) < seq(251) [INFO] [stdout] 2025-09-01T11:52:12.851569Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:12.859728Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeRecovery(RecoverableStateDetails { id: 16065943122104467468, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:12.863456Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:12.865570Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(15): Added, allow_drop: true, next_sequence: 251, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:12.869217Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:12.873583Z INFO Node{id=12}: sequenced_broadcast: 66|Subscriber rejected, max_seq(238) < seq(241) [INFO] [stdout] 2025-09-01T11:52:12.876343Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 251) [INFO] [stdout] 2025-09-01T11:52:12.876508Z INFO Node{id=27}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:12.915883Z INFO Node{id=27}: sequenced_broadcast: 83|SequencedBroadcastWorker Started id=83 [INFO] [stdout] 2025-09-01T11:52:12.927384Z INFO Node{id=12}: sequenced_broadcast: 66|Subscriber rejected, max_seq(240) < seq(241) [INFO] [stdout] 2025-09-01T11:52:12.927593Z INFO Node{id=27}: sequenced_broadcast: 41|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:12.927614Z INFO Node{id=27}: sequenced_broadcast: 41|SequencedBroadcastWorker Stopped id=41 elapsed=2.608886938s iter=2 [INFO] [stdout] 2025-09-01T11:52:12.931666Z ERROR Node{id=12}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 241, max: 240 } [INFO] [stdout] 2025-09-01T11:52:12.931698Z ERROR Node{id=12}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:12.939334Z ERROR Node{id=10}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:12.945385Z WARN Node{id=10}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:12.945414Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:12.952383Z INFO Node{id=10}:connect{target=12}:WritePeer{remote=12}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:13.509643Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:13.511242Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(278) < seq(283) [INFO] [stdout] 2025-09-01T11:52:13.562669Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(281) < seq(283) [INFO] [stdout] 2025-09-01T11:52:13.593551Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 283, max: 281 } [INFO] [stdout] 2025-09-01T11:52:13.593591Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:13.618384Z ERROR Node{id=28}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:13.633416Z WARN Node{id=28}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:13.633451Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:13.652641Z INFO Node{id=28}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:13.989472Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:13.989522Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:52:13.996749Z INFO Node{id=10}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:14.029250Z INFO Node{id=10}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:14.031978Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:52:14.086501Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:14.096359Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(312) < seq(315) [INFO] [stdout] 2025-09-01T11:52:14.163010Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(16): Added, allow_drop: true, next_sequence: 315, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:14.182841Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727534029) } [INFO] [stdout] 2025-09-01T11:52:14.224060Z INFO Node{id=10}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=195 [INFO] [stdout] 2025-09-01T11:52:14.224104Z INFO Node{id=10}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:14.252863Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 315) [INFO] [stdout] 2025-09-01T11:52:14.252986Z INFO Node{id=30}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:14.260293Z ERROR Node{id=22}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:52:14.261090Z INFO Node{id=30}: sequenced_broadcast: 84|SequencedBroadcastWorker Started id=84 [INFO] [stdout] 2025-09-01T11:52:14.262827Z WARN Node{id=22}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:14.262848Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:14.293255Z INFO Node{id=30}: sequenced_broadcast: 44|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:14.293293Z INFO Node{id=30}: sequenced_broadcast: 44|SequencedBroadcastWorker Stopped id=44 elapsed=3.974370561s iter=2 [INFO] [stdout] 2025-09-01T11:52:14.306010Z INFO Node{id=22}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:14.319349Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:14.333883Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:14.334385Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:14.351555Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(322) < seq(326) [INFO] [stdout] 2025-09-01T11:52:14.377681Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(324) < seq(326) [INFO] [stdout] 2025-09-01T11:52:14.400665Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 326, max: 324 } [INFO] [stdout] 2025-09-01T11:52:14.400706Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:14.403843Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:52:14.403956Z ERROR Node{id=10}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=22 leader=1 local=10 [INFO] [stdout] 2025-09-01T11:52:14.422172Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:14.422248Z INFO Node{id=10}:connect{target=22}:WritePeer{remote=22}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:14.447378Z ERROR Node{id=29}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:14.447495Z INFO Node{id=10}:connect{target=22}:ReadPeer{remote=22}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:14.447652Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:14.481194Z WARN Node{id=29}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:14.481230Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:14.497824Z INFO Node{id=29}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:14.497957Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:14.653804Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:14.653927Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:52:14.670069Z INFO Node{id=28}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:14.670177Z INFO Node{id=28}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:14.692755Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:52:14.843193Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727534670) } [INFO] [stdout] 2025-09-01T11:52:14.901526Z INFO Node{id=28}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=231 [INFO] [stdout] 2025-09-01T11:52:14.901592Z INFO Node{id=28}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:15.021069Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:15.112386Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:52:15.112420Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=8 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:52:15.141299Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:15.141381Z INFO Node{id=28}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:15.173032Z INFO Node{id=28}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:15.210870Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:15.225112Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:15.299938Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:15.299997Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:52:15.307589Z INFO Node{id=22}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:15.307677Z INFO Node{id=22}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:15.334001Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:52:15.398799Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:15.433433Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(363) < seq(366) [INFO] [stdout] 2025-09-01T11:52:15.450874Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:15.450915Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:52:15.489663Z INFO Node{id=10}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:15.489790Z INFO Node{id=10}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:15.490513Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:15.490545Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:52:15.503582Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:52:15.503719Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(17): Added, allow_drop: true, next_sequence: 366, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:15.504241Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727535307) } [INFO] [stdout] 2025-09-01T11:52:15.504744Z INFO Node{id=29}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:15.548937Z INFO Node{id=29}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:15.551528Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:52:15.581572Z INFO Node{id=22}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=274 [INFO] [stdout] 2025-09-01T11:52:15.581609Z INFO Node{id=22}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:15.652716Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 366) [INFO] [stdout] 2025-09-01T11:52:15.652870Z INFO Node{id=17}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:15.681670Z INFO Node{id=17}: sequenced_broadcast: 85|SequencedBroadcastWorker Started id=85 [INFO] [stdout] 2025-09-01T11:52:15.701473Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727535489) } [INFO] [stdout] 2025-09-01T11:52:15.704262Z INFO Node{id=17}: sequenced_broadcast: 31|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:15.704295Z INFO Node{id=17}: sequenced_broadcast: 31|SequencedBroadcastWorker Stopped id=31 elapsed=5.386162031s iter=2 [INFO] [stdout] 2025-09-01T11:52:15.759777Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727535548) } [INFO] [stdout] 2025-09-01T11:52:15.768645Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:15.784338Z INFO Node{id=10}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=295 [INFO] [stdout] 2025-09-01T11:52:15.784384Z INFO Node{id=10}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:15.823656Z INFO Node{id=29}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=275 [INFO] [stdout] 2025-09-01T11:52:15.823700Z INFO Node{id=29}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:15.867373Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:52:15.889987Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:15.906840Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:15.941701Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(381) < seq(383) [INFO] [stdout] 2025-09-01T11:52:15.942683Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:15.969585Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:52:15.969913Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:15.990468Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeRecovery(RecoverableStateDetails { id: 6302458865832610014, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:16.012182Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:52:16.013641Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:16.013712Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:16.021517Z INFO Node{id=3}: sequenced_broadcast: 75|Subscriber rejected, max_seq(369) < seq(373) [INFO] [stdout] 2025-09-01T11:52:16.021588Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(18): Added, allow_drop: true, next_sequence: 383, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:16.059876Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:16.073027Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:16.083190Z INFO Node{id=3}: sequenced_broadcast: 75|Subscriber rejected, max_seq(372) < seq(373) [INFO] [stdout] 2025-09-01T11:52:16.104576Z ERROR Node{id=3}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 373, max: 372 } [INFO] [stdout] 2025-09-01T11:52:16.104614Z ERROR Node{id=3}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:16.105327Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:16.105725Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeRecovery(RecoverableStateDetails { id: 16065943122104467468, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:16.106066Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeRecovery(RecoverableStateDetails { id: 2950095792267309118, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:16.106399Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 383) [INFO] [stdout] 2025-09-01T11:52:16.106517Z INFO Node{id=16}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:16.114736Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:16.114845Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:16.115545Z INFO Node{id=16}: sequenced_broadcast: 86|SequencedBroadcastWorker Started id=86 [INFO] [stdout] 2025-09-01T11:52:16.139978Z ERROR Node{id=22}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:16.145946Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:16.146400Z INFO Node{id=16}: sequenced_broadcast: 30|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:16.146422Z INFO Node{id=16}: sequenced_broadcast: 30|SequencedBroadcastWorker Stopped id=30 elapsed=5.828370342s iter=2 [INFO] [stdout] 2025-09-01T11:52:16.147331Z WARN Node{id=22}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:16.147349Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:16.148093Z INFO Node{id=30}: sequenced_broadcast: 84|Subscriber rejected, max_seq(380) < seq(382) [INFO] [stdout] 2025-09-01T11:52:16.148929Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:16.148958Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:52:16.149644Z INFO Node{id=22}:connect{target=3}:WritePeer{remote=3}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:16.173948Z INFO Node{id=32}: sequenced_broadcast: 79|Subscriber rejected, max_seq(378) < seq(380) [INFO] [stdout] 2025-09-01T11:52:16.177369Z INFO Node{id=28}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:16.177451Z INFO Node{id=28}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:16.222170Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:52:16.255438Z INFO Node{id=32}: sequenced_broadcast: 79|Subscriber rejected, max_seq(379) < seq(380) [INFO] [stdout] 2025-09-01T11:52:16.292579Z ERROR Node{id=32}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 380, max: 379 } [INFO] [stdout] 2025-09-01T11:52:16.293961Z ERROR Node{id=32}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:16.301960Z INFO Node{id=30}: sequenced_broadcast: 84|Subscriber(1): Added, allow_drop: true, next_sequence: 382, min_allowed_seq: 315 [INFO] [stdout] 2025-09-01T11:52:16.361983Z ERROR Node{id=29}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:16.377344Z WARN Node{id=29}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:16.377379Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:16.389635Z INFO Node{id=29}:connect{target=32}:WritePeer{remote=32}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:16.402390Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 382) [INFO] [stdout] 2025-09-01T11:52:16.402515Z INFO Node{id=10}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:16.402556Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727536177) } [INFO] [stdout] 2025-09-01T11:52:16.417127Z INFO Node{id=10}: sequenced_broadcast: 87|SequencedBroadcastWorker Started id=87 [INFO] [stdout] 2025-09-01T11:52:16.439319Z INFO Node{id=10}: sequenced_broadcast: 24|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:16.439353Z INFO Node{id=10}: sequenced_broadcast: 24|SequencedBroadcastWorker Stopped id=24 elapsed=6.121639242s iter=2 [INFO] [stdout] 2025-09-01T11:52:16.457366Z INFO Node{id=28}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=280 [INFO] [stdout] 2025-09-01T11:52:16.457591Z INFO Node{id=28}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:16.503416Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:16.547809Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(407) < seq(410) [INFO] [stdout] 2025-09-01T11:52:16.563579Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:16.587654Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(19): Added, allow_drop: true, next_sequence: 410, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:16.602164Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:52:16.602199Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=18 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:52:16.616206Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:16.616264Z INFO Node{id=28}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:16.634553Z INFO Node{id=28}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:16.646448Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 410) [INFO] [stdout] 2025-09-01T11:52:16.646590Z INFO Node{id=18}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:16.647619Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:16.661518Z INFO Node{id=18}: sequenced_broadcast: 88|SequencedBroadcastWorker Started id=88 [INFO] [stdout] 2025-09-01T11:52:16.667077Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:16.682872Z INFO Node{id=18}: sequenced_broadcast: 32|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:16.682908Z INFO Node{id=18}: sequenced_broadcast: 32|SequencedBroadcastWorker Stopped id=32 elapsed=6.364720137s iter=2 [INFO] [stdout] 2025-09-01T11:52:16.779117Z ERROR Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:52:16.816376Z INFO Node{id=5}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:16.818495Z INFO Node{id=5}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:16.818571Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:52:16.830466Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:17.169152Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:17.169204Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:52:17.190861Z INFO Node{id=22}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:17.190963Z INFO Node{id=22}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:17.201251Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:52:17.329869Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727537190) } [INFO] [stdout] 2025-09-01T11:52:17.388412Z INFO Node{id=22}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=198 [INFO] [stdout] 2025-09-01T11:52:17.388453Z INFO Node{id=22}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:17.389171Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:17.389204Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:52:17.414385Z INFO Node{id=29}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:17.414487Z INFO Node{id=29}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:17.425585Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:52:17.490157Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:17.599002Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:52:17.599041Z ERROR Node{id=22}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=20 leader=1 local=22 [INFO] [stdout] 2025-09-01T11:52:17.616924Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:17.617000Z INFO Node{id=22}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:17.635559Z INFO Node{id=22}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:17.635676Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:17.635707Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:52:17.636097Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:17.653750Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727537414) } [INFO] [stdout] 2025-09-01T11:52:17.655012Z INFO Node{id=28}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:17.655079Z INFO Node{id=28}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:17.679691Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:52:17.686361Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:17.729379Z INFO Node{id=29}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=315 [INFO] [stdout] 2025-09-01T11:52:17.729424Z INFO Node{id=29}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:17.793598Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:17.835399Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:17.835439Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=1 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:17.846664Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:17.846776Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:17.877202Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:52:17.877266Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:52:17.956054Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:18.022680Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:18.048425Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeRecovery(RecoverableStateDetails { id: 2950095792267309118, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:18.069772Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:18.070407Z INFO Node{id=17}: sequenced_broadcast: 85|Subscriber rejected, max_seq(459) < seq(462) [INFO] [stdout] 2025-09-01T11:52:18.178729Z INFO Node{id=17}: sequenced_broadcast: 85|Subscriber(1): Added, allow_drop: true, next_sequence: 462, min_allowed_seq: 366 [INFO] [stdout] 2025-09-01T11:52:18.256398Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 462) [INFO] [stdout] 2025-09-01T11:52:18.256540Z INFO Node{id=29}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:18.266926Z INFO Node{id=29}: sequenced_broadcast: 89|SequencedBroadcastWorker Started id=89 [INFO] [stdout] 2025-09-01T11:52:18.281226Z INFO Node{id=29}: sequenced_broadcast: 43|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:18.281258Z INFO Node{id=29}: sequenced_broadcast: 43|SequencedBroadcastWorker Stopped id=43 elapsed=7.962392093s iter=2 [INFO] [stdout] 2025-09-01T11:52:18.373461Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:18.396058Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(491) < seq(495) [INFO] [stdout] 2025-09-01T11:52:18.447479Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(493) < seq(495) [INFO] [stdout] 2025-09-01T11:52:18.461418Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 495, max: 493 } [INFO] [stdout] 2025-09-01T11:52:18.461463Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:18.505382Z ERROR Node{id=19}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:18.516783Z WARN Node{id=19}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:18.516821Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:18.528847Z INFO Node{id=19}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:18.622318Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:18.622365Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:52:18.641340Z INFO Node{id=22}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:18.641449Z INFO Node{id=22}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:18.652188Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:52:18.723826Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727538641) } [INFO] [stdout] 2025-09-01T11:52:18.759871Z INFO Node{id=22}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=118 [INFO] [stdout] 2025-09-01T11:52:18.759924Z INFO Node{id=22}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:18.815048Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:18.851720Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:52:18.851776Z ERROR Node{id=22}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=8 leader=1 local=22 [INFO] [stdout] 2025-09-01T11:52:18.863130Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:18.863214Z INFO Node{id=22}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:18.880010Z INFO Node{id=22}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:18.880457Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:18.901271Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:19.052956Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727537655) } [INFO] [stdout] 2025-09-01T11:52:19.080697Z INFO Node{id=28}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=1425 [INFO] [stdout] 2025-09-01T11:52:19.080746Z INFO Node{id=28}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:19.363384Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:19.365847Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(575) < seq(581) [INFO] [stdout] 2025-09-01T11:52:19.424445Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(580) < seq(581) [INFO] [stdout] 2025-09-01T11:52:19.437179Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 581, max: 580 } [INFO] [stdout] 2025-09-01T11:52:19.437227Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:19.460405Z ERROR Node{id=20}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:19.465851Z WARN Node{id=20}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:19.465879Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:19.471664Z INFO Node{id=20}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:19.535619Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:19.535671Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:52:19.537585Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:19.544035Z INFO Node{id=19}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:19.544126Z INFO Node{id=19}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:19.554450Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:52:19.560789Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(591) < seq(595) [INFO] [stdout] 2025-09-01T11:52:19.583343Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(593) < seq(595) [INFO] [stdout] 2025-09-01T11:52:19.586736Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 595, max: 593 } [INFO] [stdout] 2025-09-01T11:52:19.593819Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:19.631965Z ERROR Node{id=21}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:19.634395Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727539544) } [INFO] [stdout] 2025-09-01T11:52:19.637379Z WARN Node{id=21}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:19.637402Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:19.644463Z INFO Node{id=21}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:19.665953Z INFO Node{id=19}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=121 [INFO] [stdout] 2025-09-01T11:52:19.666005Z INFO Node{id=19}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:19.787142Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:19.798426Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(606) < seq(612) [INFO] [stdout] 2025-09-01T11:52:19.824203Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:19.854224Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(610) < seq(612) [INFO] [stdout] 2025-09-01T11:52:19.867060Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 612, max: 610 } [INFO] [stdout] 2025-09-01T11:52:19.867103Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:19.868752Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:52:19.875867Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:19.875902Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:52:19.889383Z INFO Node{id=22}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:19.889499Z INFO Node{id=22}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:19.890462Z ERROR Node{id=23}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:19.898668Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:52:19.899988Z WARN Node{id=23}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:19.900010Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:19.910956Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:19.911892Z INFO Node{id=23}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:19.935556Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeRecovery(RecoverableStateDetails { id: 3017628437321749879, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:19.937527Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:19.977196Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:19.985127Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727539889) } [INFO] [stdout] 2025-09-01T11:52:19.989991Z INFO Node{id=32}: sequenced_broadcast: 79|Subscriber rejected, max_seq(597) < seq(603) [INFO] [stdout] 2025-09-01T11:52:20.015716Z INFO Node{id=22}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=126 [INFO] [stdout] 2025-09-01T11:52:20.015782Z INFO Node{id=22}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:20.042436Z INFO Node{id=32}: sequenced_broadcast: 79|Subscriber rejected, max_seq(602) < seq(603) [INFO] [stdout] 2025-09-01T11:52:20.049159Z ERROR Node{id=32}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 603, max: 602 } [INFO] [stdout] 2025-09-01T11:52:20.049196Z ERROR Node{id=32}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:20.075846Z ERROR Node{id=19}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:20.084701Z WARN Node{id=19}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:20.084734Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:20.096968Z INFO Node{id=19}:connect{target=32}:WritePeer{remote=32}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:20.124817Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:20.154598Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:52:20.242039Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:20.263077Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeRecovery(RecoverableStateDetails { id: 6302458865832610014, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:20.278144Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:20.279509Z INFO Node{id=26}: sequenced_broadcast: 82|Subscriber rejected, max_seq(629) < seq(631) [INFO] [stdout] 2025-09-01T11:52:20.279867Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:20.333036Z INFO Node{id=26}: sequenced_broadcast: 82|Subscriber(1): Added, allow_drop: true, next_sequence: 631, min_allowed_seq: 233 [INFO] [stdout] 2025-09-01T11:52:20.340802Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.359348Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.364876Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.365495Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.372440Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 631) [INFO] [stdout] 2025-09-01T11:52:20.372585Z INFO Node{id=22}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:20.373550Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.374020Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.377746Z INFO Node{id=22}: sequenced_broadcast: 90|SequencedBroadcastWorker Started id=90 [INFO] [stdout] 2025-09-01T11:52:20.379670Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.379934Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.383878Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.383969Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.385107Z INFO Node{id=22}: sequenced_broadcast: 36|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:20.385126Z INFO Node{id=22}: sequenced_broadcast: 36|SequencedBroadcastWorker Stopped id=36 elapsed=10.066698457s iter=2 [INFO] [stdout] 2025-09-01T11:52:20.391981Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.395549Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.395625Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.402675Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.411299Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.413611Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.415750Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.415948Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.438258Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.473262Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:20.473319Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:52:20.476016Z INFO Node{id=20}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:20.476092Z INFO Node{id=20}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:20.479441Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:52:20.483183Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.483878Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.507380Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727540476) } [INFO] [stdout] 2025-09-01T11:52:20.520270Z INFO Node{id=20}:connect{target=19}: sharedstate::worker::sync_manager: got pong from peer latency=44 [INFO] [stdout] 2025-09-01T11:52:20.520323Z INFO Node{id=20}:connect{target=19}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:20.537892Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:20.552496Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 19 } [INFO] [stdout] 2025-09-01T11:52:20.552537Z ERROR Node{id=20}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=19 leader=1 local=20 [INFO] [stdout] 2025-09-01T11:52:20.556270Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:20.556349Z INFO Node{id=20}:connect{target=19}:WritePeer{remote=19}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:20.560309Z INFO Node{id=20}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:20.560882Z ERROR ReadClient{remote=20}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:20.566315Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:20.584031Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:20.643366Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:20.643418Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:52:20.646801Z INFO Node{id=21}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:20.646882Z INFO Node{id=21}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:20.649706Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:52:20.793002Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:20.806031Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:52:20.904300Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:20.904350Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:52:20.907520Z INFO Node{id=23}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:20.907621Z INFO Node{id=23}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:20.910792Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:52:20.935208Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727540907) } [INFO] [stdout] 2025-09-01T11:52:20.935243Z INFO Node{id=5}: sharedstate::worker::sync_manager: Discovered Peer client_id=23 [INFO] [stdout] 2025-09-01T11:52:20.944490Z INFO Node{id=23}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=37 [INFO] [stdout] 2025-09-01T11:52:20.944531Z INFO Node{id=23}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:20.958443Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:20.969908Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727540646) } [INFO] [stdout] 2025-09-01T11:52:20.973505Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 5 } [INFO] [stdout] 2025-09-01T11:52:20.973533Z ERROR Node{id=23}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=5 leader=1 local=23 [INFO] [stdout] 2025-09-01T11:52:20.976966Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:20.977022Z INFO Node{id=23}:connect{target=5}:WritePeer{remote=5}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:20.980344Z INFO Node{id=23}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:20.981368Z INFO Node{id=21}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=335 [INFO] [stdout] 2025-09-01T11:52:20.981399Z INFO Node{id=21}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:20.983261Z ERROR ReadClient{remote=23}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:20.985284Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:21.087551Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:21.087600Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:52:21.090774Z INFO Node{id=19}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:21.090864Z INFO Node{id=19}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:21.094059Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:52:21.124949Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727541090) } [INFO] [stdout] 2025-09-01T11:52:21.133557Z INFO Node{id=19}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=43 [INFO] [stdout] 2025-09-01T11:52:21.133600Z INFO Node{id=19}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:21.153685Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:21.167171Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:52:21.198372Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:21.209002Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:21.226991Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeRecovery(RecoverableStateDetails { id: 3017628437321749879, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:21.250844Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:21.261858Z INFO Node{id=3}: sequenced_broadcast: 75|Subscriber rejected, max_seq(859) < seq(861) [INFO] [stdout] 2025-09-01T11:52:21.271879Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:21.286233Z INFO Node{id=3}: sequenced_broadcast: 75|Subscriber(1): Added, allow_drop: true, next_sequence: 861, min_allowed_seq: 31 [INFO] [stdout] 2025-09-01T11:52:21.303692Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 861) [INFO] [stdout] 2025-09-01T11:52:21.303863Z INFO Node{id=19}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:21.309264Z INFO Node{id=19}: sequenced_broadcast: 91|SequencedBroadcastWorker Started id=91 [INFO] [stdout] 2025-09-01T11:52:21.312739Z INFO Node{id=19}: sequenced_broadcast: 33|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:21.312782Z INFO Node{id=19}: sequenced_broadcast: 33|SequencedBroadcastWorker Stopped id=33 elapsed=10.994536139s iter=2 [INFO] [stdout] 2025-09-01T11:52:21.565865Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:21.565911Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:52:21.571615Z INFO Node{id=20}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:21.571727Z INFO Node{id=20}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:21.577451Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:52:21.899167Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:21.914702Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:52:21.926713Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727531754) } [INFO] [stdout] 2025-09-01T11:52:21.939418Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:52:21.939460Z INFO Node{id=5}: sharedstate::worker::sync_manager: Broadcasted 2 peers to 1 peers [INFO] [stdout] 2025-09-01T11:52:21.979502Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:21.979550Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:52:21.982805Z INFO Node{id=23}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:21.982912Z INFO Node{id=23}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:21.986225Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:52:22.007030Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727541982) } [INFO] [stdout] 2025-09-01T11:52:22.016353Z INFO Node{id=23}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=34 [INFO] [stdout] 2025-09-01T11:52:22.016401Z INFO Node{id=23}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:22.056568Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:22.069331Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:52:22.086410Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:22.089687Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeRecovery(RecoverableStateDetails { id: 17456534934631183880, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:22.092842Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:22.096339Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:22.099342Z INFO Node{id=16}: sequenced_broadcast: 86|Subscriber rejected, max_seq(1083) < seq(1086) [INFO] [stdout] 2025-09-01T11:52:22.118152Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:22.125926Z INFO Node{id=16}: sequenced_broadcast: 86|Subscriber(1): Added, allow_drop: true, next_sequence: 1086, min_allowed_seq: 383 [INFO] [stdout] 2025-09-01T11:52:22.127428Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(1111) < seq(1119) [INFO] [stdout] 2025-09-01T11:52:22.143242Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1086) [INFO] [stdout] 2025-09-01T11:52:22.143410Z INFO Node{id=23}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:22.146602Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(1116) < seq(1119) [INFO] [stdout] 2025-09-01T11:52:22.146819Z INFO Node{id=23}: sequenced_broadcast: 92|SequencedBroadcastWorker Started id=92 [INFO] [stdout] 2025-09-01T11:52:22.149936Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 1119, max: 1116 } [INFO] [stdout] 2025-09-01T11:52:22.149965Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:22.151217Z INFO Node{id=23}: sequenced_broadcast: 37|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:22.151238Z INFO Node{id=23}: sequenced_broadcast: 37|SequencedBroadcastWorker Stopped id=37 elapsed=11.832756692s iter=2 [INFO] [stdout] 2025-09-01T11:52:22.156731Z ERROR Node{id=9}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:22.160194Z WARN Node{id=9}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:22.160219Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:22.163836Z INFO Node{id=9}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:22.208405Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:22.211393Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(1134) < seq(1144) [INFO] [stdout] 2025-09-01T11:52:22.232188Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(1139) < seq(1144) [INFO] [stdout] 2025-09-01T11:52:22.235695Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 1144, max: 1139 } [INFO] [stdout] 2025-09-01T11:52:22.235726Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:52:22.242381Z ERROR Node{id=8}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:22.245649Z WARN Node{id=8}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:22.245674Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:22.248834Z INFO Node{id=8}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:22.831461Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:22.839434Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:22.843495Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 2920180286697105469, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:22.849147Z ERROR Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:52:22.852005Z INFO Node{id=5}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:22.852078Z INFO Node{id=5}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:22.852136Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:52:22.854567Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:23.166281Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:23.166335Z INFO Node{id=9}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:52:23.169494Z INFO Node{id=9}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:23.169584Z INFO Node{id=9}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:23.171990Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=9 [INFO] [stdout] 2025-09-01T11:52:23.201383Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: Ping(1756727543169) } [INFO] [stdout] 2025-09-01T11:52:23.210160Z INFO Node{id=9}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=41 [INFO] [stdout] 2025-09-01T11:52:23.210202Z INFO Node{id=9}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:23.222689Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:23.234074Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:52:23.248657Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:23.248918Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:23.248949Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:52:23.251691Z INFO Node{id=8}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:23.251791Z INFO Node{id=8}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:23.254911Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:52:23.256038Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeRecovery(RecoverableStateDetails { id: 5502544115740205408, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:23.257870Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:23.266688Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:23.272479Z INFO Node{id=18}: sequenced_broadcast: 88|Subscriber rejected, max_seq(1401) < seq(1403) [INFO] [stdout] 2025-09-01T11:52:23.281584Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727543251) } [INFO] [stdout] 2025-09-01T11:52:23.289815Z INFO Node{id=8}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=38 [INFO] [stdout] 2025-09-01T11:52:23.289860Z INFO Node{id=8}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:23.297304Z INFO Node{id=18}: sequenced_broadcast: 88|Subscriber(1): Added, allow_drop: true, next_sequence: 1403, min_allowed_seq: 410 [INFO] [stdout] 2025-09-01T11:52:23.311252Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:23.311950Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1403) [INFO] [stdout] 2025-09-01T11:52:23.312068Z INFO Node{id=9}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:23.315140Z INFO Node{id=9}: sequenced_broadcast: 93|SequencedBroadcastWorker Started id=93 [INFO] [stdout] 2025-09-01T11:52:23.318182Z INFO Node{id=9}: sequenced_broadcast: 23|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:23.318203Z INFO Node{id=9}: sequenced_broadcast: 23|SequencedBroadcastWorker Stopped id=23 elapsed=13.000560448s iter=2 [INFO] [stdout] 2025-09-01T11:52:23.325209Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:52:23.340157Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:23.349822Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeRecovery(RecoverableStateDetails { id: 2409269800524822834, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:23.352450Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:23.352599Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:23.355173Z INFO Node{id=2}: sequenced_broadcast: 74|Subscriber rejected, max_seq(1429) < seq(1432) [INFO] [stdout] 2025-09-01T11:52:23.385594Z INFO Node{id=2}: sequenced_broadcast: 74|Subscriber(1): Added, allow_drop: true, next_sequence: 1432, min_allowed_seq: 25 [INFO] [stdout] 2025-09-01T11:52:23.400560Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1432) [INFO] [stdout] 2025-09-01T11:52:23.400689Z INFO Node{id=8}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:23.403578Z INFO Node{id=8}: sequenced_broadcast: 94|SequencedBroadcastWorker Started id=94 [INFO] [stdout] 2025-09-01T11:52:23.406986Z INFO Node{id=8}: sequenced_broadcast: 22|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:23.407012Z INFO Node{id=8}: sequenced_broadcast: 22|SequencedBroadcastWorker Stopped id=22 elapsed=13.089466397s iter=2 [INFO] [stdout] 2025-09-01T11:52:23.856424Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:23.856466Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=1 peers=2 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:23.859466Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:23.859556Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:23.862745Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:52:24.016966Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:24.022565Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(1619) < seq(1621) [INFO] [stdout] 2025-09-01T11:52:24.046644Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(20): Added, allow_drop: true, next_sequence: 1621, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:52:24.073959Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1621) [INFO] [stdout] 2025-09-01T11:52:24.074100Z INFO Node{id=31}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:24.077145Z INFO Node{id=31}: sequenced_broadcast: 95|SequencedBroadcastWorker Started id=95 [INFO] [stdout] 2025-09-01T11:52:24.080849Z INFO Node{id=31}: sequenced_broadcast: 45|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:24.080873Z INFO Node{id=31}: sequenced_broadcast: 45|SequencedBroadcastWorker Stopped id=45 elapsed=13.761895739s iter=2 [INFO] [stdout] 2025-09-01T11:52:24.359932Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727541571) } [INFO] [stdout] 2025-09-01T11:52:24.370808Z INFO Node{id=20}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=2799 [INFO] [stdout] 2025-09-01T11:52:24.370850Z INFO Node{id=20}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:24.457095Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:24.467043Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:24.472713Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeRecovery(RecoverableStateDetails { id: 2604719619735459382, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:25.942543Z ERROR Node{id=8}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:52:25.945355Z WARN Node{id=8}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:25.945376Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:25.963062Z ERROR ReadClient{remote=8}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:25.975478Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:25.975548Z INFO Node{id=2}: sequenced_broadcast: 1|Subscriber(74): channel closed, dropping [INFO] [stdout] 2025-09-01T11:52:26.331685Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:26.336495Z INFO Node{id=30}: sequenced_broadcast: 84|Subscriber rejected, max_seq(1501) < seq(1503) [INFO] [stdout] 2025-09-01T11:52:26.360532Z INFO Node{id=30}: sequenced_broadcast: 84|Subscriber(2): Added, allow_drop: true, next_sequence: 1503, min_allowed_seq: 315 [INFO] [stdout] 2025-09-01T11:52:26.376232Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1503) [INFO] [stdout] 2025-09-01T11:52:26.376370Z INFO Node{id=28}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:26.379203Z INFO Node{id=28}: sequenced_broadcast: 96|SequencedBroadcastWorker Started id=96 [INFO] [stdout] 2025-09-01T11:52:26.382752Z INFO Node{id=28}: sequenced_broadcast: 42|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:26.382794Z INFO Node{id=28}: sequenced_broadcast: 42|SequencedBroadcastWorker Stopped id=42 elapsed=16.063986938s iter=2 [INFO] [stdout] 2025-09-01T11:52:26.960437Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:26.960483Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:52:26.963138Z INFO Node{id=8}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:26.963224Z INFO Node{id=8}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:26.966137Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:52:26.989283Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727546963) } [INFO] [stdout] 2025-09-01T11:52:26.998799Z INFO Node{id=8}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=35 [INFO] [stdout] 2025-09-01T11:52:26.998840Z INFO Node{id=8}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:27.011830Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:27.025856Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:52:27.033951Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:27.037586Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 1833, state_accept_seq: 1832, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:27.039038Z INFO Node{id=7}: sequenced_broadcast: 77|Subscriber(1): Added, allow_drop: true, next_sequence: 1833, min_allowed_seq: 362 [INFO] [stdout] 2025-09-01T11:52:27.041945Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:27.055798Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 1833) [INFO] [stdout] 2025-09-01T11:52:27.055881Z INFO Node{id=8}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:52:27.469129Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727537846) } [INFO] [stdout] 2025-09-01T11:52:27.902090Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:28.140780Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:28.153885Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:52:28.196940Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:28.202933Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:28.309411Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:28.363951Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:52:28.862962Z ERROR Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:52:28.865752Z INFO Node{id=5}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:28.865855Z INFO Node{id=5}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:28.865932Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:52:28.871024Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:29.225995Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:29.228363Z INFO Node{id=26}: sequenced_broadcast: 82|Subscriber rejected, max_seq(2092) < seq(2095) [INFO] [stdout] 2025-09-01T11:52:29.250848Z INFO Node{id=26}: sequenced_broadcast: 82|Subscriber(2): Added, allow_drop: true, next_sequence: 2095, min_allowed_seq: 233 [INFO] [stdout] 2025-09-01T11:52:29.265387Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 2095) [INFO] [stdout] 2025-09-01T11:52:29.265555Z INFO Node{id=21}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:29.272645Z INFO Node{id=21}: sequenced_broadcast: 97|SequencedBroadcastWorker Started id=97 [INFO] [stdout] 2025-09-01T11:52:29.275682Z INFO Node{id=21}: sequenced_broadcast: 35|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:29.275704Z INFO Node{id=21}: sequenced_broadcast: 35|SequencedBroadcastWorker Stopped id=35 elapsed=18.957332948s iter=2 [INFO] [stdout] 2025-09-01T11:52:29.832941Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:29.840720Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 30]) [INFO] [stdout] 2025-09-01T11:52:29.868949Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:29.868988Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=1 peers=2 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:29.871682Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:29.871815Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:29.875166Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:52:29.888275Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:52:29.902088Z INFO Node{id=30}: sequenced_broadcast: 1|Subscriber(84): channel closed, dropping [INFO] [stdout] 2025-09-01T11:52:29.902699Z ERROR Node{id=10}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:29.907918Z WARN Node{id=10}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:29.907949Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:30.157671Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:30.367375Z ERROR ReadClient{remote=12}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:52:30.686120Z INFO Node{id=1}: sequenced_broadcast: 2|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:52:30.692272Z ERROR Node{id=12}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:30.740653Z WARN Node{id=12}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:30.740686Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:30.815746Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:30.921522Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:30.921575Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:52:30.924159Z INFO Node{id=10}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:30.924249Z INFO Node{id=10}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:30.926926Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:52:30.947972Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727550924) } [INFO] [stdout] 2025-09-01T11:52:30.955587Z INFO Node{id=10}:connect{target=19}: sharedstate::worker::sync_manager: got pong from peer latency=31 [INFO] [stdout] 2025-09-01T11:52:30.955628Z INFO Node{id=10}:connect{target=19}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:30.966872Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:30.978702Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 19 } [INFO] [stdout] 2025-09-01T11:52:30.990964Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:30.993477Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 2095, state_accept_seq: 2094, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:31.001080Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:31.001182Z INFO Node{id=19}: sequenced_broadcast: 91|Subscriber(1): Added, allow_drop: true, next_sequence: 2095, min_allowed_seq: 861 [INFO] [stdout] 2025-09-01T11:52:31.016804Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 2095) [INFO] [stdout] 2025-09-01T11:52:31.016884Z INFO Node{id=10}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:52:31.188942Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:31.236748Z ERROR ReadClient{remote=7}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:52:31.467476Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727543859) } [INFO] [stdout] 2025-09-01T11:52:31.752622Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:31.763030Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17]) [INFO] [stdout] 2025-09-01T11:52:31.779439Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:31.779484Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:52:31.782528Z INFO Node{id=12}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:31.782632Z INFO Node{id=12}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:31.787683Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:52:31.814742Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727551782) } [INFO] [stdout] 2025-09-01T11:52:31.825232Z INFO Node{id=12}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=43 [INFO] [stdout] 2025-09-01T11:52:31.825276Z INFO Node{id=12}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:31.848347Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:31.862266Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:52:31.878581Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:31.881546Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 3420, state_accept_seq: 3419, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:31.884888Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:31.886644Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:31.888869Z INFO Node{id=29}: sequenced_broadcast: 89|Subscriber rejected, max_seq(2461) < seq(2462) [INFO] [stdout] 2025-09-01T11:52:31.913903Z INFO Node{id=29}: sequenced_broadcast: 89|Subscriber(1): Added, allow_drop: true, next_sequence: 2462, min_allowed_seq: 462 [INFO] [stdout] 2025-09-01T11:52:31.920741Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:31.929599Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 2462) [INFO] [stdout] 2025-09-01T11:52:31.929748Z INFO Node{id=12}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:31.932985Z INFO Node{id=12}: sequenced_broadcast: 98|SequencedBroadcastWorker Started id=98 [INFO] [stdout] 2025-09-01T11:52:31.936347Z INFO Node{id=12}: sequenced_broadcast: 66|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:31.936369Z INFO Node{id=12}: sequenced_broadcast: 66|SequencedBroadcastWorker Stopped id=66 elapsed=21.578507832s iter=1462 [INFO] [stdout] 2025-09-01T11:52:32.033722Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:32.042215Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeRecovery(RecoverableStateDetails { id: 8084228702317741667, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:32.043610Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:32.082337Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:32.827586Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:52:32.835542Z ERROR Node{id=5}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:32.838685Z ERROR Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: channel closed waiting for pong [INFO] [stdout] 2025-09-01T11:52:32.841080Z INFO Node{id=5}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:32.841172Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:52:32.928229Z ERROR ReadClient{remote=23}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:52:32.939872Z ERROR Node{id=23}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:32.945407Z WARN Node{id=23}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:32.945446Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:32.950283Z INFO Node{id=23}:connect{target=16}:WritePeer{remote=16}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:32.952268Z INFO Node{id=16}: sequenced_broadcast: 1|Subscriber(86): channel closed, dropping [INFO] [stdout] 2025-09-01T11:52:33.245526Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:33.698400Z ERROR Node{id=24}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:52:33.701435Z WARN Node{id=24}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:33.701459Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:33.846285Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:33.846329Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=1 peers=2 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:33.850445Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:33.850552Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:33.855458Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:52:33.901029Z INFO Node{id=1}: sequenced_broadcast: 9|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:52:33.921833Z ERROR Node{id=7}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:33.964582Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:33.964629Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:52:33.967820Z INFO Node{id=23}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:33.967920Z INFO Node{id=23}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:33.973109Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:52:34.716424Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:34.716472Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:52:34.719404Z INFO Node{id=24}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:34.719500Z INFO Node{id=24}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:34.722361Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:52:34.753040Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727554719) } [INFO] [stdout] 2025-09-01T11:52:34.766276Z INFO Node{id=24}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=47 [INFO] [stdout] 2025-09-01T11:52:34.766321Z INFO Node{id=24}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:34.782920Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:34.794394Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:52:34.813865Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:34.816559Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 982, state_accept_seq: 981, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:34.820729Z INFO Node{id=6}: sequenced_broadcast: 78|Subscriber rejected, seq(982) < min_allowed(2441) [INFO] [stdout] 2025-09-01T11:52:34.822240Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:34.826042Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:34.831924Z INFO Node{id=6}: sequenced_broadcast: 78|Subscriber(1): Added, allow_drop: true, next_sequence: 4492, min_allowed_seq: 2444 [INFO] [stdout] 2025-09-01T11:52:34.848467Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4492) [INFO] [stdout] 2025-09-01T11:52:34.848615Z INFO Node{id=24}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:34.851453Z INFO Node{id=24}: sequenced_broadcast: 99|SequencedBroadcastWorker Started id=99 [INFO] [stdout] 2025-09-01T11:52:34.855001Z INFO Node{id=24}: sequenced_broadcast: 80|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:34.855027Z INFO Node{id=24}: sequenced_broadcast: 80|SequencedBroadcastWorker Stopped id=80 elapsed=23.385278915s iter=347 [INFO] [stdout] 2025-09-01T11:52:35.050002Z ERROR Node{id=6}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:52:35.307912Z WARN Node{id=6}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:35.307948Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:35.813806Z ERROR ReadClient{remote=24}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:36.060608Z ERROR WriteClient{remote=20}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:52:36.245698Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:36.250147Z INFO Node{id=17}: sequenced_broadcast: 85|Subscriber rejected, max_seq(3240) < seq(3241) [INFO] [stdout] 2025-09-01T11:52:36.274081Z INFO Node{id=17}: sequenced_broadcast: 85|Subscriber(2): Added, allow_drop: true, next_sequence: 3241, min_allowed_seq: 1193 [INFO] [stdout] 2025-09-01T11:52:36.296119Z INFO Node{id=17}: sequenced_broadcast: 2|Subscriber(85): channel closed, dropping [INFO] [stdout] 2025-09-01T11:52:36.323490Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:36.323554Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=4 leader=1 [INFO] [stdout] 2025-09-01T11:52:36.328324Z INFO Node{id=6}:connect{target=4}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:36.328412Z INFO Node{id=6}:connect{target=4}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:36.330701Z INFO Node{id=4}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:52:36.355996Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727556328) } [INFO] [stdout] 2025-09-01T11:52:36.365969Z INFO Node{id=6}:connect{target=4}: sharedstate::worker::sync_manager: got pong from peer latency=37 [INFO] [stdout] 2025-09-01T11:52:36.366015Z INFO Node{id=6}:connect{target=4}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:36.381519Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:36.394663Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 4 } [INFO] [stdout] 2025-09-01T11:52:36.405470Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:36.408715Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4579, state_accept_seq: 4578, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:36.411147Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:36.413410Z INFO Node{id=4}: sequenced_broadcast: 76|Subscriber(1): Added, allow_drop: true, next_sequence: 4579, min_allowed_seq: 2588 [INFO] [stdout] 2025-09-01T11:52:36.428711Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4579) [INFO] [stdout] 2025-09-01T11:52:36.428820Z INFO Node{id=6}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:52:36.543696Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:36.630674Z ERROR Node{id=15}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:52:36.634437Z WARN Node{id=15}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:36.634462Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:36.687674Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3]) [INFO] [stdout] 2025-09-01T11:52:36.695212Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:36.703211Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 26]) [INFO] [stdout] 2025-09-01T11:52:37.034675Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:37.123043Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727549871) } [INFO] [stdout] 2025-09-01T11:52:37.127173Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:37.646713Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:37.654405Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:37.654455Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:52:37.656823Z INFO Node{id=15}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:37.656920Z INFO Node{id=15}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:37.659120Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:52:38.156470Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:38.163222Z WARN Node{id=7}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:38.163250Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:38.379170Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:38.853824Z ERROR Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:52:38.855946Z INFO Node{id=5}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:38.856034Z INFO Node{id=5}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:38.856111Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:52:38.858525Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:38.970462Z ERROR Node{id=23}:connect{target=18}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:52:38.972430Z INFO Node{id=23}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:38.972508Z INFO Node{id=23}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:38.972590Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(18) [INFO] [stdout] 2025-09-01T11:52:38.974843Z ERROR ReadClient{remote=23}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:39.174981Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:39.175024Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:52:39.177544Z INFO Node{id=7}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:39.177619Z INFO Node{id=7}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:39.180114Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:52:39.355425Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:39.364019Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:39.364068Z ERROR Node{id=8}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:39.859650Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:39.859691Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=1 peers=2 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:52:39.862068Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:39.862171Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:39.864682Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:52:39.975531Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:39.975586Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:52:39.977587Z INFO Node{id=23}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:39.977713Z INFO Node{id=23}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:39.980037Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:52:39.994611Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727559977) } [INFO] [stdout] 2025-09-01T11:52:40.001799Z INFO Node{id=23}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=24 [INFO] [stdout] 2025-09-01T11:52:40.001842Z INFO Node{id=23}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:40.016171Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:40.022654Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:40.025936Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:52:40.032697Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 18]) [INFO] [stdout] 2025-09-01T11:52:40.040425Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:40.044022Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 2829, state_accept_seq: 2828, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:40.047642Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:40.047713Z INFO Node{id=14}: sequenced_broadcast: 72|Subscriber rejected, seq(2829) < min_allowed(4109) [INFO] [stdout] 2025-09-01T11:52:40.055706Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:40.060625Z INFO Node{id=14}: sequenced_broadcast: 72|Subscriber rejected, max_seq(6161) < seq(6162) [INFO] [stdout] 2025-09-01T11:52:40.086393Z INFO Node{id=14}: sequenced_broadcast: 72|Subscriber(1): Added, allow_drop: true, next_sequence: 6162, min_allowed_seq: 4114 [INFO] [stdout] 2025-09-01T11:52:40.099544Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6162) [INFO] [stdout] 2025-09-01T11:52:40.099711Z INFO Node{id=23}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:40.102088Z INFO Node{id=23}: sequenced_broadcast: 100|SequencedBroadcastWorker Started id=100 [INFO] [stdout] 2025-09-01T11:52:40.104739Z INFO Node{id=23}: sequenced_broadcast: 92|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:40.104780Z INFO Node{id=23}: sequenced_broadcast: 92|SequencedBroadcastWorker Stopped id=92 elapsed=17.957939147s iter=917 [INFO] [stdout] 2025-09-01T11:52:40.345552Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.345687Z ERROR Node{id=11}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:52:40.357019Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.360214Z ERROR Node{id=3}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:52:40.365455Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.368184Z INFO Node{id=4}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:40.371296Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.371393Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.372825Z ERROR Node{id=27}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:52:40.374280Z INFO Node{id=26}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:40.376251Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.376349Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.377202Z INFO Node{id=32}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:40.377264Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.379130Z INFO Node{id=14}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:40.380642Z ERROR Node{id=30}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:52:40.383122Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.383387Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.383833Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.384048Z ERROR Node{id=18}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:52:40.384608Z INFO Node{id=2}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:40.386551Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.387003Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.388245Z INFO Node{id=13}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:40.390926Z ERROR Node{id=17}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:52:40.391458Z ERROR Node{id=16}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:52:40.398265Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.399909Z INFO Node{id=25}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:40.461584Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727553967) } [INFO] [stdout] 2025-09-01T11:52:40.494634Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727553850) } [INFO] [stdout] 2025-09-01T11:52:40.730303Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:40.732625Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:40.732652Z ERROR Node{id=21}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:40.980461Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:40.981903Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727557656) } [INFO] [stdout] 2025-09-01T11:52:40.984560Z INFO Node{id=15}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=3328 [INFO] [stdout] 2025-09-01T11:52:40.984593Z INFO Node{id=15}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:40.984886Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:40.987363Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19]) [INFO] [stdout] 2025-09-01T11:52:41.003675Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.007935Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:41.007980Z ERROR Node{id=28}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.008961Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:41.009602Z INFO Node{id=28}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:41.010507Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.127250Z ERROR ReadClient{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.131645Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.132065Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.132663Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.133326Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.134986Z ERROR ReadClient{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.137266Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.138372Z ERROR ReadClient{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.138775Z ERROR ReadClient{remote=25}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.142021Z ERROR ReadClient{remote=16}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.143600Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.144621Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.146459Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.147861Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.323085Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.364476Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.364522Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.365574Z INFO Node{id=11}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.365653Z INFO Node{id=11}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.366212Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:52:41.366282Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:41.369977Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.370489Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.370520Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.371138Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:52:41.371184Z INFO Node{id=1}: sharedstate::worker::sync_manager: Broadcasted 31 peers to 31 peers [INFO] [stdout] 2025-09-01T11:52:41.371945Z INFO Node{id=3}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.372018Z INFO Node{id=3}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.372794Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727561365) } [INFO] [stdout] 2025-09-01T11:52:41.372911Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:52:41.373867Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:41.373887Z ERROR Node{id=8}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.377094Z INFO Node{id=11}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=12 [INFO] [stdout] 2025-09-01T11:52:41.377130Z INFO Node{id=11}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.378080Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.378111Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.378946Z INFO Node{id=4}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.379014Z INFO Node{id=4}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.379984Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:52:41.380640Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727561372) } [INFO] [stdout] 2025-09-01T11:52:41.380828Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.383135Z INFO Node{id=3}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:52:41.383167Z INFO Node{id=3}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.384024Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:52:41.384047Z ERROR Node{id=11}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[8] peer=8 leader=1 local=11 [INFO] [stdout] 2025-09-01T11:52:41.385418Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.385450Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=28 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.385972Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:41.386021Z INFO Node{id=11}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:41.386487Z INFO Node{id=26}:connect{target=28}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.386565Z INFO Node{id=26}:connect{target=28}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.387141Z INFO Node{id=11}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:41.387689Z INFO Node{id=28}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:52:41.387873Z ERROR ReadClient{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.389145Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:41.389227Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.392229Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.392259Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.393125Z INFO Node{id=14}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.393194Z INFO Node{id=14}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.393720Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:52:41.393750Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=20 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:52:41.394120Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727561386) } [INFO] [stdout] 2025-09-01T11:52:41.394207Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:52:41.394669Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.394698Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.395455Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:41.395535Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.395568Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.395775Z INFO Node{id=30}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.395835Z INFO Node{id=30}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.395918Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:41.396008Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.396031Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.396547Z INFO Node{id=3}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:41.396619Z INFO Node{id=3}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:41.396697Z INFO Node{id=18}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.396755Z INFO Node{id=18}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.396821Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:52:41.396878Z INFO Node{id=26}:connect{target=28}: sharedstate::worker::sync_manager: got pong from peer latency=10 [INFO] [stdout] 2025-09-01T11:52:41.396902Z INFO Node{id=26}:connect{target=28}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.396981Z INFO Node{id=32}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.397063Z INFO Node{id=32}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.397956Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.398111Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:52:41.398624Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:52:41.398924Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.398951Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.400079Z INFO Node{id=17}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.400141Z INFO Node{id=17}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.401413Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:52:41.402204Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727561393) } [INFO] [stdout] 2025-09-01T11:52:41.403076Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.403186Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.403215Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.403910Z INFO Node{id=13}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.403969Z INFO Node{id=13}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.404061Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.404086Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.404949Z INFO Node{id=14}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:52:41.404976Z INFO Node{id=14}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.405019Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:52:41.405094Z INFO Node{id=27}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.405151Z INFO Node{id=27}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.405388Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727561395) } [INFO] [stdout] 2025-09-01T11:52:41.406154Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:52:41.406917Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727561397) } [INFO] [stdout] 2025-09-01T11:52:41.407332Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727561396) } [INFO] [stdout] 2025-09-01T11:52:41.409015Z INFO Node{id=30}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=14 [INFO] [stdout] 2025-09-01T11:52:41.409041Z INFO Node{id=30}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.410294Z INFO Node{id=32}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:52:41.410321Z INFO Node{id=32}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.410353Z INFO Node{id=18}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=14 [INFO] [stdout] 2025-09-01T11:52:41.410373Z INFO Node{id=18}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.410448Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727561400) } [INFO] [stdout] 2025-09-01T11:52:41.410547Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.410575Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.411348Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.411500Z INFO Node{id=16}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.411570Z INFO Node{id=16}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.412406Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:41.412608Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:52:41.413532Z INFO Node{id=17}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:52:41.413575Z INFO Node{id=17}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.414734Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.414793Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727561405) } [INFO] [stdout] 2025-09-01T11:52:41.416028Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.416156Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.416186Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.416977Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.417303Z INFO Node{id=25}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.417367Z INFO Node{id=25}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.418166Z INFO Node{id=27}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:52:41.418194Z INFO Node{id=27}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.418970Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:52:41.419755Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.420042Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:52:41.420515Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:52:41.420535Z ERROR Node{id=30}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=27 leader=1 local=30 [INFO] [stdout] 2025-09-01T11:52:41.420837Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:41.421264Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:52:41.421285Z ERROR Node{id=14}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=15 leader=1 local=14 [INFO] [stdout] 2025-09-01T11:52:41.421552Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 28 } [INFO] [stdout] 2025-09-01T11:52:41.421573Z ERROR Node{id=26}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=28 leader=1 local=26 [INFO] [stdout] 2025-09-01T11:52:41.421791Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:41.423062Z INFO Node{id=30}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:41.423141Z INFO Node{id=30}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:41.424019Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:41.424063Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.424093Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:41.424270Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.424687Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:41.424835Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:41.424979Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:41.425008Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:52:41.425404Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:52:41.425930Z INFO Node{id=2}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:41.425998Z INFO Node{id=2}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:41.426363Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 6274, state_accept_seq: 6273, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:41.426454Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:41.426921Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:41.427050Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:52:41.427653Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:41.427890Z INFO Node{id=14}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:41.427953Z INFO Node{id=14}:connect{target=15}:WritePeer{remote=15}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:41.428397Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:41.428497Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:41.428566Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:52:41.428582Z ERROR Node{id=27}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=7 leader=1 local=27 [INFO] [stdout] 2025-09-01T11:52:41.428733Z INFO Node{id=26}:connect{target=28}:ReadPeer{remote=28}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:41.428819Z INFO Node{id=26}:connect{target=28}:WritePeer{remote=28}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:41.429207Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:52:41.429236Z ERROR Node{id=18}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=27 leader=1 local=18 [INFO] [stdout] 2025-09-01T11:52:41.429609Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.429866Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:41.429907Z INFO Node{id=27}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:41.430334Z ERROR ReadClient{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.430424Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:41.430618Z INFO Node{id=22}: sequenced_broadcast: 90|Subscriber(1): Added, allow_drop: true, next_sequence: 4027, min_allowed_seq: 1979 [INFO] [stdout] 2025-09-01T11:52:41.431261Z INFO Node{id=27}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:41.431621Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4035, state_accept_seq: 4034, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:41.432048Z ERROR ReadClient{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.432741Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:41.437072Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:41.437309Z INFO Node{id=10}: sequenced_broadcast: 87|Subscriber(1): Added, allow_drop: true, next_sequence: 4035, min_allowed_seq: 2051 [INFO] [stdout] 2025-09-01T11:52:41.437388Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:41.437580Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:41.440920Z INFO Node{id=18}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:41.441017Z INFO Node{id=18}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:41.441936Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4027) [INFO] [stdout] 2025-09-01T11:52:41.442050Z INFO Node{id=32}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:41.442148Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.442567Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727561425) } [INFO] [stdout] 2025-09-01T11:52:41.443249Z INFO Node{id=32}: sequenced_broadcast: 101|SequencedBroadcastWorker Started id=101 [INFO] [stdout] 2025-09-01T11:52:41.444969Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4035) [INFO] [stdout] 2025-09-01T11:52:41.445034Z INFO Node{id=17}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:52:41.445253Z INFO Node{id=32}: sequenced_broadcast: 79|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:41.445273Z INFO Node{id=32}: sequenced_broadcast: 79|SequencedBroadcastWorker Stopped id=79 elapsed=30.207732693s iter=2583 [INFO] [stdout] 2025-09-01T11:52:41.447680Z INFO Node{id=2}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=22 [INFO] [stdout] 2025-09-01T11:52:41.447714Z INFO Node{id=2}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.457206Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.458697Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:52:41.462480Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:52:41.469648Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:41.475059Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 5654, state_accept_seq: 5653, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:41.476217Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:41.480127Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:41.483405Z INFO Node{id=10}: sequenced_broadcast: 87|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2051 [INFO] [stdout] 2025-09-01T11:52:41.492175Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:52:41.492291Z INFO Node{id=2}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:41.493397Z INFO Node{id=2}: sequenced_broadcast: 102|SequencedBroadcastWorker Started id=102 [INFO] [stdout] 2025-09-01T11:52:41.495871Z INFO Node{id=2}: sequenced_broadcast: 74|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:41.495892Z INFO Node{id=2}: sequenced_broadcast: 74|SequencedBroadcastWorker Stopped id=74 elapsed=31.062572124s iter=2461 [INFO] [stdout] 2025-09-01T11:52:41.512021Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.518729Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:52:41.518782Z ERROR Node{id=15}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=18 leader=1 local=15 [INFO] [stdout] 2025-09-01T11:52:41.520690Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:41.520752Z INFO Node{id=15}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:41.526709Z INFO Node{id=15}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:41.528616Z ERROR ReadClient{remote=15}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.643964Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727561379) } [INFO] [stdout] 2025-09-01T11:52:41.647947Z INFO Node{id=4}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=268 [INFO] [stdout] 2025-09-01T11:52:41.647993Z INFO Node{id=4}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:41.674218Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:41.703495Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.712352Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:52:41.712392Z ERROR Node{id=4}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=18 leader=1 local=4 [INFO] [stdout] 2025-09-01T11:52:41.713678Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:41.717300Z INFO Node{id=4}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:41.717389Z INFO Node{id=4}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:41.719062Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:41.726216Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:41.764438Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:41.864332Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:41.870358Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:41.873674Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29]) [INFO] [stdout] 2025-09-01T11:52:41.916074Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:41.916943Z INFO Node{id=21}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:41.917401Z ERROR ReadClient{remote=21}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:42.014018Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:42.014066Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:52:42.015419Z INFO Node{id=28}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:42.015492Z INFO Node{id=28}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:42.016193Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:52:42.028281Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727562015) } [INFO] [stdout] 2025-09-01T11:52:42.032610Z INFO Node{id=28}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:52:42.032642Z INFO Node{id=28}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:42.038347Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.042380Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:52:42.046606Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:42.047391Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4037, state_accept_seq: 4036, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:42.048889Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:42.049122Z INFO Node{id=23}: sequenced_broadcast: 100|Subscriber rejected, seq(4037) < min_allowed(6162) [INFO] [stdout] 2025-09-01T11:52:42.051404Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:42.053803Z INFO Node{id=23}: sequenced_broadcast: 100|Subscriber(1): Added, allow_drop: true, next_sequence: 6217, min_allowed_seq: 6162 [INFO] [stdout] 2025-09-01T11:52:42.058007Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6217) [INFO] [stdout] 2025-09-01T11:52:42.058102Z INFO Node{id=28}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:42.058704Z INFO Node{id=28}: sequenced_broadcast: 103|SequencedBroadcastWorker Started id=103 [INFO] [stdout] 2025-09-01T11:52:42.059656Z INFO Node{id=28}: sequenced_broadcast: 96|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:42.059675Z INFO Node{id=28}: sequenced_broadcast: 96|SequencedBroadcastWorker Stopped id=96 elapsed=15.680445583s iter=1294 [INFO] [stdout] 2025-09-01T11:52:42.119897Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.158116Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.177344Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.180188Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.220635Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.249487Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.301124Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.302445Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727559177) } [INFO] [stdout] 2025-09-01T11:52:42.305523Z INFO Node{id=7}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=3128 [INFO] [stdout] 2025-09-01T11:52:42.305567Z INFO Node{id=7}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:42.315168Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.333377Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.387366Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:42.387415Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:52:42.389148Z INFO Node{id=11}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:42.389223Z INFO Node{id=11}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:42.390782Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:52:42.397081Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:42.397120Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:52:42.398557Z INFO Node{id=3}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:42.398629Z INFO Node{id=3}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:42.399669Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:52:42.400724Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727562389) } [INFO] [stdout] 2025-09-01T11:52:42.405065Z INFO Node{id=11}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:52:42.405101Z INFO Node{id=11}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:42.410111Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727562398) } [INFO] [stdout] 2025-09-01T11:52:42.411462Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.413393Z INFO Node{id=3}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:52:42.413429Z INFO Node{id=3}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:42.417329Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:52:42.417356Z ERROR Node{id=11}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=13 leader=1 local=11 [INFO] [stdout] 2025-09-01T11:52:42.419659Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:42.419713Z INFO Node{id=11}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:42.420613Z INFO Node{id=11}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:42.421688Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.421846Z ERROR ReadClient{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:42.422797Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:42.425897Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:42.425930Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:52:42.426702Z INFO Node{id=30}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:42.426792Z INFO Node{id=30}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:42.427720Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:52:42.429009Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:42.429038Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:52:42.429421Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:42.429449Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:52:42.429659Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:52:42.429681Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=21 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:52:42.430818Z INFO Node{id=26}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:42.430884Z INFO Node{id=26}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:42.430934Z INFO Node{id=14}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:42.430988Z INFO Node{id=14}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:42.431496Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:42.431523Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:52:42.432293Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:42.432449Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:52:42.432521Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:52:42.433315Z INFO Node{id=27}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:42.433369Z INFO Node{id=27}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:42.433856Z INFO Node{id=3}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:42.433920Z INFO Node{id=3}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:42.434850Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:42.435003Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:52:42.435714Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:42.439172Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:42.439207Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:52:42.439601Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727562426) } [INFO] [stdout] 2025-09-01T11:52:42.440491Z INFO Node{id=18}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:42.440566Z INFO Node{id=18}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:42.442041Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:52:42.442775Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727562430) } [INFO] [stdout] 2025-09-01T11:52:42.442875Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727562430) } [INFO] [stdout] 2025-09-01T11:52:42.443632Z INFO Node{id=30}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:52:42.443659Z INFO Node{id=30}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:42.445326Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727562433) } [INFO] [stdout] 2025-09-01T11:52:42.446653Z INFO Node{id=26}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:52:42.446678Z INFO Node{id=26}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:42.446703Z INFO Node{id=14}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:52:42.446726Z INFO Node{id=14}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:42.449737Z INFO Node{id=27}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:52:42.449778Z INFO Node{id=27}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:42.451040Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.452905Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727562440) } [INFO] [stdout] 2025-09-01T11:52:42.453572Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.454570Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:52:42.454590Z ERROR Node{id=30}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=13 leader=1 local=30 [INFO] [stdout] 2025-09-01T11:52:42.455504Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.455622Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.455888Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:42.456349Z INFO Node{id=18}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:52:42.456377Z INFO Node{id=18}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:42.456907Z INFO Node{id=30}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:42.457518Z INFO Node{id=30}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:42.457784Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:42.458726Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:52:42.460354Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:52:42.460418Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:42.462240Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:52:42.462259Z ERROR Node{id=14}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=7 leader=1 local=14 [INFO] [stdout] 2025-09-01T11:52:42.462915Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.463235Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:42.463951Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:42.464685Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:42.464770Z INFO Node{id=14}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:42.464846Z INFO Node{id=14}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:42.466044Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:42.466109Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:42.467050Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4027, state_accept_seq: 4026, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:42.467389Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:42.468130Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 6272, state_accept_seq: 6271, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:42.468862Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:52:42.468885Z ERROR Node{id=18}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=13 leader=1 local=18 [INFO] [stdout] 2025-09-01T11:52:42.469416Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:42.469930Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:42.470184Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:42.470673Z INFO Node{id=6}: sequenced_broadcast: 78|Subscriber(2): Added, allow_drop: true, next_sequence: 4027, min_allowed_seq: 2588 [INFO] [stdout] 2025-09-01T11:52:42.471226Z INFO Node{id=18}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:42.471285Z INFO Node{id=18}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:42.472146Z INFO Node{id=9}: sequenced_broadcast: 93|Subscriber(1): Added, allow_drop: true, next_sequence: 4391, min_allowed_seq: 2343 [INFO] [stdout] 2025-09-01T11:52:42.472402Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:42.473097Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:42.475558Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4027) [INFO] [stdout] 2025-09-01T11:52:42.475619Z INFO Node{id=26}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:52:42.477045Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4391) [INFO] [stdout] 2025-09-01T11:52:42.477125Z INFO Node{id=27}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:42.478306Z INFO Node{id=27}: sequenced_broadcast: 104|SequencedBroadcastWorker Started id=104 [INFO] [stdout] 2025-09-01T11:52:42.479219Z INFO Node{id=27}: sequenced_broadcast: 83|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:42.479236Z INFO Node{id=27}: sequenced_broadcast: 83|SequencedBroadcastWorker Stopped id=83 elapsed=29.563317501s iter=2537 [INFO] [stdout] 2025-09-01T11:52:42.522517Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:42.522571Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:52:42.524195Z INFO Node{id=15}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:42.524263Z INFO Node{id=15}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:42.527946Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:52:42.539073Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727562524) } [INFO] [stdout] 2025-09-01T11:52:42.542847Z INFO Node{id=15}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:52:42.542877Z INFO Node{id=15}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:42.552037Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.557923Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:52:42.562964Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:42.564503Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 1731, state_accept_seq: 1730, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:42.567206Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:42.567311Z INFO Node{id=2}: sequenced_broadcast: 102|Subscriber rejected, seq(1731) < min_allowed(4099) [INFO] [stdout] 2025-09-01T11:52:42.570118Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:42.573301Z INFO Node{id=2}: sequenced_broadcast: 102|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:52:42.580344Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:52:42.580441Z INFO Node{id=15}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:42.582101Z INFO Node{id=15}: sequenced_broadcast: 105|SequencedBroadcastWorker Started id=105 [INFO] [stdout] 2025-09-01T11:52:42.583836Z INFO Node{id=15}: sequenced_broadcast: 68|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:42.583855Z INFO Node{id=15}: sequenced_broadcast: 68|SequencedBroadcastWorker Stopped id=68 elapsed=32.211448752s iter=733 [INFO] [stdout] 2025-09-01T11:52:42.588551Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.715522Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:42.715573Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:52:42.717347Z INFO Node{id=4}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:42.717420Z INFO Node{id=4}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:42.721293Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:52:42.731012Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727562717) } [INFO] [stdout] 2025-09-01T11:52:42.735118Z INFO Node{id=4}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:52:42.735157Z INFO Node{id=4}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:42.744701Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.760263Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:42.760474Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:52:42.766010Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:42.768677Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4636, state_accept_seq: 4635, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:42.770037Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.770182Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:42.773028Z INFO Node{id=12}: sequenced_broadcast: 98|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2462 [INFO] [stdout] 2025-09-01T11:52:42.780249Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:52:42.780352Z INFO Node{id=4}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:42.782101Z INFO Node{id=4}: sequenced_broadcast: 106|SequencedBroadcastWorker Started id=106 [INFO] [stdout] 2025-09-01T11:52:42.783497Z INFO Node{id=4}: sequenced_broadcast: 76|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:42.783515Z INFO Node{id=4}: sequenced_broadcast: 76|SequencedBroadcastWorker Stopped id=76 elapsed=32.006188344s iter=1926 [INFO] [stdout] 2025-09-01T11:52:42.787904Z ERROR Node{id=6}:connect{target=4}:ReadPeer{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:42.792936Z WARN Node{id=6}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:42.792958Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:42.921803Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:42.921846Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:52:42.925220Z INFO Node{id=21}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:42.925292Z INFO Node{id=21}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:42.927394Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:52:42.940655Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727562925) } [INFO] [stdout] 2025-09-01T11:52:42.947267Z INFO Node{id=21}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=22 [INFO] [stdout] 2025-09-01T11:52:42.947322Z INFO Node{id=21}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:42.956044Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:42.961902Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:52:42.971202Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:42.973064Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4027, state_accept_seq: 4026, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:42.976087Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:42.976160Z INFO Node{id=15}: sequenced_broadcast: 105|Subscriber rejected, seq(4027) < min_allowed(4099) [INFO] [stdout] 2025-09-01T11:52:42.978799Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:42.982551Z INFO Node{id=15}: sequenced_broadcast: 105|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:52:42.990803Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:52:42.990951Z INFO Node{id=21}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:42.992115Z INFO Node{id=21}: sequenced_broadcast: 107|SequencedBroadcastWorker Started id=107 [INFO] [stdout] 2025-09-01T11:52:42.993605Z INFO Node{id=21}: sequenced_broadcast: 97|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:42.993621Z INFO Node{id=21}: sequenced_broadcast: 97|SequencedBroadcastWorker Stopped id=97 elapsed=13.720945984s iter=1011 [INFO] [stdout] 2025-09-01T11:52:43.136194Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:43.376276Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:43.389564Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:43.395835Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:43.395874Z ERROR Node{id=8}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:43.421707Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:43.421752Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:52:43.424855Z INFO Node{id=11}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:43.424933Z INFO Node{id=11}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:43.429650Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:52:43.434947Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:43.434986Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:52:43.437462Z INFO Node{id=3}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:43.437550Z INFO Node{id=3}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:43.438989Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:52:43.440604Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727559862) } [INFO] [stdout] 2025-09-01T11:52:43.445870Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=3583 [INFO] [stdout] 2025-09-01T11:52:43.445917Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:43.446619Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727563424) } [INFO] [stdout] 2025-09-01T11:52:43.454062Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727563437) } [INFO] [stdout] 2025-09-01T11:52:43.455467Z INFO Node{id=11}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=31 [INFO] [stdout] 2025-09-01T11:52:43.455499Z INFO Node{id=11}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:43.457275Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:43.457308Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:52:43.459369Z INFO Node{id=30}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:43.459444Z INFO Node{id=30}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:43.460397Z INFO Node{id=3}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=23 [INFO] [stdout] 2025-09-01T11:52:43.460425Z INFO Node{id=3}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:43.461694Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:52:43.470345Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:43.470391Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:52:43.471458Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:43.472445Z INFO Node{id=14}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:43.472522Z INFO Node{id=14}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:43.472813Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:43.472842Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:52:43.474836Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:52:43.474950Z INFO Node{id=18}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:43.475013Z INFO Node{id=18}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:43.477578Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:52:43.480119Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:43.481483Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:52:43.481505Z ERROR Node{id=11}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=7 leader=1 local=11 [INFO] [stdout] 2025-09-01T11:52:43.482480Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727563459) } [INFO] [stdout] 2025-09-01T11:52:43.484193Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:43.484253Z INFO Node{id=11}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:43.486269Z INFO Node{id=11}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:43.486982Z ERROR ReadClient{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:43.492018Z INFO Node{id=30}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=33 [INFO] [stdout] 2025-09-01T11:52:43.492057Z INFO Node{id=30}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:43.492928Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:43.497796Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727563475) } [INFO] [stdout] 2025-09-01T11:52:43.498965Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727563472) } [INFO] [stdout] 2025-09-01T11:52:43.504320Z INFO Node{id=18}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=29 [INFO] [stdout] 2025-09-01T11:52:43.504361Z INFO Node{id=18}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:43.505867Z INFO Node{id=14}:connect{target=19}: sharedstate::worker::sync_manager: got pong from peer latency=33 [INFO] [stdout] 2025-09-01T11:52:43.505897Z INFO Node{id=14}:connect{target=19}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:43.506401Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:43.511232Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:43.517935Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:52:43.519023Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:43.521289Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727561411) } [INFO] [stdout] 2025-09-01T11:52:43.522964Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:43.525388Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:43.526577Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:52:43.526602Z ERROR Node{id=18}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=20 leader=1 local=18 [INFO] [stdout] 2025-09-01T11:52:43.530841Z INFO Node{id=16}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=2119 [INFO] [stdout] 2025-09-01T11:52:43.530878Z INFO Node{id=16}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:43.531151Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:43.532696Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:43.533402Z INFO Node{id=18}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:43.533470Z INFO Node{id=18}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:43.533686Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:52:43.536820Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:43.538336Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 19 } [INFO] [stdout] 2025-09-01T11:52:43.541021Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:43.541453Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:43.542973Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:43.545039Z INFO Node{id=12}: sequenced_broadcast: 98|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2462 [INFO] [stdout] 2025-09-01T11:52:43.553419Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:43.555149Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:52:43.555216Z INFO Node{id=3}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:52:43.555386Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4037, state_accept_seq: 4036, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:43.560196Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 6217, state_accept_seq: 6216, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:43.562146Z INFO Node{id=29}: sequenced_broadcast: 89|Subscriber(2): Added, allow_drop: true, next_sequence: 4037, min_allowed_seq: 2051 [INFO] [stdout] 2025-09-01T11:52:43.562577Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:43.574227Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4037) [INFO] [stdout] 2025-09-01T11:52:43.574313Z INFO Node{id=30}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:52:43.577196Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:43.584166Z INFO Node{id=19}: sequenced_broadcast: 91|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2051 [INFO] [stdout] 2025-09-01T11:52:43.595403Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:52:43.595530Z INFO Node{id=14}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:43.598967Z INFO Node{id=14}: sequenced_broadcast: 108|SequencedBroadcastWorker Started id=108 [INFO] [stdout] 2025-09-01T11:52:43.600494Z INFO Node{id=30}: sequenced_broadcast: 2|Subscriber(84): channel closed, dropping [INFO] [stdout] 2025-09-01T11:52:43.601198Z INFO Node{id=14}: sequenced_broadcast: 72|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:43.601217Z INFO Node{id=14}: sequenced_broadcast: 72|SequencedBroadcastWorker Stopped id=72 elapsed=33.212520697s iter=2632 [INFO] [stdout] 2025-09-01T11:52:43.607002Z INFO ReadClient{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:43.612400Z ERROR Node{id=23}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:43.614720Z WARN Node{id=23}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:43.614743Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:43.772202Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:43.778381Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 17]) [INFO] [stdout] 2025-09-01T11:52:43.798262Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:43.798316Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:52:43.800626Z INFO Node{id=6}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:43.800699Z INFO Node{id=6}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:43.802984Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:52:43.823202Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727563800) } [INFO] [stdout] 2025-09-01T11:52:43.823240Z INFO Node{id=5}: sharedstate::worker::sync_manager: Discovered Peer client_id=6 [INFO] [stdout] 2025-09-01T11:52:43.825832Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:52:43.825862Z INFO Node{id=5}: sharedstate::worker::sync_manager: Broadcasted 3 peers to 2 peers [INFO] [stdout] 2025-09-01T11:52:43.830151Z INFO Node{id=6}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=30 [INFO] [stdout] 2025-09-01T11:52:43.830190Z INFO Node{id=6}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:43.832698Z ERROR Node{id=6}:connect{target=5}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:52:43.832829Z INFO Node{id=6}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:43.834944Z INFO Node{id=6}:connect{target=5}:WritePeer{remote=5}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:43.838047Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:43.842316Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(5) [INFO] [stdout] 2025-09-01T11:52:43.843698Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:44.487038Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:44.487102Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:52:44.489928Z INFO Node{id=11}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:44.490012Z INFO Node{id=11}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:44.492488Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:52:44.517493Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727564490) } [INFO] [stdout] 2025-09-01T11:52:44.526918Z INFO Node{id=11}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=36 [INFO] [stdout] 2025-09-01T11:52:44.526964Z INFO Node{id=11}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:44.534463Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:44.534509Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:52:44.536115Z INFO Node{id=18}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:44.536181Z INFO Node{id=18}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:44.537999Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:52:44.539394Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:44.547345Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:52:44.547382Z ERROR Node{id=11}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=23 leader=1 local=11 [INFO] [stdout] 2025-09-01T11:52:44.550340Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:44.550401Z INFO Node{id=11}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:44.552326Z INFO Node{id=11}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:44.552965Z ERROR ReadClient{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:44.557971Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:44.625004Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:44.625051Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:52:44.626672Z INFO Node{id=23}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:44.626734Z INFO Node{id=23}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:44.628802Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:52:44.635917Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:44.644524Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727564626) } [INFO] [stdout] 2025-09-01T11:52:44.649890Z INFO Node{id=23}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=23 [INFO] [stdout] 2025-09-01T11:52:44.649931Z INFO Node{id=23}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:44.659241Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:44.666939Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:52:44.675572Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:44.678626Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 6217, state_accept_seq: 6216, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:44.680872Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:44.682641Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:44.685522Z INFO Node{id=27}: sequenced_broadcast: 104|Subscriber(1): Added, allow_drop: true, next_sequence: 4391, min_allowed_seq: 4391 [INFO] [stdout] 2025-09-01T11:52:44.698132Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4391) [INFO] [stdout] 2025-09-01T11:52:44.698236Z INFO Node{id=23}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:44.702096Z INFO Node{id=23}: sequenced_broadcast: 109|SequencedBroadcastWorker Started id=109 [INFO] [stdout] 2025-09-01T11:52:44.704040Z INFO Node{id=23}: sequenced_broadcast: 100|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:44.704062Z INFO Node{id=23}: sequenced_broadcast: 100|SequencedBroadcastWorker Stopped id=100 elapsed=4.601949508s iter=34 [INFO] [stdout] 2025-09-01T11:52:44.708910Z INFO ReadClient{remote=28}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:44.711626Z ERROR Node{id=28}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:44.713963Z WARN Node{id=28}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:44.713984Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:44.716163Z INFO Node{id=28}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:44.799224Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:44.844456Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:44.844504Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:52:44.846819Z INFO Node{id=6}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:44.846901Z INFO Node{id=6}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:44.851780Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:52:44.877367Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727564846) } [INFO] [stdout] 2025-09-01T11:52:44.887399Z INFO Node{id=6}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=41 [INFO] [stdout] 2025-09-01T11:52:44.887441Z INFO Node{id=6}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:44.906006Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:44.918005Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:52:44.932264Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:44.946601Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4636, state_accept_seq: 4635, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:44.955623Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:52:44.959656Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:44.963577Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:44.964555Z INFO Node{id=32}: sequenced_broadcast: 101|Subscriber rejected, max_seq(4594) < seq(4597) [INFO] [stdout] 2025-09-01T11:52:44.966108Z ERROR Node{id=5}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:44.968325Z ERROR Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: channel closed waiting for leader path [INFO] [stdout] 2025-09-01T11:52:44.970427Z INFO Node{id=5}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:44.970506Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:52:44.988072Z INFO Node{id=32}: sequenced_broadcast: 101|Subscriber(1): Added, allow_drop: true, next_sequence: 4597, min_allowed_seq: 4027 [INFO] [stdout] 2025-09-01T11:52:45.006626Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4597) [INFO] [stdout] 2025-09-01T11:52:45.006753Z INFO Node{id=6}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:45.009087Z INFO Node{id=6}: sequenced_broadcast: 110|SequencedBroadcastWorker Started id=110 [INFO] [stdout] 2025-09-01T11:52:45.011413Z INFO Node{id=6}: sequenced_broadcast: 78|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:45.011433Z INFO Node{id=6}: sequenced_broadcast: 78|SequencedBroadcastWorker Stopped id=78 elapsed=34.042147402s iter=2156 [INFO] [stdout] 2025-09-01T11:52:45.014538Z ERROR ReadClient{remote=26}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:52:45.017402Z ERROR Node{id=24}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:45.017484Z ERROR Node{id=26}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:45.019442Z WARN Node{id=24}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:45.019464Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:45.109911Z WARN Node{id=26}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:45.109946Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:45.212865Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:45.225484Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:52:45.284514Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:45.295032Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:52:45.392511Z ERROR Node{id=20}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:52:45.394488Z WARN Node{id=20}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:45.394511Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:45.395855Z INFO Node{id=20}:connect{target=17}:WritePeer{remote=17}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:45.398105Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:45.407795Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:45.413117Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:45.413149Z ERROR Node{id=8}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:45.553020Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:45.553067Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:52:45.554984Z INFO Node{id=11}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:45.555057Z INFO Node{id=11}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:45.556379Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:52:45.598670Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727564536) } [INFO] [stdout] 2025-09-01T11:52:45.601564Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727565555) } [INFO] [stdout] 2025-09-01T11:52:45.603281Z INFO Node{id=18}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=1067 [INFO] [stdout] 2025-09-01T11:52:45.603317Z INFO Node{id=18}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:45.606318Z INFO Node{id=11}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=51 [INFO] [stdout] 2025-09-01T11:52:45.606350Z INFO Node{id=11}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:45.649165Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:45.655202Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:52:45.700462Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:45.704939Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 6264, state_accept_seq: 6263, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:45.705577Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:45.725297Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:45.725344Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:52:45.726793Z INFO Node{id=28}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:45.726866Z INFO Node{id=28}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:45.728581Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:52:45.733146Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:45.753655Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:45.757173Z INFO Node{id=32}: sequenced_broadcast: 101|Subscriber(2): Added, allow_drop: true, next_sequence: 4636, min_allowed_seq: 4027 [INFO] [stdout] 2025-09-01T11:52:45.764666Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4636) [INFO] [stdout] 2025-09-01T11:52:45.764782Z INFO Node{id=11}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:45.766020Z INFO Node{id=11}: sequenced_broadcast: 111|SequencedBroadcastWorker Started id=111 [INFO] [stdout] 2025-09-01T11:52:45.767715Z INFO Node{id=11}: sequenced_broadcast: 49|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:45.767735Z INFO Node{id=11}: sequenced_broadcast: 49|SequencedBroadcastWorker Stopped id=49 elapsed=35.422998715s iter=2646 [INFO] [stdout] 2025-09-01T11:52:45.973136Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:45.973179Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=2 peers=3 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:52:45.974487Z INFO Node{id=5}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:45.974552Z INFO Node{id=5}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:45.976139Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:52:45.988081Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727565974) } [INFO] [stdout] 2025-09-01T11:52:45.992545Z INFO Node{id=5}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:52:45.992580Z INFO Node{id=5}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:46.002650Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:46.010697Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:52:46.017426Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:46.019354Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeRecovery(RecoverableStateDetails { id: 12240540419735701133, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:46.024182Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:46.024308Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:46.027391Z INFO Node{id=6}: sequenced_broadcast: 110|Subscriber(1): Added, allow_drop: true, next_sequence: 4636, min_allowed_seq: 4597 [INFO] [stdout] 2025-09-01T11:52:46.033421Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:46.033463Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:52:46.035113Z INFO Node{id=24}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:46.035186Z INFO Node{id=24}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:46.035961Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4636) [INFO] [stdout] 2025-09-01T11:52:46.036051Z INFO Node{id=5}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:46.036978Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:52:46.037658Z INFO Node{id=5}: sequenced_broadcast: 112|SequencedBroadcastWorker Started id=112 [INFO] [stdout] 2025-09-01T11:52:46.039445Z INFO Node{id=5}: sequenced_broadcast: 19|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:46.039467Z INFO Node{id=5}: sequenced_broadcast: 19|SequencedBroadcastWorker Stopped id=19 elapsed=35.722105865s iter=2 [INFO] [stdout] 2025-09-01T11:52:46.049706Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727566035) } [INFO] [stdout] 2025-09-01T11:52:46.054297Z INFO Node{id=24}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=19 [INFO] [stdout] 2025-09-01T11:52:46.054331Z INFO Node{id=24}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:46.063100Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:46.069474Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:52:46.078640Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:46.081590Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4636, state_accept_seq: 4635, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:46.083188Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:46.084802Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:46.087309Z INFO Node{id=21}: sequenced_broadcast: 107|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:52:46.098004Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:52:46.098134Z INFO Node{id=24}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:46.100243Z INFO Node{id=24}: sequenced_broadcast: 113|SequencedBroadcastWorker Started id=113 [INFO] [stdout] 2025-09-01T11:52:46.102944Z INFO Node{id=24}: sequenced_broadcast: 99|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:46.102974Z INFO Node{id=24}: sequenced_broadcast: 99|SequencedBroadcastWorker Stopped id=99 elapsed=11.251496444s iter=74 [INFO] [stdout] 2025-09-01T11:52:46.126616Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:46.126665Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:52:46.128491Z INFO Node{id=26}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:46.128566Z INFO Node{id=26}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:46.130179Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:52:46.148353Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727566128) } [INFO] [stdout] 2025-09-01T11:52:46.155134Z INFO Node{id=26}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=27 [INFO] [stdout] 2025-09-01T11:52:46.155172Z INFO Node{id=26}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:46.168006Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:46.182891Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:52:46.182934Z ERROR Node{id=26}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[8] peer=8 leader=1 local=26 [INFO] [stdout] 2025-09-01T11:52:46.184381Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:46.187128Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:46.188599Z INFO Node{id=26}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:46.189427Z INFO Node{id=26}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:46.193609Z ERROR ReadClient{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:46.374778Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:46.398029Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:46.398077Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:52:46.400474Z INFO Node{id=20}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:46.400546Z INFO Node{id=20}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:46.403037Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:52:46.408324Z ERROR Node{id=13}:connect{target=26}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:52:46.410136Z INFO Node{id=13}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:46.410209Z INFO Node{id=13}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:46.410269Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(26) [INFO] [stdout] 2025-09-01T11:52:46.412671Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:46.419702Z ERROR Node{id=25}:connect{target=26}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:52:46.427745Z INFO Node{id=25}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:46.427879Z INFO Node{id=25}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:46.427963Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(26) [INFO] [stdout] 2025-09-01T11:52:46.430612Z ERROR ReadClient{remote=25}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:46.689981Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:46.706552Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:46.712748Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:46.890155Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:46.895631Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4463, state_accept_seq: 4462, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:46.896930Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:47.029391Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:47.033284Z INFO Node{id=8}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:47.037692Z ERROR ReadClient{remote=8}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:47.080815Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:47.189992Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:47.190038Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:52:47.192272Z INFO Node{id=26}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:47.192355Z INFO Node{id=26}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:47.194666Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:52:47.212810Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727567192) } [INFO] [stdout] 2025-09-01T11:52:47.221635Z INFO Node{id=26}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=29 [INFO] [stdout] 2025-09-01T11:52:47.221683Z INFO Node{id=26}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:47.235331Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:47.247850Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:52:47.247889Z ERROR Node{id=26}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=13 leader=1 local=26 [INFO] [stdout] 2025-09-01T11:52:47.250896Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:47.252934Z INFO Node{id=26}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:47.253017Z INFO Node{id=26}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:47.254936Z ERROR ReadClient{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:47.255667Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:47.412950Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:47.412999Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:52:47.414709Z INFO Node{id=13}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:47.414798Z INFO Node{id=13}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:47.416361Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:52:47.431079Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:47.431124Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:52:47.433343Z INFO Node{id=25}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:47.433411Z INFO Node{id=25}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:47.435831Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:52:47.453430Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727567433) } [INFO] [stdout] 2025-09-01T11:52:47.459945Z INFO Node{id=25}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=26 [INFO] [stdout] 2025-09-01T11:52:47.459986Z INFO Node{id=25}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:47.472659Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:47.478621Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:52:47.489087Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:47.491569Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 6276, state_accept_seq: 6275, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:47.493270Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:47.495283Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:47.497956Z INFO Node{id=14}: sequenced_broadcast: 108|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:52:47.508017Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:52:47.508136Z INFO Node{id=25}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:47.510028Z INFO Node{id=25}: sequenced_broadcast: 114|SequencedBroadcastWorker Started id=114 [INFO] [stdout] 2025-09-01T11:52:47.512078Z INFO Node{id=25}: sequenced_broadcast: 81|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:47.512100Z INFO Node{id=25}: sequenced_broadcast: 81|SequencedBroadcastWorker Stopped id=81 elapsed=35.040125888s iter=2558 [INFO] [stdout] 2025-09-01T11:52:47.654219Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:47.666231Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:52:47.745852Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727561403) } [INFO] [stdout] 2025-09-01T11:52:47.768090Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727561417) } [INFO] [stdout] 2025-09-01T11:52:47.900904Z ERROR WriteClient{remote=12}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:52:47.987867Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727565726) } [INFO] [stdout] 2025-09-01T11:52:47.994026Z INFO Node{id=28}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=2268 [INFO] [stdout] 2025-09-01T11:52:47.994065Z INFO Node{id=28}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:48.042775Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:48.042826Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:52:48.045101Z INFO Node{id=8}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:48.045164Z INFO Node{id=8}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:48.046959Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:52:48.197001Z ERROR Node{id=23}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:52:48.201298Z WARN Node{id=23}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:48.201325Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:48.203457Z ERROR ReadClient{remote=23}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:48.255007Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:48.255055Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:52:48.256505Z INFO Node{id=26}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:48.256573Z INFO Node{id=26}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:48.257987Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:52:48.274986Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727568256) } [INFO] [stdout] 2025-09-01T11:52:48.280020Z INFO Node{id=26}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=24 [INFO] [stdout] 2025-09-01T11:52:48.280060Z INFO Node{id=26}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:48.294174Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:48.300568Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:52:48.309044Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:48.313723Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4279, state_accept_seq: 4278, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:48.314053Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:48.320918Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:48.325569Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4636, state_accept_seq: 4635, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:48.327461Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:48.340563Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:48.343926Z INFO Node{id=14}: sequenced_broadcast: 108|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:52:48.352145Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:52:48.352261Z INFO Node{id=26}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:48.354434Z INFO Node{id=26}: sequenced_broadcast: 115|SequencedBroadcastWorker Started id=115 [INFO] [stdout] 2025-09-01T11:52:48.358097Z INFO Node{id=26}: sequenced_broadcast: 82|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:48.358123Z INFO Node{id=26}: sequenced_broadcast: 82|SequencedBroadcastWorker Stopped id=82 elapsed=35.586209899s iter=2286 [INFO] [stdout] 2025-09-01T11:52:48.363887Z ERROR Node{id=22}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:48.364587Z WARN Node{id=22}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:48.364604Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:48.691868Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:48.695131Z INFO Node{id=17}: sequenced_broadcast: 85|Subscriber(3): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2051 [INFO] [stdout] 2025-09-01T11:52:48.756897Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(12): lag started thresh(8192) < lag(8194) [INFO] [stdout] 2025-09-01T11:52:48.820553Z ERROR ReadClient{remote=15}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:52:48.878540Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:49.214016Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:49.214067Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:52:49.216728Z INFO Node{id=23}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:49.216826Z INFO Node{id=23}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:49.218926Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:52:49.242174Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727569216) } [INFO] [stdout] 2025-09-01T11:52:49.255928Z INFO Node{id=23}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=39 [INFO] [stdout] 2025-09-01T11:52:49.256022Z INFO Node{id=23}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:49.272625Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:49.284521Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:52:49.299143Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:49.303356Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4391, state_accept_seq: 4390, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:49.305487Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:49.307277Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:49.314654Z INFO Node{id=25}: sequenced_broadcast: 114|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:52:49.330035Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:52:49.330157Z INFO Node{id=23}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:49.333668Z INFO Node{id=23}: sequenced_broadcast: 116|SequencedBroadcastWorker Started id=116 [INFO] [stdout] 2025-09-01T11:52:49.335794Z INFO Node{id=23}: sequenced_broadcast: 109|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:49.335829Z INFO Node{id=23}: sequenced_broadcast: 109|SequencedBroadcastWorker Stopped id=109 elapsed=4.633706371s iter=2 [INFO] [stdout] 2025-09-01T11:52:49.374226Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:49.374275Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:52:49.376508Z INFO Node{id=22}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:49.376585Z INFO Node{id=22}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:49.380601Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:52:49.567090Z ERROR WriteClient{remote=16}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:52:50.014214Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:50.021034Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3]) [INFO] [stdout] 2025-09-01T11:52:50.035047Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:50.069568Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:50.075932Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:50.075966Z ERROR Node{id=9}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:50.763470Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(12): lag too high for too long (2.006571338s), dropping [INFO] [stdout] 2025-09-01T11:52:50.902745Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(4): lag started thresh(8192) < lag(8193) [INFO] [stdout] 2025-09-01T11:52:50.989325Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:51.029012Z ERROR Node{id=7}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:52:51.031244Z WARN Node{id=7}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:51.031266Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:51.032637Z INFO Node{id=7}:connect{target=3}:WritePeer{remote=3}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:51.034720Z ERROR ReadClient{remote=7}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:51.326037Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727567414) } [INFO] [stdout] 2025-09-01T11:52:51.330922Z INFO Node{id=13}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=3916 [INFO] [stdout] 2025-09-01T11:52:51.330976Z INFO Node{id=13}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:51.403608Z ERROR Node{id=20}:connect{target=26}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:52:51.405235Z INFO Node{id=20}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:51.405310Z INFO Node{id=20}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:51.405370Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(26) [INFO] [stdout] 2025-09-01T11:52:51.407313Z ERROR ReadClient{remote=20}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:51.426070Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:51.426926Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:51.461459Z ERROR WriteClient{remote=24}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:52:51.464553Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:51.465561Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:51.468931Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:51.468955Z ERROR Node{id=32}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:51.491510Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:51.757100Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:51.766858Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4391, state_accept_seq: 4390, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:51.767140Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:51.771387Z ERROR Node{id=18}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:51.772375Z WARN Node{id=18}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:51.772393Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:51.774176Z INFO Node{id=18}:connect{target=12}:WritePeer{remote=12}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:51.878182Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:51.935923Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727566400) } [INFO] [stdout] 2025-09-01T11:52:52.033645Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:52.033693Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:52:52.035719Z INFO Node{id=7}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:52.035809Z INFO Node{id=7}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:52.037122Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:52:52.060399Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727572035) } [INFO] [stdout] 2025-09-01T11:52:52.065576Z INFO Node{id=7}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=30 [INFO] [stdout] 2025-09-01T11:52:52.065618Z INFO Node{id=7}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:52.078233Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:52.084594Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:52.087410Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:52.093098Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:52:52.094357Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:52.094379Z ERROR Node{id=9}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:52.111480Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:52.115709Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:52.117609Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4279, state_accept_seq: 4278, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:52.120848Z INFO Node{id=6}: sequenced_broadcast: 110|Subscriber rejected, seq(4279) < min_allowed(4597) [INFO] [stdout] 2025-09-01T11:52:52.143046Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:52.147197Z INFO Node{id=6}: sequenced_broadcast: 110|Subscriber(2): Added, allow_drop: true, next_sequence: 4636, min_allowed_seq: 4597 [INFO] [stdout] 2025-09-01T11:52:52.154681Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4636) [INFO] [stdout] 2025-09-01T11:52:52.154812Z INFO Node{id=7}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:52.156162Z INFO Node{id=7}: sequenced_broadcast: 117|SequencedBroadcastWorker Started id=117 [INFO] [stdout] 2025-09-01T11:52:52.157868Z INFO Node{id=7}: sequenced_broadcast: 77|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:52.157886Z INFO Node{id=7}: sequenced_broadcast: 77|SequencedBroadcastWorker Stopped id=77 elapsed=41.30653378s iter=2129 [INFO] [stdout] 2025-09-01T11:52:52.279087Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727568045) } [INFO] [stdout] 2025-09-01T11:52:52.286037Z INFO Node{id=8}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=4241 [INFO] [stdout] 2025-09-01T11:52:52.286080Z INFO Node{id=8}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:52.351072Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:52.361726Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:52:52.408095Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:52.408139Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:52:52.410206Z INFO Node{id=20}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:52.410268Z INFO Node{id=20}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:52.411472Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:52:52.462500Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:52.471382Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:52.476795Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([9]) [INFO] [stdout] 2025-09-01T11:52:52.559753Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:52.649439Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:52.652560Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2]) [INFO] [stdout] 2025-09-01T11:52:52.762953Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:52.773605Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:52.773656Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:52:52.776144Z INFO Node{id=18}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:52.776219Z INFO Node{id=18}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:52.778278Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:52:52.908911Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(4): lag too high for too long (2.006166988s), dropping [INFO] [stdout] 2025-09-01T11:52:52.971237Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:52.988301Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:53.007854Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 15]) [INFO] [stdout] 2025-09-01T11:52:53.471214Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:53.519277Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:53.524106Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:53.536025Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:53.539796Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:53.540051Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:53.540071Z ERROR Node{id=32}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:53.788579Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:54.096597Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:54.109938Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:54.116276Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:54.116309Z ERROR Node{id=9}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:54.363691Z ERROR Node{id=9}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:52:54.365133Z WARN Node{id=9}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:54.365153Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:54.369018Z ERROR ReadClient{remote=9}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:54.379179Z ERROR Node{id=22}:connect{target=19}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:52:54.381357Z INFO Node{id=22}:connect{target=19}:WritePeer{remote=19}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:54.381434Z INFO Node{id=22}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:54.382845Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:54.393559Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(19) [INFO] [stdout] 2025-09-01T11:52:54.479362Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:54.487390Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:54.492206Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:54.492234Z ERROR Node{id=27}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:54.664049Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727569376) } [INFO] [stdout] 2025-09-01T11:52:54.921948Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:55.071579Z ERROR Node{id=4}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:52:55.073035Z WARN Node{id=4}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:55.073053Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:55.076433Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:55.297672Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:55.335827Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:55.339864Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 17]) [INFO] [stdout] 2025-09-01T11:52:55.374135Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:55.374179Z INFO Node{id=9}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:52:55.376256Z INFO Node{id=9}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:55.376326Z INFO Node{id=9}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:55.377828Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=9 [INFO] [stdout] 2025-09-01T11:52:55.394931Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:55.394975Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:52:55.396829Z INFO Node{id=22}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:55.396896Z INFO Node{id=22}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:55.398433Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:52:55.447019Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:55.450648Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10]) [INFO] [stdout] 2025-09-01T11:52:55.543728Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:55.601267Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:55.607662Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:55.607702Z ERROR Node{id=32}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:55.657746Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:56.012099Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:56.072064Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:56.080240Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:56.080279Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:52:56.081941Z INFO Node{id=4}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:56.082004Z INFO Node{id=4}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:56.082196Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:56.083877Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:52:56.100143Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727576082) } [INFO] [stdout] 2025-09-01T11:52:56.105852Z INFO Node{id=4}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=23 [INFO] [stdout] 2025-09-01T11:52:56.105886Z INFO Node{id=4}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:56.114700Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:56.122610Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:52:56.122653Z ERROR Node{id=4}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=9 leader=1 local=4 [INFO] [stdout] 2025-09-01T11:52:56.124053Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:56.124114Z INFO Node{id=4}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:56.125620Z INFO Node{id=4}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:56.125812Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:56.129675Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:56.270429Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:56.276136Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:52:56.315505Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:56.322078Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:52:56.378354Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727572410) } [INFO] [stdout] 2025-09-01T11:52:56.382019Z INFO Node{id=20}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=3972 [INFO] [stdout] 2025-09-01T11:52:56.382051Z INFO Node{id=20}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:56.496022Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:56.506384Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:56.511019Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:56.511051Z ERROR Node{id=27}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:56.735723Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:56.945745Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727572776) } [INFO] [stdout] 2025-09-01T11:52:56.954662Z INFO Node{id=18}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=4178 [INFO] [stdout] 2025-09-01T11:52:56.954704Z INFO Node{id=18}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:57.130013Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:57.130060Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:52:57.132211Z INFO Node{id=4}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:57.132284Z INFO Node{id=4}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:57.134552Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:52:57.237518Z ERROR Node{id=12}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:52:57.238741Z WARN Node{id=12}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:57.238780Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:57.356153Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(14): lag started thresh(8192) < lag(8195) [INFO] [stdout] 2025-09-01T11:52:57.366266Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:57.371324Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:57.373091Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 6217, state_accept_seq: 6216, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:57.376935Z ERROR Node{id=28}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:57.378953Z WARN Node{id=28}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:57.378974Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:57.380404Z INFO Node{id=28}:connect{target=29}:WritePeer{remote=29}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:57.395578Z ERROR ReadClient{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:57.480431Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:57.609975Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:57.678000Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:57.685527Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19]) [INFO] [stdout] 2025-09-01T11:52:57.729648Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:57.731613Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:57.738680Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:57.738716Z ERROR Node{id=32}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:57.866174Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(8): lag started thresh(8192) < lag(8193) [INFO] [stdout] 2025-09-01T11:52:57.866615Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:52:57.872278Z ERROR Node{id=17}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:57.881975Z WARN Node{id=17}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:57.882010Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:58.167307Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: Ping(1756727575376) } [INFO] [stdout] 2025-09-01T11:52:58.187119Z INFO Node{id=9}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=2811 [INFO] [stdout] 2025-09-01T11:52:58.187167Z INFO Node{id=9}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:58.251423Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:58.251468Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:52:58.253110Z INFO Node{id=12}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:58.253182Z INFO Node{id=12}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:58.254534Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:52:58.271125Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727578253) } [INFO] [stdout] 2025-09-01T11:52:58.278186Z INFO Node{id=12}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=25 [INFO] [stdout] 2025-09-01T11:52:58.278232Z INFO Node{id=12}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:58.288638Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:58.302874Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:58.302935Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:58.304674Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:52:58.304709Z ERROR Node{id=12}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=8 leader=1 local=12 [INFO] [stdout] 2025-09-01T11:52:58.306981Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:58.307199Z INFO Node{id=12}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:58.308616Z INFO Node{id=12}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:58.310057Z ERROR ReadClient{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:58.367077Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:58.368314Z INFO Node{id=31}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:58.381255Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:58.381305Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:52:58.383114Z INFO Node{id=28}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:58.383187Z INFO Node{id=28}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:58.384632Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:52:58.399032Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727578383) } [INFO] [stdout] 2025-09-01T11:52:58.408063Z INFO Node{id=28}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=25 [INFO] [stdout] 2025-09-01T11:52:58.408113Z INFO Node{id=28}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:58.419913Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:58.429142Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:52:58.444392Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:58.447324Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 6217, state_accept_seq: 6216, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:58.448976Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:58.462998Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:52:58.465996Z INFO Node{id=2}: sequenced_broadcast: 102|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:52:58.473978Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:52:58.474104Z INFO Node{id=28}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:52:58.475743Z INFO Node{id=28}: sequenced_broadcast: 118|SequencedBroadcastWorker Started id=118 [INFO] [stdout] 2025-09-01T11:52:58.478082Z INFO Node{id=28}: sequenced_broadcast: 103|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:52:58.478106Z INFO Node{id=28}: sequenced_broadcast: 103|SequencedBroadcastWorker Stopped id=103 elapsed=16.419380039s iter=2 [INFO] [stdout] 2025-09-01T11:52:58.513892Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:58.524566Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:58.529388Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:58.529417Z ERROR Node{id=27}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:58.892251Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:58.892307Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:52:58.893721Z INFO Node{id=17}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:58.893807Z INFO Node{id=17}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:58.895089Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:52:58.899699Z ERROR Node{id=16}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:52:58.900831Z WARN Node{id=16}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:52:58.900851Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:58.902137Z INFO Node{id=16}:connect{target=17}:WritePeer{remote=17}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:58.903629Z ERROR ReadClient{remote=16}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:58.907104Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727578893) } [INFO] [stdout] 2025-09-01T11:52:58.911730Z INFO Node{id=17}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:52:58.911784Z INFO Node{id=17}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:58.919116Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:58.931516Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:52:58.931590Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=31 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:52:58.934075Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:58.936535Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:52:58.937468Z ERROR Node{id=17}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:52:58.938409Z INFO Node{id=17}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:58.939640Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:59.286012Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:59.292990Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:59.297000Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 14, 25]) [INFO] [stdout] 2025-09-01T11:52:59.308446Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:59.308489Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:52:59.311150Z INFO Node{id=12}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:59.311215Z INFO Node{id=12}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:59.312657Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:52:59.324895Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727579311) } [INFO] [stdout] 2025-09-01T11:52:59.329137Z INFO Node{id=12}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:52:59.329171Z INFO Node{id=12}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:59.335903Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:59.342959Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:52:59.348516Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:59.350129Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:59.352100Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:52:59.352859Z INFO Node{id=21}: sequenced_broadcast: 107|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:52:59.359333Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(14): lag too high for too long (2.003182499s), dropping [INFO] [stdout] 2025-09-01T11:52:59.360639Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:52:59.360699Z INFO Node{id=12}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:52:59.376254Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:59.376312Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:52:59.377824Z INFO Node{id=31}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:59.377893Z INFO Node{id=31}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:59.379496Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:52:59.389329Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:59.389859Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727579377) } [INFO] [stdout] 2025-09-01T11:52:59.389889Z INFO Node{id=5}: sharedstate::worker::sync_manager: Discovered Peer client_id=31 [INFO] [stdout] 2025-09-01T11:52:59.391504Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:52:59.391531Z INFO Node{id=5}: sharedstate::worker::sync_manager: Broadcasted 4 peers to 3 peers [INFO] [stdout] 2025-09-01T11:52:59.394362Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([32]) [INFO] [stdout] 2025-09-01T11:52:59.394523Z INFO Node{id=31}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:52:59.394556Z INFO Node{id=31}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:52:59.396592Z ERROR Node{id=31}:connect{target=5}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:52:59.396984Z INFO Node{id=31}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:52:59.397546Z INFO Node{id=31}:connect{target=5}:WritePeer{remote=5}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:52:59.397946Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(5) [INFO] [stdout] 2025-09-01T11:52:59.398932Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:59.404400Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:59.411007Z ERROR WriteClient{remote=19}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:52:59.475622Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:59.526959Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:52:59.674523Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:59.679159Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([6]) [INFO] [stdout] 2025-09-01T11:52:59.739947Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:52:59.819447Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:52:59.823429Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:52:59.823453Z ERROR Node{id=32}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:52:59.871217Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(8): lag too high for too long (2.005042999s), dropping [INFO] [stdout] 2025-09-01T11:52:59.902854Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:59.902896Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:52:59.904028Z INFO Node{id=16}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:59.904093Z INFO Node{id=16}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:59.905432Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:52:59.939373Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:52:59.939416Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:52:59.940716Z INFO Node{id=17}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:52:59.940802Z INFO Node{id=17}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:52:59.944199Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:52:59.954869Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:52:59.957585Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4279, state_accept_seq: 4278, history: [] }) } [INFO] [stdout] 2025-09-01T11:52:59.958962Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:00.023428Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:00.069565Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:00.074982Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:53:00.077389Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727575396) } [INFO] [stdout] 2025-09-01T11:53:00.081856Z INFO Node{id=22}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=4685 [INFO] [stdout] 2025-09-01T11:53:00.081887Z INFO Node{id=22}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:00.134199Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:00.138505Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 17, 29]) [INFO] [stdout] 2025-09-01T11:53:00.188215Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:00.196877Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([32]) [INFO] [stdout] 2025-09-01T11:53:00.259754Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:00.266783Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 15, 21, 12]) [INFO] [stdout] 2025-09-01T11:53:00.266825Z WARN Node{id=3}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:53:00.268230Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:00.269331Z INFO Node{id=3}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:00.272173Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:00.329982Z ERROR Node{id=20}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:53:00.331551Z WARN Node{id=20}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:00.331572Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:00.333023Z INFO Node{id=20}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:00.334295Z ERROR ReadClient{remote=20}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:00.399311Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:00.399354Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=28 leader=1 [INFO] [stdout] 2025-09-01T11:53:00.400730Z INFO Node{id=31}:connect{target=28}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:00.400817Z INFO Node{id=31}:connect{target=28}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:00.402099Z INFO Node{id=28}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:53:00.412707Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727580400) } [INFO] [stdout] 2025-09-01T11:53:00.416462Z INFO Node{id=31}:connect{target=28}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:53:00.416492Z INFO Node{id=31}:connect{target=28}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:00.422977Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:00.428513Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 28 } [INFO] [stdout] 2025-09-01T11:53:00.434597Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:00.435849Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:00.436649Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 13922, state_accept_seq: 13921, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:00.437910Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:00.439450Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:00.440787Z INFO Node{id=28}: sequenced_broadcast: 118|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:00.442696Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:53:00.442716Z ERROR Node{id=9}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=3 leader=1 local=9 [INFO] [stdout] 2025-09-01T11:53:00.443907Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:00.444917Z INFO Node{id=9}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:00.444988Z INFO Node{id=9}:connect{target=3}:WritePeer{remote=3}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:00.446701Z ERROR ReadClient{remote=9}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:00.447223Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:00.447302Z INFO Node{id=31}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:00.448468Z INFO Node{id=31}: sequenced_broadcast: 119|SequencedBroadcastWorker Started id=119 [INFO] [stdout] 2025-09-01T11:53:00.450038Z INFO Node{id=31}: sequenced_broadcast: 95|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:00.450056Z INFO Node{id=31}: sequenced_broadcast: 95|SequencedBroadcastWorker Stopped id=95 elapsed=36.372887707s iter=5210 [INFO] [stdout] 2025-09-01T11:53:00.533781Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:00.542204Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:00.546501Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:00.546527Z ERROR Node{id=27}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:00.791444Z ERROR Node{id=24}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:53:00.793550Z WARN Node{id=24}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:00.793573Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:00.797091Z INFO Node{id=24}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:00.798404Z ERROR ReadClient{remote=24}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:00.898749Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:00.903134Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 6262, state_accept_seq: 6261, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:00.903428Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:01.276837Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:01.276881Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:53:01.280863Z INFO Node{id=3}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:01.280931Z INFO Node{id=3}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:01.282106Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:01.300255Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727581280) } [INFO] [stdout] 2025-09-01T11:53:01.300293Z INFO Node{id=5}: sharedstate::worker::sync_manager: Discovered Peer client_id=3 [INFO] [stdout] 2025-09-01T11:53:01.308217Z INFO Node{id=3}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=28 [INFO] [stdout] 2025-09-01T11:53:01.308268Z INFO Node{id=3}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:01.318957Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:01.335814Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:01.335857Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:53:01.336607Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:01.337130Z INFO Node{id=20}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:01.337203Z INFO Node{id=20}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:01.339987Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:53:01.352252Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 5 } [INFO] [stdout] 2025-09-01T11:53:01.352282Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[5] peer=5 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:01.355695Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:01.357328Z INFO Node{id=3}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:01.357409Z INFO Node{id=3}:connect{target=5}:WritePeer{remote=5}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:01.358164Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727581337) } [INFO] [stdout] 2025-09-01T11:53:01.360914Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:01.364772Z INFO Node{id=20}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=27 [INFO] [stdout] 2025-09-01T11:53:01.364804Z INFO Node{id=20}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:01.377889Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:01.386650Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:53:01.386689Z ERROR Node{id=20}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=13 leader=1 local=20 [INFO] [stdout] 2025-09-01T11:53:01.388372Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:01.388433Z INFO Node{id=20}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:01.389847Z INFO Node{id=20}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:01.390148Z ERROR ReadClient{remote=20}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:01.393320Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:01.396544Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:01.426873Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:01.430591Z INFO Node{id=32}:connect{target=22}:ReadPeer{remote=22}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:01.447351Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727579904) } [INFO] [stdout] 2025-09-01T11:53:01.447637Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:01.447668Z INFO Node{id=9}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:53:01.449832Z INFO Node{id=9}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:01.449911Z INFO Node{id=9}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:01.451860Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=9 [INFO] [stdout] 2025-09-01T11:53:01.453310Z INFO Node{id=16}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=1549 [INFO] [stdout] 2025-09-01T11:53:01.453344Z INFO Node{id=16}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:01.479282Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:01.581977Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:01.588243Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:53:01.682018Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:01.735202Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727577132) } [INFO] [stdout] 2025-09-01T11:53:01.740337Z INFO Node{id=4}:connect{target=19}: sharedstate::worker::sync_manager: got pong from peer latency=4608 [INFO] [stdout] 2025-09-01T11:53:01.740374Z INFO Node{id=4}:connect{target=19}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:01.806079Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:01.806153Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:53:01.809129Z INFO Node{id=24}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:01.809218Z INFO Node{id=24}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:01.815896Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:53:02.042795Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:02.047494Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19]) [INFO] [stdout] 2025-09-01T11:53:02.095244Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:02.198527Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:02.356794Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:02.356841Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:53:02.358281Z INFO Node{id=3}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:02.358357Z INFO Node{id=3}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:02.359830Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:02.367977Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727579940) } [INFO] [stdout] 2025-09-01T11:53:02.370601Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727582358) } [INFO] [stdout] 2025-09-01T11:53:02.372193Z INFO Node{id=17}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=2432 [INFO] [stdout] 2025-09-01T11:53:02.372222Z INFO Node{id=17}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:02.375185Z INFO Node{id=3}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:53:02.375212Z INFO Node{id=3}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:02.382541Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:02.390818Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:02.390861Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:53:02.392364Z INFO Node{id=20}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:02.392433Z INFO Node{id=20}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:02.393949Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:53:02.394145Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:02.407626Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:53:02.407663Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=18 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:02.409684Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:02.411517Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:02.412433Z INFO Node{id=3}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:02.412507Z INFO Node{id=3}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:02.414279Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:02.437523Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:02.437566Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:53:02.438638Z INFO Node{id=32}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:02.438702Z INFO Node{id=32}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:02.439922Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:53:02.462464Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:02.463567Z INFO Node{id=27}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:02.463634Z INFO Node{id=27}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:02.465057Z ERROR ReadClient{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:02.505076Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:02.653949Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:02.755519Z ERROR Node{id=22}:connect{target=12}:WritePeer{remote=12}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:53:02.912936Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:03.008754Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:03.101429Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:03.104790Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14]) [INFO] [stdout] 2025-09-01T11:53:03.141039Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:03.149682Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 15]) [INFO] [stdout] 2025-09-01T11:53:03.380864Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:03.384008Z INFO Node{id=10}: sequenced_broadcast: 87|Subscriber(3): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2051 [INFO] [stdout] 2025-09-01T11:53:03.384034Z INFO Node{id=10}: sequenced_broadcast: 1|Subscriber(87): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:03.392064Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:03.392231Z INFO Node{id=8}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:03.394625Z INFO Node{id=8}: sequenced_broadcast: 120|SequencedBroadcastWorker Started id=120 [INFO] [stdout] 2025-09-01T11:53:03.396943Z INFO Node{id=8}: sequenced_broadcast: 94|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:03.396963Z INFO Node{id=8}: sequenced_broadcast: 94|SequencedBroadcastWorker Stopped id=94 elapsed=39.993362055s iter=1320 [INFO] [stdout] 2025-09-01T11:53:03.412243Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:03.412297Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:53:03.413951Z INFO Node{id=3}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:03.414020Z INFO Node{id=3}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:03.415482Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:03.469709Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:03.469776Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:53:03.471306Z INFO Node{id=27}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:03.471374Z INFO Node{id=27}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:03.473235Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:53:03.522789Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:03.527148Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeRecovery(RecoverableStateDetails { id: 8084228702317741667, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:03.556022Z ERROR WriteClient{remote=7}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:53:04.228332Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:04.233675Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:53:04.393512Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:53:04.393560Z INFO Node{id=5}: sharedstate::worker::sync_manager: Broadcasted 5 peers to 4 peers [INFO] [stdout] 2025-09-01T11:53:04.451242Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:04.458955Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:53:04.459010Z ERROR Node{id=22}: sharedstate::worker::sync_manager: failed to send recovery request to peer [INFO] [stdout] 2025-09-01T11:53:04.460335Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:04.461407Z INFO Node{id=22}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:04.671316Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:04.678737Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:53:04.678789Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=3 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:04.680174Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:04.681741Z INFO Node{id=17}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:04.681834Z INFO Node{id=17}:connect{target=3}:WritePeer{remote=3}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:04.683201Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:04.738364Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: Ping(1756727581449) } [INFO] [stdout] 2025-09-01T11:53:04.742381Z INFO Node{id=9}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=3293 [INFO] [stdout] 2025-09-01T11:53:04.742413Z INFO Node{id=9}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:04.920963Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:04.922375Z INFO Node{id=6}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:04.925138Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(13): lag started thresh(8192) < lag(8194) [INFO] [stdout] 2025-09-01T11:53:04.984464Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727581809) } [INFO] [stdout] 2025-09-01T11:53:04.988877Z INFO Node{id=24}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=3179 [INFO] [stdout] 2025-09-01T11:53:04.988914Z INFO Node{id=24}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:04.990078Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:05.130667Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:05.133544Z INFO Node{id=12}: sequenced_broadcast: 98|Subscriber(3): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2462 [INFO] [stdout] 2025-09-01T11:53:05.133570Z INFO Node{id=12}: sequenced_broadcast: 2|Subscriber(98): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:05.281298Z ERROR Node{id=2}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:53:05.281466Z WARN Node{id=2}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:05.281484Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:05.312632Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:05.342416Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:05.431512Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:05.435098Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14]) [INFO] [stdout] 2025-09-01T11:53:05.438592Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727582392) } [INFO] [stdout] 2025-09-01T11:53:05.442001Z INFO Node{id=20}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=3049 [INFO] [stdout] 2025-09-01T11:53:05.442030Z INFO Node{id=20}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:05.462249Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:05.462299Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:53:05.463625Z INFO Node{id=22}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:05.463685Z INFO Node{id=22}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:05.464821Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:53:05.507921Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727582438) } [INFO] [stdout] 2025-09-01T11:53:05.511637Z INFO Node{id=32}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=3073 [INFO] [stdout] 2025-09-01T11:53:05.511678Z INFO Node{id=32}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:05.576566Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:05.580754Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:05.580920Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4391, state_accept_seq: 4390, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:05.657010Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:05.658426Z INFO Node{id=11}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:05.659053Z ERROR ReadClient{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:05.677429Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:05.681596Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:05.681631Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=4 leader=1 [INFO] [stdout] 2025-09-01T11:53:05.681997Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 19 } [INFO] [stdout] 2025-09-01T11:53:05.682867Z INFO Node{id=17}:connect{target=4}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:05.682931Z INFO Node{id=17}:connect{target=4}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:05.686470Z INFO Node{id=4}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:05.695882Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727585682) } [INFO] [stdout] 2025-09-01T11:53:05.699174Z INFO Node{id=17}:connect{target=4}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:53:05.699202Z INFO Node{id=17}:connect{target=4}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:05.706248Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:05.714817Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 4 } [INFO] [stdout] 2025-09-01T11:53:05.714848Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=4 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:05.716036Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:05.716546Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:05.717893Z INFO Node{id=17}:connect{target=4}:ReadPeer{remote=4}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:05.717970Z INFO Node{id=17}:connect{target=4}:WritePeer{remote=4}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:05.719063Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:05.929250Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:05.929306Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:53:05.934849Z INFO Node{id=6}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:05.934929Z INFO Node{id=6}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:05.936181Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:53:05.945631Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727585934) } [INFO] [stdout] 2025-09-01T11:53:05.952201Z INFO Node{id=6}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:53:05.952252Z INFO Node{id=6}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:05.960054Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:05.972441Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:53:05.972479Z ERROR Node{id=6}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=22 leader=1 local=6 [INFO] [stdout] 2025-09-01T11:53:05.972966Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:05.974002Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:05.975235Z ERROR Node{id=6}:connect{target=22}:ReadPeer{remote=22}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:53:05.975351Z INFO Node{id=6}:connect{target=22}:WritePeer{remote=22}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:05.978784Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:06.014408Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:06.018235Z INFO Node{id=5}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:06.020918Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:06.162093Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:06.194122Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:06.204952Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:53:06.278353Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:06.290408Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:06.290457Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:53:06.291384Z INFO Node{id=2}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:06.291449Z INFO Node{id=2}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:06.292690Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:53:06.324171Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:06.331739Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:06.333748Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:06.406960Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:06.411098Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeRecovery(RecoverableStateDetails { id: 8084228702317741667, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:06.411304Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:06.459412Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:06.461720Z INFO Node{id=29}: sequenced_broadcast: 89|Subscriber(3): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2051 [INFO] [stdout] 2025-09-01T11:53:06.465672Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:06.470897Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:06.470925Z ERROR Node{id=29}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:06.471084Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:06.471170Z INFO Node{id=20}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:06.472513Z INFO Node{id=20}: sequenced_broadcast: 121|SequencedBroadcastWorker Started id=121 [INFO] [stdout] 2025-09-01T11:53:06.474529Z INFO Node{id=20}: sequenced_broadcast: 34|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:06.474550Z INFO Node{id=20}: sequenced_broadcast: 34|SequencedBroadcastWorker Stopped id=34 elapsed=56.156237119s iter=2 [INFO] [stdout] 2025-09-01T11:53:06.526098Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:06.529983Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:06.530011Z ERROR Node{id=15}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:06.597234Z ERROR Node{id=27}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:53:06.598363Z ERROR Node{id=27}:connect{target=2}: sharedstate::worker::sync_manager: channel closed waiting for pong [INFO] [stdout] 2025-09-01T11:53:06.599524Z INFO Node{id=27}:connect{target=2}:WritePeer{remote=2}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:06.599606Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(2) [INFO] [stdout] 2025-09-01T11:53:06.600840Z ERROR ReadClient{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:06.658663Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:06.666217Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:06.666260Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:53:06.667488Z INFO Node{id=11}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:06.667557Z INFO Node{id=11}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:06.668368Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:53:06.680898Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727586667) } [INFO] [stdout] 2025-09-01T11:53:06.684077Z INFO Node{id=11}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:53:06.684109Z INFO Node{id=11}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:06.692164Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:06.700203Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:53:06.706019Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:06.707173Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4636, state_accept_seq: 4635, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:06.708661Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:06.710327Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:06.713209Z INFO Node{id=26}: sequenced_broadcast: 115|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:06.719714Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:06.721445Z INFO Node{id=11}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:06.721986Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:06.722019Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:53:06.722619Z INFO Node{id=11}: sequenced_broadcast: 122|SequencedBroadcastWorker Started id=122 [INFO] [stdout] 2025-09-01T11:53:06.723674Z INFO Node{id=17}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:06.723745Z INFO Node{id=17}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:06.724738Z INFO Node{id=11}: sequenced_broadcast: 111|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:06.724777Z INFO Node{id=11}: sequenced_broadcast: 111|SequencedBroadcastWorker Stopped id=111 elapsed=20.958735181s iter=2 [INFO] [stdout] 2025-09-01T11:53:06.724895Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:06.738170Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727586723) } [INFO] [stdout] 2025-09-01T11:53:06.743662Z INFO Node{id=17}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:53:06.743707Z INFO Node{id=17}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:06.759611Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:06.778907Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:06.811208Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:53:06.811244Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[15] peer=15 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:06.814500Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:06.815993Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:06.817257Z INFO Node{id=17}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:06.817356Z INFO Node{id=17}:connect{target=15}:WritePeer{remote=15}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:06.818869Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:06.928957Z ERROR WriteClient{remote=13}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:53:06.929985Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(13): lag too high for too long (2.004846698s), dropping [INFO] [stdout] 2025-09-01T11:53:06.975570Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:06.975618Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:53:06.976855Z INFO Node{id=6}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:06.976918Z INFO Node{id=6}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:06.978268Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:53:07.026463Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:07.026507Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=4 peers=5 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:53:07.027831Z INFO Node{id=5}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:07.027898Z INFO Node{id=5}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:07.029347Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:53:07.040468Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727587027) } [INFO] [stdout] 2025-09-01T11:53:07.044916Z INFO Node{id=5}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:53:07.044950Z INFO Node{id=5}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:07.052043Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:07.062195Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:53:07.068561Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:07.070848Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4636, state_accept_seq: 4635, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:07.073038Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:07.073123Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:07.076107Z INFO Node{id=31}: sequenced_broadcast: 119|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:07.082749Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:07.082872Z INFO Node{id=5}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:07.084269Z INFO Node{id=5}: sequenced_broadcast: 123|SequencedBroadcastWorker Started id=123 [INFO] [stdout] 2025-09-01T11:53:07.086230Z INFO Node{id=5}: sequenced_broadcast: 112|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:07.086251Z INFO Node{id=5}: sequenced_broadcast: 112|SequencedBroadcastWorker Stopped id=112 elapsed=21.0485731s iter=2 [INFO] [stdout] 2025-09-01T11:53:07.231382Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727583414) } [INFO] [stdout] 2025-09-01T11:53:07.234752Z INFO Node{id=3}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=3820 [INFO] [stdout] 2025-09-01T11:53:07.234818Z INFO Node{id=3}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:07.291160Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727583471) } [INFO] [stdout] 2025-09-01T11:53:07.401322Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:07.603303Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:07.603352Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:53:07.604796Z INFO Node{id=27}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:07.604861Z INFO Node{id=27}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:07.606152Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:53:07.687302Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:07.817471Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:07.817517Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:53:07.818602Z INFO Node{id=17}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:07.818672Z INFO Node{id=17}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:07.819885Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:07.830327Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727587818) } [INFO] [stdout] 2025-09-01T11:53:07.833687Z INFO Node{id=17}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:53:07.833720Z INFO Node{id=17}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:07.841198Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:07.850943Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:53:07.850981Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 17, 29, 30] peer=30 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:07.852073Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:07.852368Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:07.853671Z ERROR Node{id=17}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:53:07.853751Z INFO Node{id=17}:connect{target=30}:WritePeer{remote=30}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:07.855421Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:07.906857Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:07.909737Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:07.915345Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4463, state_accept_seq: 4462, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:07.917952Z ERROR Node{id=16}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:07.918910Z WARN Node{id=16}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:07.918927Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:07.919993Z INFO Node{id=16}:connect{target=2}:WritePeer{remote=2}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:07.969272Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727586976) } [INFO] [stdout] 2025-09-01T11:53:07.973924Z INFO Node{id=6}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=997 [INFO] [stdout] 2025-09-01T11:53:07.973959Z INFO Node{id=6}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:08.334468Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:08.339664Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:53:08.433372Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:08.474294Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:08.531968Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:08.623078Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:08.628271Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:53:08.743108Z ERROR Node{id=19}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:53:08.745080Z WARN Node{id=19}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:08.745101Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:08.757606Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:08.853418Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:08.853467Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=28 leader=1 [INFO] [stdout] 2025-09-01T11:53:08.854599Z INFO Node{id=17}:connect{target=28}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:08.854672Z INFO Node{id=17}:connect{target=28}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:08.855501Z INFO Node{id=28}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:08.864121Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727588854) } [INFO] [stdout] 2025-09-01T11:53:08.867455Z INFO Node{id=17}:connect{target=28}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:53:08.867488Z INFO Node{id=17}:connect{target=28}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:08.872414Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:08.875913Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 28 } [INFO] [stdout] 2025-09-01T11:53:08.883327Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:08.884906Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:08.888087Z INFO Node{id=28}: sequenced_broadcast: 118|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:08.889045Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:08.892231Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:08.892302Z INFO Node{id=17}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:08.907339Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:08.911286Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:53:08.919968Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:08.920014Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:53:08.921385Z INFO Node{id=16}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:08.921456Z INFO Node{id=16}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:08.922496Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:53:08.931540Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727588921) } [INFO] [stdout] 2025-09-01T11:53:08.934623Z INFO Node{id=16}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:53:08.934656Z INFO Node{id=16}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:08.940558Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:08.945170Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:53:08.949874Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:08.951446Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4463, state_accept_seq: 4462, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:08.952769Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:08.953071Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:08.953892Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:08.956506Z INFO Node{id=25}: sequenced_broadcast: 114|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:08.964035Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:08.964137Z INFO Node{id=16}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:08.965276Z INFO Node{id=16}: sequenced_broadcast: 124|SequencedBroadcastWorker Started id=124 [INFO] [stdout] 2025-09-01T11:53:08.967004Z INFO Node{id=16}: sequenced_broadcast: 86|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:08.967022Z INFO Node{id=16}: sequenced_broadcast: 86|SequencedBroadcastWorker Stopped id=86 elapsed=52.851456373s iter=2021 [INFO] [stdout] 2025-09-01T11:53:09.089424Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727585463) } [INFO] [stdout] 2025-09-01T11:53:09.093479Z INFO Node{id=22}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=3630 [INFO] [stdout] 2025-09-01T11:53:09.093517Z INFO Node{id=22}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:09.095891Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:09.103316Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:53:09.203022Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:09.208466Z INFO Node{id=14}: sequenced_broadcast: 108|Subscriber(3): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:09.215159Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:09.215296Z INFO Node{id=18}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:09.216192Z INFO Node{id=18}: sequenced_broadcast: 125|SequencedBroadcastWorker Started id=125 [INFO] [stdout] 2025-09-01T11:53:09.218700Z INFO Node{id=18}: sequenced_broadcast: 88|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:09.218720Z INFO Node{id=18}: sequenced_broadcast: 88|SequencedBroadcastWorker Stopped id=88 elapsed=52.557165041s iter=1996 [INFO] [stdout] 2025-09-01T11:53:09.300036Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:09.317103Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:09.321926Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:09.322145Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:09.344060Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:09.350897Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:09.357560Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 15, 21]) [INFO] [stdout] 2025-09-01T11:53:09.379831Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:09.383000Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:09.522034Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:09.524781Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17]) [INFO] [stdout] 2025-09-01T11:53:09.750148Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:09.750195Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:53:09.751639Z INFO Node{id=19}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:09.751714Z INFO Node{id=19}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:09.754310Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:09.873381Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727586291) } [INFO] [stdout] 2025-09-01T11:53:09.880064Z INFO Node{id=2}:connect{target=19}: sharedstate::worker::sync_manager: got pong from peer latency=3589 [INFO] [stdout] 2025-09-01T11:53:09.880105Z INFO Node{id=2}:connect{target=19}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:09.915117Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:09.920145Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4636, state_accept_seq: 4635, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:09.921043Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:10.140571Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:10.430083Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:10.544862Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727589751) } [INFO] [stdout] 2025-09-01T11:53:10.549534Z INFO Node{id=19}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=798 [INFO] [stdout] 2025-09-01T11:53:10.549573Z INFO Node{id=19}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:10.744192Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:10.750003Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10]) [INFO] [stdout] 2025-09-01T11:53:10.781085Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:10.786940Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:53:10.786975Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14] peer=14 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:10.789338Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:10.789396Z INFO Node{id=3}:connect{target=14}:WritePeer{remote=14}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:10.790933Z INFO Node{id=3}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:10.792392Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:10.876476Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:10.882838Z INFO Node{id=17}: sequenced_broadcast: 85|Subscriber(4): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2051 [INFO] [stdout] 2025-09-01T11:53:10.889626Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:10.889738Z INFO Node{id=6}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:10.890775Z INFO Node{id=6}: sequenced_broadcast: 126|SequencedBroadcastWorker Started id=126 [INFO] [stdout] 2025-09-01T11:53:10.892930Z INFO Node{id=6}: sequenced_broadcast: 110|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:10.892951Z INFO Node{id=6}: sequenced_broadcast: 110|SequencedBroadcastWorker Stopped id=110 elapsed=25.883830383s iter=23 [INFO] [stdout] 2025-09-01T11:53:11.011040Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:11.015169Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29]) [INFO] [stdout] 2025-09-01T11:53:11.073811Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:11.078208Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28]) [INFO] [stdout] 2025-09-01T11:53:11.085804Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727587604) } [INFO] [stdout] 2025-09-01T11:53:11.089826Z INFO Node{id=27}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=3485 [INFO] [stdout] 2025-09-01T11:53:11.089860Z INFO Node{id=27}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:11.372747Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:53:11.372826Z INFO Node{id=1}: sharedstate::worker::sync_manager: Broadcasted 31 peers to 31 peers [INFO] [stdout] 2025-09-01T11:53:11.487312Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:11.497627Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:53:11.497666Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 29] peer=29 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:11.499403Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:11.500482Z INFO Node{id=19}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:11.500549Z INFO Node{id=19}:connect{target=29}:WritePeer{remote=29}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:11.502100Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:11.590570Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:11.791609Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:11.791657Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=11 leader=1 [INFO] [stdout] 2025-09-01T11:53:11.793442Z INFO Node{id=3}:connect{target=11}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:11.793517Z INFO Node{id=3}:connect{target=11}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:11.795400Z INFO Node{id=11}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:11.806679Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727591793) } [INFO] [stdout] 2025-09-01T11:53:11.812656Z INFO Node{id=3}:connect{target=11}: sharedstate::worker::sync_manager: got pong from peer latency=19 [INFO] [stdout] 2025-09-01T11:53:11.812696Z INFO Node{id=3}:connect{target=11}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:11.820620Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:11.828833Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 11 } [INFO] [stdout] 2025-09-01T11:53:11.828870Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 26, 11] peer=11 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:11.830643Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:11.830692Z INFO Node{id=3}:connect{target=11}:WritePeer{remote=11}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:11.831910Z INFO Node{id=3}:connect{target=11}:ReadPeer{remote=11}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:11.833140Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:11.833727Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:11.862134Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:11.863864Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4391, state_accept_seq: 4390, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:11.866513Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:12.048984Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:12.060974Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:12.066881Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:53:12.072050Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:12.073447Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4636, state_accept_seq: 4635, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:12.074541Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:12.075831Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:12.077659Z INFO Node{id=6}: sequenced_broadcast: 126|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:12.084876Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:12.084985Z INFO Node{id=22}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:12.086185Z INFO Node{id=22}: sequenced_broadcast: 127|SequencedBroadcastWorker Started id=127 [INFO] [stdout] 2025-09-01T11:53:12.087984Z INFO Node{id=22}: sequenced_broadcast: 90|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:12.088005Z INFO Node{id=22}: sequenced_broadcast: 90|SequencedBroadcastWorker Stopped id=90 elapsed=51.71021634s iter=2086 [INFO] [stdout] 2025-09-01T11:53:12.089343Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:53:12.155936Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:12.159002Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:12.160429Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:12.162338Z INFO Node{id=10}: sequenced_broadcast: 87|Subscriber(4): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2051 [INFO] [stdout] 2025-09-01T11:53:12.171717Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:12.171819Z INFO Node{id=24}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:12.501516Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:12.501561Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:53:12.502959Z INFO Node{id=19}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:12.503081Z INFO Node{id=19}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:12.504836Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:12.515613Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727592503) } [INFO] [stdout] 2025-09-01T11:53:12.519838Z INFO Node{id=19}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:53:12.519895Z INFO Node{id=19}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:12.526948Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:12.541108Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:12.545129Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:53:12.545159Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 15, 21, 12] peer=12 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:12.547596Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:12.549485Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:12.552437Z INFO Node{id=19}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:12.552522Z INFO Node{id=19}:connect{target=12}:WritePeer{remote=12}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:12.554227Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:12.655118Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:12.657024Z INFO Node{id=15}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:12.773298Z ERROR ReadClient{remote=15}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:12.795844Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:12.799999Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4636, state_accept_seq: 4635, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:12.800179Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:12.820160Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:12.832535Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:12.832582Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:53:12.833812Z INFO Node{id=3}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:12.833876Z INFO Node{id=3}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:12.836311Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:12.848797Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727592833) } [INFO] [stdout] 2025-09-01T11:53:12.853642Z INFO Node{id=3}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:53:12.853680Z INFO Node{id=3}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:12.861217Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:12.867130Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:53:12.867160Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 8] peer=8 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:12.868349Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:12.868398Z INFO Node{id=3}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:12.869666Z INFO Node{id=3}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:12.869931Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:12.872877Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:12.888055Z ERROR Node{id=7}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:53:12.889606Z WARN Node{id=7}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:12.889626Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:13.042748Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:13.042804Z ERROR Node{id=19}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:53:13.047497Z ERROR Node{id=4}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:13.049196Z WARN Node{id=4}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:13.049214Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:13.050433Z INFO Node{id=4}:connect{target=19}:WritePeer{remote=19}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:13.106872Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:13.151549Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:13.160919Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:13.165663Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:13.165696Z ERROR Node{id=21}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:13.180577Z ERROR Node{id=6}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:53:13.182684Z WARN Node{id=6}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:13.182704Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:13.230741Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:13.551730Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:13.551802Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:53:13.552822Z INFO Node{id=19}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:13.552886Z INFO Node{id=19}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:13.554255Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:13.564272Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727593552) } [INFO] [stdout] 2025-09-01T11:53:13.564312Z INFO Node{id=5}: sharedstate::worker::sync_manager: Discovered Peer client_id=19 [INFO] [stdout] 2025-09-01T11:53:13.566381Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:53:13.566412Z INFO Node{id=5}: sharedstate::worker::sync_manager: Broadcasted 6 peers to 5 peers [INFO] [stdout] 2025-09-01T11:53:13.568683Z INFO Node{id=19}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:53:13.568713Z INFO Node{id=19}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:13.571209Z ERROR Node{id=19}:connect{target=5}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:53:13.571711Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(5) [INFO] [stdout] 2025-09-01T11:53:13.571813Z INFO Node{id=19}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:13.572367Z INFO Node{id=19}:connect{target=5}:WritePeer{remote=5}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:13.573859Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:13.576253Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:13.638375Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:13.653108Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 19 } [INFO] [stdout] 2025-09-01T11:53:13.653145Z ERROR Node{id=2}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=19 leader=1 local=2 [INFO] [stdout] 2025-09-01T11:53:13.655239Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:13.656474Z INFO Node{id=2}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:13.656553Z INFO Node{id=2}:connect{target=19}:WritePeer{remote=19}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:13.660476Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:13.662239Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:13.662273Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:53:13.663311Z INFO Node{id=15}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:13.663381Z INFO Node{id=15}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:13.664316Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:53:13.675426Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727593663) } [INFO] [stdout] 2025-09-01T11:53:13.681615Z INFO Node{id=15}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:53:13.681657Z INFO Node{id=15}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:13.689012Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:13.695673Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:53:13.695702Z ERROR Node{id=15}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=32 leader=1 local=15 [INFO] [stdout] 2025-09-01T11:53:13.698967Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:13.700410Z INFO Node{id=15}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:13.700484Z INFO Node{id=15}:connect{target=32}:WritePeer{remote=32}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:13.701424Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:13.702225Z ERROR ReadClient{remote=15}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:13.870392Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:13.870440Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:53:13.872205Z INFO Node{id=3}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:13.872285Z INFO Node{id=3}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:13.873901Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:13.897968Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:13.898012Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:53:13.899262Z INFO Node{id=7}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:13.899339Z INFO Node{id=7}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:13.900451Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:53:14.051394Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:14.051437Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:53:14.052853Z INFO Node{id=4}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:14.052923Z INFO Node{id=4}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:14.054341Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:53:14.081211Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:14.084167Z INFO Node{id=10}: sequenced_broadcast: 87|Subscriber(5): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2051 [INFO] [stdout] 2025-09-01T11:53:14.091229Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:14.091349Z INFO Node{id=9}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:14.092736Z INFO Node{id=9}: sequenced_broadcast: 128|SequencedBroadcastWorker Started id=128 [INFO] [stdout] 2025-09-01T11:53:14.094599Z INFO Node{id=9}: sequenced_broadcast: 93|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:14.094619Z INFO Node{id=9}: sequenced_broadcast: 93|SequencedBroadcastWorker Stopped id=93 elapsed=50.779457598s iter=1556 [INFO] [stdout] 2025-09-01T11:53:14.190881Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:14.190925Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:53:14.191886Z INFO Node{id=6}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:14.191952Z INFO Node{id=6}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:14.195485Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:53:14.207479Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727594191) } [INFO] [stdout] 2025-09-01T11:53:14.214037Z INFO Node{id=6}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=23 [INFO] [stdout] 2025-09-01T11:53:14.214079Z INFO Node{id=6}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:14.226175Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:14.241020Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:14.246888Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:53:14.246916Z ERROR Node{id=6}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=13 leader=1 local=6 [INFO] [stdout] 2025-09-01T11:53:14.248181Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:14.250829Z INFO Node{id=6}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:14.250904Z INFO Node{id=6}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:14.252620Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:14.490650Z ERROR Node{id=5}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:53:14.492231Z WARN Node{id=5}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:14.492250Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:14.493876Z INFO Node{id=5}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:14.495116Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:14.574178Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:14.574223Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:53:14.575629Z INFO Node{id=19}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:14.575697Z INFO Node{id=19}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:14.577163Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:14.586892Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727594575) } [INFO] [stdout] 2025-09-01T11:53:14.591215Z INFO Node{id=19}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:53:14.591248Z INFO Node{id=19}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:14.597798Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:14.607234Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:14.608811Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:53:14.608836Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 9] peer=9 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:14.610481Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:14.611496Z INFO Node{id=19}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:14.611577Z INFO Node{id=19}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:14.613059Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:14.651350Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:14.656936Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:14.656975Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:53:14.658009Z INFO Node{id=2}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:14.658073Z INFO Node{id=2}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:14.659276Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:53:14.669456Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727594658) } [INFO] [stdout] 2025-09-01T11:53:14.673809Z INFO Node{id=2}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:53:14.673841Z INFO Node{id=2}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:14.681002Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:14.692478Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:14.700489Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:14.700532Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:53:14.702074Z INFO Node{id=15}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:14.702145Z INFO Node{id=15}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:14.703552Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:53:14.706431Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:53:14.706453Z ERROR Node{id=2}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 15, 21, 12] peer=12 leader=1 local=2 [INFO] [stdout] 2025-09-01T11:53:14.711474Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:14.713343Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:14.713557Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727594702) } [INFO] [stdout] 2025-09-01T11:53:14.719355Z INFO Node{id=2}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:14.719445Z INFO Node{id=2}:connect{target=12}:WritePeer{remote=12}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:14.721379Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:14.722503Z INFO Node{id=15}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:53:14.722533Z INFO Node{id=15}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:14.729629Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:14.738250Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:53:14.740986Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:14.744407Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:14.744748Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:14.748252Z INFO Node{id=22}: sequenced_broadcast: 127|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:14.758081Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:14.758157Z INFO Node{id=15}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:14.903673Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:14.911454Z INFO Node{id=10}: sequenced_broadcast: 87|Subscriber(6): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2051 [INFO] [stdout] 2025-09-01T11:53:14.918594Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:14.918705Z INFO Node{id=32}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:14.920211Z INFO Node{id=32}: sequenced_broadcast: 129|SequencedBroadcastWorker Started id=129 [INFO] [stdout] 2025-09-01T11:53:14.921740Z INFO Node{id=32}: sequenced_broadcast: 101|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:14.921778Z INFO Node{id=32}: sequenced_broadcast: 101|SequencedBroadcastWorker Stopped id=101 elapsed=33.478507398s iter=322 [INFO] [stdout] 2025-09-01T11:53:14.972637Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:14.978237Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:53:15.038173Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727593899) } [INFO] [stdout] 2025-09-01T11:53:15.043327Z INFO Node{id=7}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=1144 [INFO] [stdout] 2025-09-01T11:53:15.043365Z INFO Node{id=7}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:15.167473Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:15.173732Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:15.178243Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 6, 22, 15]) [INFO] [stdout] 2025-09-01T11:53:15.207146Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(1): lag started thresh(8192) < lag(8194) [INFO] [stdout] 2025-09-01T11:53:15.250014Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:15.250065Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:53:15.251811Z INFO Node{id=6}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:15.251885Z INFO Node{id=6}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:15.253375Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:53:15.440969Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:15.501086Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:15.501132Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=5 peers=6 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:53:15.502907Z INFO Node{id=5}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:15.502984Z INFO Node{id=5}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:15.506355Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:53:15.612998Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:15.613047Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:53:15.614174Z INFO Node{id=19}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:15.614247Z INFO Node{id=19}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:15.616150Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:15.644241Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727595614) } [INFO] [stdout] 2025-09-01T11:53:15.649878Z INFO Node{id=19}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=35 [INFO] [stdout] 2025-09-01T11:53:15.649918Z INFO Node{id=19}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:15.677312Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:15.688866Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:53:15.688902Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=6 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:15.693749Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:15.695394Z INFO Node{id=19}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:15.695474Z INFO Node{id=19}:connect{target=6}:WritePeer{remote=6}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:15.697477Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:15.714582Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:15.714626Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:53:15.716407Z INFO Node{id=2}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:15.716470Z INFO Node{id=2}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:15.718262Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:53:15.719055Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:15.732267Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727595716) } [INFO] [stdout] 2025-09-01T11:53:15.732309Z INFO Node{id=5}: sharedstate::worker::sync_manager: Discovered Peer client_id=2 [INFO] [stdout] 2025-09-01T11:53:15.737860Z INFO Node{id=2}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=21 [INFO] [stdout] 2025-09-01T11:53:15.737893Z INFO Node{id=2}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:15.746869Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:15.759365Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 5 } [INFO] [stdout] 2025-09-01T11:53:15.759400Z ERROR Node{id=2}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=5 leader=1 local=2 [INFO] [stdout] 2025-09-01T11:53:15.763068Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:15.763591Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:15.765053Z ERROR Node{id=2}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:53:15.765135Z INFO Node{id=2}:connect{target=5}:WritePeer{remote=5}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:15.766910Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:16.022517Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727594052) } [INFO] [stdout] 2025-09-01T11:53:16.028300Z INFO Node{id=4}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=1976 [INFO] [stdout] 2025-09-01T11:53:16.028344Z INFO Node{id=4}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:16.206815Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:16.259700Z ERROR Node{id=13}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:53:16.261093Z WARN Node{id=13}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:16.261112Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:16.262216Z INFO Node{id=13}:connect{target=12}:WritePeer{remote=12}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:16.263691Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:16.568396Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:16.575874Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:53:16.695988Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:16.696031Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:53:16.697072Z INFO Node{id=19}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:16.697135Z INFO Node{id=19}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:16.698724Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:16.702080Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:16.708737Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727596697) } [INFO] [stdout] 2025-09-01T11:53:16.710283Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:16.713107Z INFO Node{id=19}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:53:16.713136Z INFO Node{id=19}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:16.714235Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 26]) [INFO] [stdout] 2025-09-01T11:53:16.721467Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:16.734130Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:16.734885Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:53:16.734908Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=13 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:16.736185Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:16.736799Z INFO Node{id=19}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:16.737476Z INFO Node{id=19}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:16.739051Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:16.765381Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:16.765430Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=4 leader=1 [INFO] [stdout] 2025-09-01T11:53:16.766620Z INFO Node{id=2}:connect{target=4}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:16.766684Z INFO Node{id=2}:connect{target=4}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:16.768224Z INFO Node{id=4}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:53:16.780994Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727596766) } [INFO] [stdout] 2025-09-01T11:53:16.785543Z INFO Node{id=2}:connect{target=4}: sharedstate::worker::sync_manager: got pong from peer latency=19 [INFO] [stdout] 2025-09-01T11:53:16.785582Z INFO Node{id=2}:connect{target=4}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:16.792897Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:16.804211Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 4 } [INFO] [stdout] 2025-09-01T11:53:16.804249Z ERROR Node{id=2}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=4 leader=1 local=2 [INFO] [stdout] 2025-09-01T11:53:16.804594Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:16.806367Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:16.807838Z ERROR Node{id=2}:connect{target=4}:ReadPeer{remote=4}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:53:16.807913Z INFO Node{id=2}:connect{target=4}:WritePeer{remote=4}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:16.809150Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:16.876670Z INFO Node{id=1}: sequenced_broadcast: 10|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:17.209665Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(1): lag too high for too long (2.002518689s), dropping [INFO] [stdout] 2025-09-01T11:53:17.263595Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:17.263643Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:53:17.265028Z INFO Node{id=13}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:17.265091Z INFO Node{id=13}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:17.266475Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:53:17.278317Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727597265) } [INFO] [stdout] 2025-09-01T11:53:17.282989Z INFO Node{id=13}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:53:17.283024Z INFO Node{id=13}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:17.289872Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:17.296240Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:53:17.301401Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:17.305670Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 6262, state_accept_seq: 6261, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:17.307126Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:17.307564Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:17.310035Z INFO Node{id=24}: sequenced_broadcast: 113|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:17.317216Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:17.317329Z INFO Node{id=13}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:17.318631Z INFO Node{id=13}: sequenced_broadcast: 130|SequencedBroadcastWorker Started id=130 [INFO] [stdout] 2025-09-01T11:53:17.320310Z INFO Node{id=13}: sequenced_broadcast: 67|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:17.320330Z INFO Node{id=13}: sequenced_broadcast: 67|SequencedBroadcastWorker Stopped id=67 elapsed=66.953901092s iter=2641 [INFO] [stdout] 2025-09-01T11:53:17.627141Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:17.738052Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:17.738097Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:53:17.740318Z INFO Node{id=19}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:17.740460Z INFO Node{id=19}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:17.741735Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:17.758884Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727597740) } [INFO] [stdout] 2025-09-01T11:53:17.767986Z INFO Node{id=19}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=27 [INFO] [stdout] 2025-09-01T11:53:17.768030Z INFO Node{id=19}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:17.773602Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:17.779923Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:17.786519Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:17.786560Z ERROR Node{id=28}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:17.793876Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:53:17.793901Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 6, 22, 15, 21] peer=21 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:17.796309Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:17.796595Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:17.797803Z INFO Node{id=19}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:17.797880Z INFO Node{id=19}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:17.799383Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:17.810394Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:17.810431Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:53:17.811923Z INFO Node{id=2}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:17.811986Z INFO Node{id=2}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:17.813354Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:53:17.824496Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727597811) } [INFO] [stdout] 2025-09-01T11:53:17.828422Z INFO Node{id=2}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:53:17.828453Z INFO Node{id=2}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:17.834791Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:17.841537Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:53:17.845775Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:17.847411Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:17.851228Z INFO Node{id=20}: sequenced_broadcast: 121|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:17.852103Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:17.857365Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:17.857436Z INFO Node{id=2}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:17.945089Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727593872) } [INFO] [stdout] 2025-09-01T11:53:17.951565Z INFO Node{id=3}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=4079 [INFO] [stdout] 2025-09-01T11:53:17.951602Z INFO Node{id=3}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:18.341570Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:18.432100Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:18.433887Z INFO Node{id=28}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:18.468599Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:18.516066Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:18.568442Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:53:18.568489Z INFO Node{id=5}: sharedstate::worker::sync_manager: Broadcasted 7 peers to 6 peers [INFO] [stdout] 2025-09-01T11:53:18.592574Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:18.597965Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:53:18.602637Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:18.608882Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4636, state_accept_seq: 4635, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:18.609286Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:18.632980Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:18.797725Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:18.797791Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:53:18.799089Z INFO Node{id=19}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:18.799151Z INFO Node{id=19}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:18.801515Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:18.878065Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:18.947741Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:19.046855Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:19.051523Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29]) [INFO] [stdout] 2025-09-01T11:53:19.105541Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:19.276922Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:19.282621Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:19.284431Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4391, state_accept_seq: 4390, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:19.287658Z ERROR Node{id=27}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:19.288966Z WARN Node{id=27}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:19.288987Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:19.290209Z INFO Node{id=27}:connect{target=14}:WritePeer{remote=14}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:19.359492Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:19.371035Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:19.377937Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 6, 22, 15, 21]) [INFO] [stdout] 2025-09-01T11:53:19.384330Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:19.442156Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:19.442207Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:53:19.446166Z INFO Node{id=28}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:19.446249Z INFO Node{id=28}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:19.447937Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:53:19.476080Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727599446) } [INFO] [stdout] 2025-09-01T11:53:19.483726Z INFO Node{id=28}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=37 [INFO] [stdout] 2025-09-01T11:53:19.483779Z INFO Node{id=28}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:19.494465Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:19.507903Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:19.516146Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:53:19.517557Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:19.525513Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:19.526036Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:19.528548Z INFO Node{id=13}: sequenced_broadcast: 130|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:19.532857Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727595251) } [INFO] [stdout] 2025-09-01T11:53:19.536586Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:19.536655Z INFO Node{id=28}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:19.538975Z INFO Node{id=6}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=4287 [INFO] [stdout] 2025-09-01T11:53:19.539004Z INFO Node{id=6}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:19.726579Z INFO Node{id=1}: sequenced_broadcast: 7|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:19.731012Z INFO Node{id=1}: sequenced_broadcast: 5|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:19.735490Z INFO Node{id=1}: sequenced_broadcast: 3|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:19.738958Z INFO Node{id=1}: sequenced_broadcast: 11|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:19.739010Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:19.750826Z INFO Node{id=1}: sequenced_broadcast: 18|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:19.750862Z INFO Node{id=1}: sequenced_broadcast: 6|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:19.754458Z INFO Node{id=1}: sequenced_broadcast: 16|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:19.758497Z INFO Node{id=1}: sequenced_broadcast: 17|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:19.758532Z INFO Node{id=1}: sequenced_broadcast: 19|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:19.816833Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727595502) } [INFO] [stdout] 2025-09-01T11:53:19.821338Z INFO Node{id=5}:connect{target=19}: sharedstate::worker::sync_manager: got pong from peer latency=4319 [INFO] [stdout] 2025-09-01T11:53:19.821370Z INFO Node{id=5}:connect{target=19}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:20.279799Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:20.283340Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14]) [INFO] [stdout] 2025-09-01T11:53:20.290366Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:20.290405Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:53:20.292027Z INFO Node{id=27}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:20.292096Z INFO Node{id=27}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:20.293371Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:53:20.460848Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber(15): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:20.752232Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:20.827221Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:20.830355Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:20.830494Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:20.832189Z INFO Node{id=29}: sequenced_broadcast: 89|Subscriber(4): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 2051 [INFO] [stdout] 2025-09-01T11:53:20.836502Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:20.836575Z INFO Node{id=4}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:20.887276Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:20.889743Z INFO Node{id=25}: sequenced_broadcast: 114|Subscriber(3): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:20.897141Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:20.897246Z INFO Node{id=7}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:20.898750Z INFO Node{id=7}: sequenced_broadcast: 131|SequencedBroadcastWorker Started id=131 [INFO] [stdout] 2025-09-01T11:53:20.901127Z INFO Node{id=7}: sequenced_broadcast: 117|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:20.901148Z INFO Node{id=7}: sequenced_broadcast: 117|SequencedBroadcastWorker Stopped id=117 elapsed=28.744965481s iter=2 [INFO] [stdout] 2025-09-01T11:53:21.017527Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:21.081457Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:21.334790Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:21.337873Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:21.703648Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:21.707094Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:21.766959Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:21.772783Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:53:21.772816Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 17, 29, 20, 2] peer=2 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:21.774443Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:21.774503Z INFO Node{id=3}:connect{target=2}:WritePeer{remote=2}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:21.775465Z INFO Node{id=3}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:21.775732Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:21.906962Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727598799) } [INFO] [stdout] 2025-09-01T11:53:21.911179Z INFO Node{id=19}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=3112 [INFO] [stdout] 2025-09-01T11:53:21.911217Z INFO Node{id=19}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:22.067428Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:22.211269Z ERROR Node{id=6}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:53:22.264312Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:22.267043Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:22.267070Z ERROR Node{id=22}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:22.518928Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:22.521910Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17]) [INFO] [stdout] 2025-09-01T11:53:22.696889Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:22.700105Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 28]) [INFO] [stdout] 2025-09-01T11:53:22.700131Z WARN Node{id=17}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:53:22.701487Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:22.701720Z INFO Node{id=17}:connect{target=28}:ReadPeer{remote=28}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:22.747896Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:22.776082Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:22.776132Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:53:22.777939Z INFO Node{id=3}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:22.778015Z INFO Node{id=3}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:22.779395Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:22.807098Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727602778) } [INFO] [stdout] 2025-09-01T11:53:22.811279Z INFO Node{id=3}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=33 [INFO] [stdout] 2025-09-01T11:53:22.811319Z INFO Node{id=3}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:22.843197Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:22.848897Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:53:22.848926Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=6 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:22.850743Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:22.850823Z INFO Node{id=3}:connect{target=6}:WritePeer{remote=6}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:22.852285Z INFO Node{id=3}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:22.853208Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:22.877469Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:22.962458Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:22.971736Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:53:22.971797Z ERROR Node{id=6}: sharedstate::worker::sync_manager: failed to send recovery request to peer [INFO] [stdout] 2025-09-01T11:53:22.973058Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:22.974591Z INFO Node{id=6}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:23.154873Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727600292) } [INFO] [stdout] 2025-09-01T11:53:23.160532Z INFO Node{id=27}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=2868 [INFO] [stdout] 2025-09-01T11:53:23.160582Z INFO Node{id=27}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:23.221802Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:23.227418Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 19 } [INFO] [stdout] 2025-09-01T11:53:23.227447Z ERROR Node{id=5}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=19 leader=1 local=5 [INFO] [stdout] 2025-09-01T11:53:23.230992Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:23.231046Z INFO Node{id=5}:connect{target=19}:WritePeer{remote=19}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:23.232298Z INFO Node{id=5}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:23.233027Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:23.647255Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:23.651270Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29]) [INFO] [stdout] 2025-09-01T11:53:23.707110Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:23.707156Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:53:23.708480Z INFO Node{id=17}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:23.708562Z INFO Node{id=17}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:23.710031Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:23.721536Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727603708) } [INFO] [stdout] 2025-09-01T11:53:23.726071Z INFO Node{id=17}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:53:23.726127Z INFO Node{id=17}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:23.733782Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:23.739837Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:53:23.739871Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 6, 22, 15, 21, 12] peer=12 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:23.743350Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:23.744937Z INFO Node{id=17}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:23.745009Z INFO Node{id=17}:connect{target=12}:WritePeer{remote=12}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:23.746512Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:23.747912Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:23.821530Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:23.823939Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14]) [INFO] [stdout] 2025-09-01T11:53:23.852199Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:23.852249Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:53:23.853631Z INFO Node{id=3}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:23.853698Z INFO Node{id=3}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:23.855023Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:23.857808Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:23.975460Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:23.975505Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:53:23.976638Z INFO Node{id=6}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:23.976712Z INFO Node{id=6}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:23.978024Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:53:23.987594Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727603976) } [INFO] [stdout] 2025-09-01T11:53:23.991233Z INFO Node{id=6}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:53:23.991267Z INFO Node{id=6}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:23.998178Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:24.007948Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:24.009023Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:53:24.012156Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:24.016346Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:24.020184Z INFO Node{id=16}: sequenced_broadcast: 124|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:24.026118Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:24.026199Z INFO Node{id=6}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:24.232513Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:24.232562Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=6 peers=7 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:53:24.234063Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:24.234137Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:24.235430Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:53:24.246781Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727604234) } [INFO] [stdout] 2025-09-01T11:53:24.251494Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:53:24.251534Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:24.259240Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:24.265175Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:53:24.270727Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:24.271003Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:24.273010Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:24.275574Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:24.275921Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, seq(4099) < min_allowed(25775) [INFO] [stdout] 2025-09-01T11:53:24.278921Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:24.281588Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(27824) < seq(27825) [INFO] [stdout] 2025-09-01T11:53:24.298181Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(21): Added, allow_drop: true, next_sequence: 27825, min_allowed_seq: 25779 [INFO] [stdout] 2025-09-01T11:53:24.304920Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 27825) [INFO] [stdout] 2025-09-01T11:53:24.305042Z INFO Node{id=5}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:24.306557Z INFO Node{id=5}: sequenced_broadcast: 132|SequencedBroadcastWorker Started id=132 [INFO] [stdout] 2025-09-01T11:53:24.309340Z INFO Node{id=5}: sequenced_broadcast: 123|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:24.309364Z INFO Node{id=5}: sequenced_broadcast: 123|SequencedBroadcastWorker Stopped id=123 elapsed=17.225057929s iter=2 [INFO] [stdout] 2025-09-01T11:53:24.369581Z ERROR ReadClient{remote=26}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:53:24.373861Z ERROR Node{id=26}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:24.374456Z WARN Node{id=26}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:24.374474Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:24.450984Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:24.460063Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:24.460324Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 28]) [INFO] [stdout] 2025-09-01T11:53:24.466222Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6]) [INFO] [stdout] 2025-09-01T11:53:24.500889Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:24.505117Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10]) [INFO] [stdout] 2025-09-01T11:53:24.741060Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:24.746225Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:24.746268Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:53:24.749524Z INFO Node{id=17}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:24.749605Z INFO Node{id=17}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:24.750870Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:24.761801Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727604749) } [INFO] [stdout] 2025-09-01T11:53:24.766183Z INFO Node{id=17}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:53:24.766217Z INFO Node{id=17}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:24.773232Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:24.785574Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:53:24.785613Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 7] peer=7 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:24.786172Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:24.787255Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:24.788585Z ERROR Node{id=17}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:53:24.788730Z INFO Node{id=17}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:24.790179Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:24.804818Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:24.814093Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:53:24.814129Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=17 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:24.815601Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:24.816720Z INFO Node{id=19}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:24.816820Z INFO Node{id=19}:connect{target=17}:WritePeer{remote=17}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:24.818276Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:24.899070Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:24.903516Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22]) [INFO] [stdout] 2025-09-01T11:53:25.180077Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:25.189399Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:25.193868Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22, 15]) [INFO] [stdout] 2025-09-01T11:53:25.382792Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:25.382841Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:53:25.384112Z INFO Node{id=26}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:25.384184Z INFO Node{id=26}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:25.385808Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:53:25.396492Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727605384) } [INFO] [stdout] 2025-09-01T11:53:25.401123Z INFO Node{id=26}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:53:25.401156Z INFO Node{id=26}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:25.408547Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:25.413091Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:53:25.420914Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:25.423305Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:25.425562Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:25.426858Z INFO Node{id=18}: sequenced_broadcast: 125|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:25.434477Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:25.434557Z INFO Node{id=26}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:25.504389Z INFO sharedstate::testing::fuzzy_test: unblock all connections and let system recover [INFO] [stdout] 2025-09-01T11:53:25.789999Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:25.790047Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:53:25.791077Z INFO Node{id=17}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:25.791154Z INFO Node{id=17}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:25.792085Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:25.798663Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727605791) } [INFO] [stdout] 2025-09-01T11:53:25.801134Z INFO Node{id=17}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=10 [INFO] [stdout] 2025-09-01T11:53:25.801167Z INFO Node{id=17}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:25.805003Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:25.812396Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:25.812931Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:53:25.812956Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 23] peer=23 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:25.814271Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:25.816037Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:25.817101Z INFO Node{id=17}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:25.817178Z INFO Node{id=17}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:25.817662Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:25.817706Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:53:25.818161Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:25.819126Z INFO Node{id=19}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:25.819195Z INFO Node{id=19}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:25.820102Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:25.834298Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727605819) } [INFO] [stdout] 2025-09-01T11:53:25.836582Z INFO Node{id=19}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:53:25.836611Z INFO Node{id=19}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:25.853747Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:25.865471Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:53:25.865510Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24] peer=24 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:25.866958Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:25.867720Z INFO Node{id=19}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:25.867810Z INFO Node{id=19}:connect{target=24}:WritePeer{remote=24}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:25.868461Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:25.873406Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:26.125897Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:26.128719Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:53:26.128743Z ERROR Node{id=27}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=17 leader=1 local=27 [INFO] [stdout] 2025-09-01T11:53:26.129777Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:26.129829Z INFO Node{id=27}:connect{target=17}:WritePeer{remote=17}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:26.130997Z INFO Node{id=27}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:26.131700Z ERROR ReadClient{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:26.333939Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:26.577884Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:26.717457Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:26.728238Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:26.730537Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 18, 26]) [INFO] [stdout] 2025-09-01T11:53:26.766521Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727603853) } [INFO] [stdout] 2025-09-01T11:53:26.776177Z INFO Node{id=3}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=2923 [INFO] [stdout] 2025-09-01T11:53:26.776222Z INFO Node{id=3}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:26.819027Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:26.819074Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:53:26.820245Z INFO Node{id=17}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:26.820318Z INFO Node{id=17}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:26.821064Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:26.832967Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727606820) } [INFO] [stdout] 2025-09-01T11:53:26.839252Z INFO Node{id=17}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=19 [INFO] [stdout] 2025-09-01T11:53:26.839304Z INFO Node{id=17}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:26.843986Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:26.857586Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:26.862106Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:53:26.862139Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 8] peer=8 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:26.869346Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:26.870114Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:26.870146Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:53:26.870393Z INFO Node{id=17}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:26.870490Z INFO Node{id=17}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:26.871314Z INFO Node{id=19}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:26.871378Z INFO Node{id=19}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:26.872119Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:26.872317Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:26.880221Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727606871) } [INFO] [stdout] 2025-09-01T11:53:26.882433Z INFO Node{id=19}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:53:26.882471Z INFO Node{id=19}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:26.886411Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:26.892839Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:26.910035Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:53:26.910072Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22, 15] peer=15 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:26.911246Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:26.912975Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:26.913956Z INFO Node{id=19}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:26.914037Z INFO Node{id=19}:connect{target=15}:WritePeer{remote=15}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:26.914841Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:27.130816Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:27.130866Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:53:27.131653Z INFO Node{id=27}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:27.131725Z INFO Node{id=27}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:27.132689Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:53:27.143774Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727607131) } [INFO] [stdout] 2025-09-01T11:53:27.146451Z INFO Node{id=27}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:53:27.146485Z INFO Node{id=27}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:27.151391Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:27.155173Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:53:27.155197Z ERROR Node{id=27}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=3 leader=1 local=27 [INFO] [stdout] 2025-09-01T11:53:27.156400Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:27.156459Z INFO Node{id=27}:connect{target=3}:WritePeer{remote=3}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:27.157239Z INFO Node{id=27}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:27.157529Z ERROR ReadClient{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:27.159079Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:27.298228Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:27.409167Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:27.434129Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:27.436812Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24]) [INFO] [stdout] 2025-09-01T11:53:27.690519Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:27.691832Z INFO Node{id=10}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:27.764878Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:27.843240Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:27.847655Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:27.850330Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20]) [INFO] [stdout] 2025-09-01T11:53:27.850357Z WARN Node{id=2}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:53:27.851660Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:27.852077Z INFO Node{id=2}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:27.853183Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:27.870873Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:27.870937Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:53:27.871679Z INFO Node{id=17}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:27.871791Z INFO Node{id=17}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:27.872830Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:27.914049Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:27.914093Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:53:27.914737Z INFO Node{id=19}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:27.914831Z INFO Node{id=19}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:27.915594Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:27.923297Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727607914) } [INFO] [stdout] 2025-09-01T11:53:27.926024Z INFO Node{id=19}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=12 [INFO] [stdout] 2025-09-01T11:53:27.926060Z INFO Node{id=19}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:27.933386Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:27.944883Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:27.950555Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:53:27.950590Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16] peer=16 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:27.952046Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:27.952790Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:27.953213Z INFO Node{id=19}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:27.953285Z INFO Node{id=19}:connect{target=16}:WritePeer{remote=16}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:27.954030Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:28.069402Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:28.144155Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:28.146649Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:28.146676Z ERROR Node{id=24}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:28.157429Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:28.157476Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:53:28.158372Z INFO Node{id=27}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:28.158442Z INFO Node{id=27}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:28.159330Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:53:28.171705Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727608158) } [INFO] [stdout] 2025-09-01T11:53:28.173610Z INFO Node{id=27}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:53:28.173640Z INFO Node{id=27}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:28.179058Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:28.181777Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:53:28.187565Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:28.188011Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4391, state_accept_seq: 4390, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:28.189512Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:28.191275Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:28.193735Z INFO Node{id=6}: sequenced_broadcast: 126|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:28.200196Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:28.200320Z INFO Node{id=27}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:28.201157Z INFO Node{id=27}: sequenced_broadcast: 133|SequencedBroadcastWorker Started id=133 [INFO] [stdout] 2025-09-01T11:53:28.202627Z INFO Node{id=27}: sequenced_broadcast: 104|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:28.202647Z INFO Node{id=27}: sequenced_broadcast: 104|SequencedBroadcastWorker Stopped id=104 elapsed=45.724322868s iter=3 [INFO] [stdout] 2025-09-01T11:53:28.598805Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:28.629837Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:28.631979Z INFO Node{id=24}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:28.696660Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:28.696703Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:53:28.697267Z INFO Node{id=10}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:28.697332Z INFO Node{id=10}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:28.697915Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:53:28.704003Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727608697) } [INFO] [stdout] 2025-09-01T11:53:28.706232Z INFO Node{id=10}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:53:28.706266Z INFO Node{id=10}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:28.710531Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:28.712667Z ERROR ReadClient{remote=24}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:28.713909Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:53:28.720653Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:28.722167Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:28.723668Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:28.723900Z INFO Node{id=27}: sequenced_broadcast: 133|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:28.727233Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:28.727304Z INFO Node{id=10}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:28.734170Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:28.736369Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:28.799618Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:28.803418Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:53:28.803445Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25] peer=25 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:28.805746Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:28.805891Z INFO Node{id=3}:connect{target=25}:WritePeer{remote=25}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:28.806984Z INFO Node{id=3}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:28.808010Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:28.855633Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:28.855684Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:53:28.856637Z INFO Node{id=2}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:28.856710Z INFO Node{id=2}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:28.857681Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:53:28.954188Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:28.954231Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:53:28.954650Z INFO Node{id=19}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:28.954713Z INFO Node{id=19}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:28.955406Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:28.960709Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727608954) } [INFO] [stdout] 2025-09-01T11:53:28.963059Z INFO Node{id=19}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:53:28.963090Z INFO Node{id=19}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:28.967123Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:28.971054Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:53:28.971080Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 7] peer=7 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:28.971609Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:28.972160Z INFO Node{id=19}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:28.972237Z INFO Node{id=19}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:28.972989Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:28.973090Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:28.982917Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:28.984497Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10]) [INFO] [stdout] 2025-09-01T11:53:29.052742Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:29.344508Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727607871) } [INFO] [stdout] 2025-09-01T11:53:29.346603Z INFO Node{id=17}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=1475 [INFO] [stdout] 2025-09-01T11:53:29.346635Z INFO Node{id=17}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:29.378307Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:29.383018Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:29.385143Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22, 15, 21]) [INFO] [stdout] 2025-09-01T11:53:29.385172Z WARN Node{id=12}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:53:29.386449Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:29.387400Z INFO Node{id=12}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:29.388383Z ERROR ReadClient{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:29.517499Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:29.526609Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:29.529194Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13]) [INFO] [stdout] 2025-09-01T11:53:29.635600Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:29.635646Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:53:29.636211Z INFO Node{id=24}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:29.636310Z INFO Node{id=24}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:29.637160Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:53:29.643134Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727609636) } [INFO] [stdout] 2025-09-01T11:53:29.645217Z INFO Node{id=24}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:53:29.645246Z INFO Node{id=24}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:29.649493Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:29.655550Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:29.657512Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:53:29.659398Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:29.661288Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:29.663311Z INFO Node{id=20}: sequenced_broadcast: 121|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:29.663336Z INFO Node{id=20}: sequenced_broadcast: 1|Subscriber(121): channel closed, dropping [INFO] [stdout] 2025-09-01T11:53:29.667099Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:29.667160Z INFO Node{id=24}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:29.724669Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:29.726644Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10]) [INFO] [stdout] 2025-09-01T11:53:29.806578Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:29.806626Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:53:29.807432Z INFO Node{id=3}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:29.807501Z INFO Node{id=3}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:29.808238Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:29.813732Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727609807) } [INFO] [stdout] 2025-09-01T11:53:29.816026Z INFO Node{id=3}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:53:29.816059Z INFO Node{id=3}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:29.819960Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:29.822900Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:53:29.822926Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22, 15] peer=15 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:29.824051Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:29.824130Z INFO Node{id=3}:connect{target=15}:WritePeer{remote=15}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:29.824402Z INFO Node{id=3}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:29.824791Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:29.825670Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:29.844037Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:29.947581Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727608856) } [INFO] [stdout] 2025-09-01T11:53:29.949507Z INFO Node{id=2}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=1093 [INFO] [stdout] 2025-09-01T11:53:29.949539Z INFO Node{id=2}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:29.972657Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:29.972702Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=28 leader=1 [INFO] [stdout] 2025-09-01T11:53:29.973183Z INFO Node{id=19}:connect{target=28}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:29.973258Z INFO Node{id=19}:connect{target=28}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:29.974085Z INFO Node{id=28}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:29.979989Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727609973) } [INFO] [stdout] 2025-09-01T11:53:29.982130Z INFO Node{id=19}:connect{target=28}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:53:29.982159Z INFO Node{id=19}:connect{target=28}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:29.985569Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:29.991313Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:29.993126Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 28 } [INFO] [stdout] 2025-09-01T11:53:29.993149Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 28] peer=28 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:29.993974Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:29.994468Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:29.994948Z INFO Node{id=19}:connect{target=28}:ReadPeer{remote=28}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:29.995010Z INFO Node{id=19}:connect{target=28}:WritePeer{remote=28}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:29.995660Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:30.132702Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:30.135867Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:53:30.135892Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25] peer=25 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:30.139473Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:30.139995Z INFO Node{id=17}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:30.140080Z INFO Node{id=17}:connect{target=25}:WritePeer{remote=25}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:30.140998Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:30.284620Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:30.390658Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:30.390703Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:53:30.391445Z INFO Node{id=12}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:30.391513Z INFO Node{id=12}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:30.392287Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:53:30.479705Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727610391) } [INFO] [stdout] 2025-09-01T11:53:30.481704Z INFO Node{id=12}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=90 [INFO] [stdout] 2025-09-01T11:53:30.481735Z INFO Node{id=12}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:30.535730Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:30.538803Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:53:30.581310Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:30.583326Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:30.583897Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:30.585247Z INFO Node{id=20}: sequenced_broadcast: 121|Subscriber(3): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:30.588582Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:30.588651Z INFO Node{id=12}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:30.825869Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:30.825918Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:53:30.826888Z INFO Node{id=3}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:30.826964Z INFO Node{id=3}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:30.827702Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:30.833590Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727610826) } [INFO] [stdout] 2025-09-01T11:53:30.836138Z INFO Node{id=3}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=10 [INFO] [stdout] 2025-09-01T11:53:30.836170Z INFO Node{id=3}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:30.840281Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:30.845328Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:53:30.845362Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 28, 31] peer=31 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:30.846799Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:30.846852Z INFO Node{id=3}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:30.847955Z INFO Node{id=3}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:30.848857Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:30.849697Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:30.886195Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:30.995900Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:30.995946Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:53:30.996426Z INFO Node{id=19}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:30.996507Z INFO Node{id=19}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:30.997337Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:31.007786Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:31.009997Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:53:31.010018Z ERROR Node{id=2}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=17 leader=1 local=2 [INFO] [stdout] 2025-09-01T11:53:31.011103Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:31.011144Z INFO Node{id=2}:connect{target=17}:WritePeer{remote=17}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:31.011825Z INFO Node{id=2}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:31.012451Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:31.141190Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:31.141234Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:53:31.141830Z INFO Node{id=17}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:31.141907Z INFO Node{id=17}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:31.142854Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:31.148142Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727611141) } [INFO] [stdout] 2025-09-01T11:53:31.150308Z INFO Node{id=17}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:53:31.150339Z INFO Node{id=17}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:31.154159Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:31.159607Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:53:31.159636Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22] peer=22 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:31.160398Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:31.160964Z INFO Node{id=17}:connect{target=22}:ReadPeer{remote=22}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:31.161034Z INFO Node{id=17}:connect{target=22}:WritePeer{remote=22}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:31.161126Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:31.161978Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:31.340891Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:31.547128Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:53:31.681583Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727610996) } [INFO] [stdout] 2025-09-01T11:53:31.683046Z INFO Node{id=19}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=687 [INFO] [stdout] 2025-09-01T11:53:31.683080Z INFO Node{id=19}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:31.708672Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:31.847209Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:31.847257Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:53:31.848044Z INFO Node{id=3}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:31.848114Z INFO Node{id=3}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:31.848204Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:31.950707Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727611848) } [INFO] [stdout] 2025-09-01T11:53:31.952223Z INFO Node{id=3}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=104 [INFO] [stdout] 2025-09-01T11:53:31.952256Z INFO Node{id=3}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:32.012678Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:32.012726Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:53:32.013518Z INFO Node{id=2}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:32.013603Z INFO Node{id=2}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:32.014346Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:53:32.036472Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:32.048473Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:32.049394Z INFO Node{id=14}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:32.066314Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:32.067656Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:53:32.067680Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16] peer=16 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:32.069096Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:32.069227Z INFO Node{id=3}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:32.069314Z INFO Node{id=3}:connect{target=16}:WritePeer{remote=16}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:32.070059Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:32.073913Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:32.125883Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727612013) } [INFO] [stdout] 2025-09-01T11:53:32.126241Z INFO Node{id=2}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=113 [INFO] [stdout] 2025-09-01T11:53:32.126272Z INFO Node{id=2}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:32.161983Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:32.162026Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:53:32.162371Z INFO Node{id=17}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:32.162472Z INFO Node{id=17}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:32.162543Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:32.165469Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727612162) } [INFO] [stdout] 2025-09-01T11:53:32.166367Z INFO Node{id=17}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=4 [INFO] [stdout] 2025-09-01T11:53:32.166395Z INFO Node{id=17}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:32.168172Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:32.170412Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:53:32.170438Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10, 9] peer=9 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:32.170520Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:32.170744Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:32.171262Z INFO Node{id=17}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:32.171347Z INFO Node{id=17}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:32.171491Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:32.174464Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:32.220034Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:32.220667Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:53:32.300516Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:32.301027Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:53:32.301056Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25] peer=25 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:32.302388Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:32.302434Z INFO Node{id=19}:connect{target=25}:WritePeer{remote=25}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:32.302520Z INFO Node{id=19}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:32.302609Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:32.315109Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:32.315728Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:32.318032Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:32.318648Z INFO Node{id=27}: sequenced_broadcast: 133|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:32.319388Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:32.319443Z INFO Node{id=2}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:32.324201Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:32.324687Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:32.479771Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:32.481335Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:32.524147Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:32.688239Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:32.688580Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29]) [INFO] [stdout] 2025-09-01T11:53:32.868303Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:33.051581Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:33.051625Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:53:33.051745Z INFO Node{id=14}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:33.051869Z INFO Node{id=14}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:33.051928Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:53:33.053990Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727613051) } [INFO] [stdout] 2025-09-01T11:53:33.054113Z INFO Node{id=14}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:53:33.054138Z INFO Node{id=14}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:33.054343Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:33.055821Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:33.061418Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:53:33.061640Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:33.062180Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:33.062308Z INFO Node{id=13}: sequenced_broadcast: 130|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:33.063024Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:33.063082Z INFO Node{id=14}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:33.070469Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:33.070518Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:53:33.071259Z INFO Node{id=3}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:33.071345Z INFO Node{id=3}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:33.071513Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:33.082074Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727613071) } [INFO] [stdout] 2025-09-01T11:53:33.083636Z INFO Node{id=3}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=12 [INFO] [stdout] 2025-09-01T11:53:33.083666Z INFO Node{id=3}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:33.088485Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:33.090438Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:53:33.090463Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13] peer=13 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:33.094178Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:33.094243Z INFO Node{id=3}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:33.094424Z INFO Node{id=3}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:33.095286Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:33.098085Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:33.172217Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:33.172266Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:53:33.172644Z INFO Node{id=17}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:33.172729Z INFO Node{id=17}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:33.173596Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:33.179143Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727613172) } [INFO] [stdout] 2025-09-01T11:53:33.179188Z INFO Node{id=5}: sharedstate::worker::sync_manager: Discovered Peer client_id=17 [INFO] [stdout] 2025-09-01T11:53:33.180634Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:53:33.180662Z INFO Node{id=5}: sharedstate::worker::sync_manager: Broadcasted 8 peers to 7 peers [INFO] [stdout] 2025-09-01T11:53:33.181514Z INFO Node{id=17}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:53:33.181541Z INFO Node{id=17}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:33.182978Z ERROR Node{id=17}:connect{target=5}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:53:33.183666Z INFO Node{id=17}:connect{target=5}:WritePeer{remote=5}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:33.183851Z INFO Node{id=17}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:33.184526Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:33.185546Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:33.185834Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(5) [INFO] [stdout] 2025-09-01T11:53:33.248375Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:33.250801Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29]) [INFO] [stdout] 2025-09-01T11:53:33.304249Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:33.304303Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:53:33.304617Z INFO Node{id=19}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:33.304696Z INFO Node{id=19}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:33.305549Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:33.311298Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727613304) } [INFO] [stdout] 2025-09-01T11:53:33.313328Z INFO Node{id=19}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:53:33.313358Z INFO Node{id=19}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:33.318437Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:33.325135Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:33.326130Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:53:33.326154Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 8] peer=8 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:33.327678Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:33.328268Z INFO Node{id=19}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:33.328478Z INFO Node{id=19}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:33.329332Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:33.652502Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:33.824480Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:33.863963Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:33.865889Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:33.865913Z ERROR Node{id=29}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:34.010896Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:34.095452Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:34.095500Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:53:34.096375Z INFO Node{id=3}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:34.096469Z INFO Node{id=3}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:34.099496Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:34.105138Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727614096) } [INFO] [stdout] 2025-09-01T11:53:34.107251Z INFO Node{id=3}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:53:34.107279Z INFO Node{id=3}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:34.111634Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:34.114922Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:53:34.114949Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 23] peer=23 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:34.116440Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:34.116494Z INFO Node{id=3}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:34.116676Z INFO Node{id=3}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:34.117497Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:34.118475Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:34.186445Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:34.186488Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:53:34.187202Z INFO Node{id=17}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:34.187283Z INFO Node{id=17}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:34.188113Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:34.196499Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727614187) } [INFO] [stdout] 2025-09-01T11:53:34.198045Z INFO Node{id=17}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:53:34.198074Z INFO Node{id=17}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:34.207681Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:34.210014Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:53:34.210047Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 29, 20, 24] peer=24 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:34.210555Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:34.211314Z INFO Node{id=17}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:34.211401Z INFO Node{id=17}:connect{target=24}:WritePeer{remote=24}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:34.213080Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:34.218861Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:34.266809Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:34.270943Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:34.273502Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:53:34.328335Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:34.328385Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=11 leader=1 [INFO] [stdout] 2025-09-01T11:53:34.329145Z INFO Node{id=19}:connect{target=11}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:34.329228Z INFO Node{id=19}:connect{target=11}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:34.330089Z INFO Node{id=11}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:34.338622Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727614329) } [INFO] [stdout] 2025-09-01T11:53:34.341127Z INFO Node{id=19}:connect{target=11}: sharedstate::worker::sync_manager: got pong from peer latency=12 [INFO] [stdout] 2025-09-01T11:53:34.341157Z INFO Node{id=19}:connect{target=11}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:34.345729Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:34.351591Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 11 } [INFO] [stdout] 2025-09-01T11:53:34.351628Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 18, 26, 11] peer=11 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:34.352088Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:34.352653Z INFO Node{id=19}:connect{target=11}:WritePeer{remote=11}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:34.352884Z INFO Node{id=19}:connect{target=11}:ReadPeer{remote=11}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:34.353581Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:34.354651Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:34.461563Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:34.467607Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:34.467677Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:34.470182Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 28]) [INFO] [stdout] 2025-09-01T11:53:34.506326Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:34.802278Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:34.803851Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16]) [INFO] [stdout] 2025-09-01T11:53:34.906357Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:34.910302Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:34.912608Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22]) [INFO] [stdout] 2025-09-01T11:53:34.975617Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:34.977874Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14]) [INFO] [stdout] 2025-09-01T11:53:35.045558Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:35.048599Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6]) [INFO] [stdout] 2025-09-01T11:53:35.117665Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:35.117710Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=28 leader=1 [INFO] [stdout] 2025-09-01T11:53:35.119553Z INFO Node{id=3}:connect{target=28}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:35.119643Z INFO Node{id=3}:connect{target=28}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:35.120407Z INFO Node{id=28}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:35.127316Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727615119) } [INFO] [stdout] 2025-09-01T11:53:35.131854Z INFO Node{id=3}:connect{target=28}: sharedstate::worker::sync_manager: got pong from peer latency=12 [INFO] [stdout] 2025-09-01T11:53:35.131901Z INFO Node{id=3}:connect{target=28}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:35.135373Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:35.138422Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 28 } [INFO] [stdout] 2025-09-01T11:53:35.138450Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 28] peer=28 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:35.139740Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:35.141837Z INFO Node{id=3}:connect{target=28}:WritePeer{remote=28}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:35.142663Z INFO Node{id=3}:connect{target=28}:ReadPeer{remote=28}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:35.143335Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:35.143395Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:35.194858Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:35.198690Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:35.201232Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22, 15]) [INFO] [stdout] 2025-09-01T11:53:35.212344Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:35.212391Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=11 leader=1 [INFO] [stdout] 2025-09-01T11:53:35.213305Z INFO Node{id=17}:connect{target=11}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:35.213382Z INFO Node{id=17}:connect{target=11}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:35.214205Z INFO Node{id=11}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:35.219423Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727615213) } [INFO] [stdout] 2025-09-01T11:53:35.221799Z INFO Node{id=17}:connect{target=11}: sharedstate::worker::sync_manager: got pong from peer latency=8 [INFO] [stdout] 2025-09-01T11:53:35.221829Z INFO Node{id=17}:connect{target=11}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:35.225604Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:35.227833Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 11 } [INFO] [stdout] 2025-09-01T11:53:35.227856Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 18, 26, 11] peer=11 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:35.229209Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:35.229364Z INFO Node{id=17}:connect{target=11}:ReadPeer{remote=11}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:35.229438Z INFO Node{id=17}:connect{target=11}:WritePeer{remote=11}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:35.231449Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:35.232360Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:35.353690Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:35.353734Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:53:35.354660Z INFO Node{id=19}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:35.354742Z INFO Node{id=19}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:35.355617Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:35.361117Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727615354) } [INFO] [stdout] 2025-09-01T11:53:35.363352Z INFO Node{id=19}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:53:35.363382Z INFO Node{id=19}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:35.367367Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:35.370518Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:53:35.370547Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 28, 31] peer=31 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:35.371943Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:35.371988Z INFO Node{id=19}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:35.372725Z INFO Node{id=19}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:35.373396Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:35.374034Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:35.414084Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:35.418144Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:35.420454Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 18]) [INFO] [stdout] 2025-09-01T11:53:35.458175Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:35.460052Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([29]) [INFO] [stdout] 2025-09-01T11:53:35.507754Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.507808Z INFO Node{id=16}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.508027Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.508045Z INFO Node{id=10}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.508112Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.508127Z INFO Node{id=29}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.508331Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.508349Z INFO Node{id=14}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.508511Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.508528Z INFO Node{id=24}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.508594Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.508609Z INFO Node{id=1}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.508633Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.508644Z INFO Node{id=2}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.508666Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.508675Z INFO Node{id=3}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.508691Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.508700Z INFO Node{id=4}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.508720Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.508729Z INFO Node{id=5}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.508746Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.508754Z INFO Node{id=7}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.510840Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.510852Z INFO Node{id=8}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.510881Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.510892Z INFO Node{id=9}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.510917Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.510929Z INFO Node{id=11}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.510954Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.510970Z INFO Node{id=12}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.511000Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.511011Z INFO Node{id=15}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.511035Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.511046Z INFO Node{id=18}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.511068Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.511078Z INFO Node{id=19}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.511102Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.511115Z INFO Node{id=21}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.511136Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.511147Z INFO Node{id=22}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.511173Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.511184Z INFO Node{id=23}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.511210Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.511222Z INFO Node{id=26}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.511246Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.511257Z INFO Node{id=28}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.511278Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.511295Z INFO Node{id=30}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.511319Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.511330Z INFO Node{id=31}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.511350Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.511360Z INFO Node{id=32}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.511856Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.511875Z INFO Node{id=25}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.512264Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.512284Z INFO Node{id=13}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.512366Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.512382Z INFO Node{id=6}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.512744Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.512780Z INFO Node{id=17}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.515073Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.515095Z INFO Node{id=27}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.515709Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:53:35.515729Z INFO Node{id=20}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:53:35.869050Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:36.024477Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:36.028460Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10]) [INFO] [stdout] 2025-09-01T11:53:36.142891Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:36.142940Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:53:36.143852Z INFO Node{id=3}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:36.143937Z INFO Node{id=3}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:36.144839Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:36.190939Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727616143) } [INFO] [stdout] 2025-09-01T11:53:36.192577Z INFO Node{id=3}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=49 [INFO] [stdout] 2025-09-01T11:53:36.192615Z INFO Node{id=3}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:36.226939Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:36.231102Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:53:36.231133Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 29, 20, 24] peer=24 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:36.231256Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:36.231287Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:53:36.232195Z INFO Node{id=17}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:36.232272Z INFO Node{id=17}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:36.232966Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:36.233020Z INFO Node{id=3}:connect{target=24}:WritePeer{remote=24}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:36.233106Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:36.233977Z INFO Node{id=3}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:36.234956Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:36.261494Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:36.372624Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:36.372680Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:53:36.373688Z INFO Node{id=19}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:36.373777Z INFO Node{id=19}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:36.374557Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:36.382348Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727616373) } [INFO] [stdout] 2025-09-01T11:53:36.384838Z INFO Node{id=19}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:53:36.384871Z INFO Node{id=19}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:36.390716Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:36.393753Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:53:36.393799Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22] peer=22 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:36.395334Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:36.395393Z INFO Node{id=19}:connect{target=22}:WritePeer{remote=22}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:36.396285Z INFO Node{id=19}:connect{target=22}:ReadPeer{remote=22}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:36.397202Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:36.398099Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:36.731424Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:36.735243Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:36.737664Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 18, 26]) [INFO] [stdout] 2025-09-01T11:53:37.233860Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:37.233912Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:53:37.234623Z INFO Node{id=3}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:37.234700Z INFO Node{id=3}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:37.235617Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:37.243431Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727617234) } [INFO] [stdout] 2025-09-01T11:53:37.245948Z INFO Node{id=3}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:53:37.245980Z INFO Node{id=3}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:37.254995Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:37.258074Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:53:37.258103Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10] peer=10 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:37.261864Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:37.261953Z INFO Node{id=3}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:37.262048Z INFO Node{id=3}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:37.262885Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:37.263780Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:37.396117Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:37.396164Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:53:37.396880Z INFO Node{id=19}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:37.396953Z INFO Node{id=19}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:37.396996Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:37.403684Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727617396) } [INFO] [stdout] 2025-09-01T11:53:37.405014Z INFO Node{id=19}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:53:37.405045Z INFO Node{id=19}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:37.410250Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:37.413253Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:53:37.413279Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 18] peer=18 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:37.414502Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:37.414548Z INFO Node{id=19}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:37.414646Z INFO Node{id=19}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:37.415192Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:37.416197Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:37.438734Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:37.461205Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:37.526593Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:37.529593Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:37.529622Z ERROR Node{id=29}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:37.530523Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:37.531417Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24]) [INFO] [stdout] 2025-09-01T11:53:37.621738Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:37.622677Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14]) [INFO] [stdout] 2025-09-01T11:53:38.145165Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727616232) } [INFO] [stdout] 2025-09-01T11:53:38.145305Z INFO Node{id=17}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=1913 [INFO] [stdout] 2025-09-01T11:53:38.145331Z INFO Node{id=17}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:38.183962Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:38.262509Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:38.262568Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=4 leader=1 [INFO] [stdout] 2025-09-01T11:53:38.263379Z INFO Node{id=3}:connect{target=4}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:38.263460Z INFO Node{id=3}:connect{target=4}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:38.263512Z INFO Node{id=4}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:38.266283Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727618263) } [INFO] [stdout] 2025-09-01T11:53:38.266361Z INFO Node{id=3}:connect{target=4}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:53:38.266381Z INFO Node{id=3}:connect{target=4}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:38.267633Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:38.267816Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 4 } [INFO] [stdout] 2025-09-01T11:53:38.267838Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 29, 4] peer=4 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:38.269036Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:38.269083Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:38.269115Z INFO Node{id=3}:connect{target=4}:ReadPeer{remote=4}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:38.269171Z INFO Node{id=3}:connect{target=4}:WritePeer{remote=4}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:38.269294Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:38.415101Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:38.415147Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:53:38.415921Z INFO Node{id=19}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:38.415997Z INFO Node{id=19}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:38.416045Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:38.419422Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727618415) } [INFO] [stdout] 2025-09-01T11:53:38.419499Z INFO Node{id=19}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=4 [INFO] [stdout] 2025-09-01T11:53:38.419521Z INFO Node{id=19}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:38.422174Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:38.422390Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:53:38.422412Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[30] peer=30 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:38.423963Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:38.424009Z INFO Node{id=19}:connect{target=30}:WritePeer{remote=30}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:38.424074Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:38.424116Z INFO Node{id=19}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:38.424233Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:38.716212Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:38.717378Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:38.718064Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27]) [INFO] [stdout] 2025-09-01T11:53:38.737460Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:38.819469Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:38.820015Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6]) [INFO] [stdout] 2025-09-01T11:53:38.985454Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:38.987152Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:38.988191Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10]) [INFO] [stdout] 2025-09-01T11:53:39.116717Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:39.116890Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([29]) [INFO] [stdout] 2025-09-01T11:53:39.241720Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:39.242084Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:39.270160Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:39.270204Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:53:39.270859Z INFO Node{id=3}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:39.270939Z INFO Node{id=3}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:39.270977Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:39.272874Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727619270) } [INFO] [stdout] 2025-09-01T11:53:39.272969Z INFO Node{id=3}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:53:39.272989Z INFO Node{id=3}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:39.274110Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:39.274255Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:53:39.274274Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10, 32] peer=32 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:39.275583Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:39.275635Z INFO Node{id=3}:connect{target=32}:WritePeer{remote=32}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:39.275704Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:39.275748Z INFO Node{id=3}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:39.275891Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:39.424258Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:39.424311Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:53:39.424952Z INFO Node{id=19}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:39.425020Z INFO Node{id=19}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:39.425058Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:39.427081Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727619425) } [INFO] [stdout] 2025-09-01T11:53:39.427162Z INFO Node{id=19}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:53:39.427184Z INFO Node{id=19}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:39.428540Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:39.428714Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:53:39.428733Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27] peer=27 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:39.428837Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:39.428881Z INFO Node{id=19}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:39.428936Z INFO Node{id=19}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:39.430324Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:39.430410Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:39.530402Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:39.531908Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:39.554338Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:39.554693Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13]) [INFO] [stdout] 2025-09-01T11:53:39.554715Z WARN Node{id=28}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:53:39.554801Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:39.554863Z INFO Node{id=28}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:39.555740Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:39.659887Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:39.717902Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:39.718095Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20]) [INFO] [stdout] 2025-09-01T11:53:39.727701Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:39.730420Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:39.730542Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10]) [INFO] [stdout] 2025-09-01T11:53:39.733422Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:39.738879Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:53:39.738916Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14] peer=14 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:39.739066Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:39.739719Z INFO Node{id=17}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:39.742855Z INFO Node{id=17}:connect{target=14}:WritePeer{remote=14}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:39.743030Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:40.276168Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:40.276212Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:53:40.276891Z INFO Node{id=3}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:40.277036Z INFO Node{id=3}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:40.277115Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:40.279138Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727620277) } [INFO] [stdout] 2025-09-01T11:53:40.279262Z INFO Node{id=3}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:53:40.279290Z INFO Node{id=3}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:40.279485Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:40.284485Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:53:40.284519Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[30] peer=30 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:40.284637Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:40.284686Z INFO Node{id=3}:connect{target=30}:WritePeer{remote=30}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:40.284794Z INFO Node{id=3}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:40.286209Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:40.286280Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:40.429120Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:40.429168Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:53:40.429885Z INFO Node{id=19}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:40.429961Z INFO Node{id=19}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:40.430017Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:40.432113Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727620429) } [INFO] [stdout] 2025-09-01T11:53:40.432194Z INFO Node{id=19}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:53:40.432214Z INFO Node{id=19}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:40.432362Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:40.433025Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:53:40.433045Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 18, 26] peer=26 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:40.433119Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:40.433190Z INFO Node{id=19}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:40.433246Z INFO Node{id=19}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:40.434424Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:40.434491Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:40.539216Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:40.547195Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:40.547402Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20]) [INFO] [stdout] 2025-09-01T11:53:40.557238Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:40.557285Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:53:40.558047Z INFO Node{id=28}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:40.558127Z INFO Node{id=28}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:40.558189Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:53:40.560405Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727620558) } [INFO] [stdout] 2025-09-01T11:53:40.560504Z INFO Node{id=28}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:53:40.560546Z INFO Node{id=28}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:40.560731Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:40.561405Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:53:40.566978Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:40.567123Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:40.567336Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:40.568345Z INFO Node{id=9}: sequenced_broadcast: 128|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:40.569252Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:40.569311Z INFO Node{id=28}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:40.742423Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:40.742468Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:53:40.742612Z INFO Node{id=17}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:40.742684Z INFO Node{id=17}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:40.742728Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:40.744996Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727620742) } [INFO] [stdout] 2025-09-01T11:53:40.745082Z INFO Node{id=17}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:53:40.745104Z INFO Node{id=17}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:40.745263Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:40.748560Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:40.754045Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:53:40.754081Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 18, 26] peer=26 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:40.754149Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:40.755469Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:40.755525Z INFO Node{id=17}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:40.755613Z INFO Node{id=17}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:40.755719Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:40.962524Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:40.964271Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:40.964302Z ERROR Node{id=29}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:41.117140Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:41.285409Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:41.285457Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:53:41.286179Z INFO Node{id=3}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:41.286270Z INFO Node{id=3}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:41.286323Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:41.288610Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727621286) } [INFO] [stdout] 2025-09-01T11:53:41.288707Z INFO Node{id=3}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:53:41.288731Z INFO Node{id=3}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:41.288944Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:41.289710Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:53:41.289733Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 18, 26] peer=26 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:41.289866Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:41.289915Z INFO Node{id=3}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:41.289979Z INFO Node{id=3}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:41.290687Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:41.290735Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:41.328931Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:41.373452Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:53:41.373509Z INFO Node{id=1}: sharedstate::worker::sync_manager: Broadcasted 31 peers to 31 peers [INFO] [stdout] 2025-09-01T11:53:41.434092Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:41.434138Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:53:41.434918Z INFO Node{id=19}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:41.435003Z INFO Node{id=19}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:41.435075Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:41.437400Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727621434) } [INFO] [stdout] 2025-09-01T11:53:41.437508Z INFO Node{id=19}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:53:41.437542Z INFO Node{id=19}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:41.437747Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:41.438490Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:53:41.438513Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 23] peer=23 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:53:41.438635Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:41.438682Z INFO Node{id=19}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:41.438781Z INFO Node{id=19}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:41.439523Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:41.440289Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:41.756388Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:41.756434Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:53:41.757107Z INFO Node{id=17}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:41.757236Z INFO Node{id=17}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:41.757316Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:41.763457Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727621757) } [INFO] [stdout] 2025-09-01T11:53:41.763583Z INFO Node{id=17}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=6 [INFO] [stdout] 2025-09-01T11:53:41.763607Z INFO Node{id=17}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:41.763803Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:41.766110Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:41.767631Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:53:41.767655Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22, 15, 21] peer=21 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:41.768601Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:41.768681Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:41.769206Z INFO Node{id=17}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:41.769273Z INFO Node{id=17}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:41.769389Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:42.221429Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:42.222713Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:42.222876Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27]) [INFO] [stdout] 2025-09-01T11:53:42.290079Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:42.290127Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:53:42.290861Z INFO Node{id=3}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:42.290951Z INFO Node{id=3}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:42.291002Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:42.325249Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:42.429144Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:42.429782Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([29]) [INFO] [stdout] 2025-09-01T11:53:42.441108Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:42.441156Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:53:42.441781Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:42.441862Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:42.441925Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:53:42.443642Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727622441) } [INFO] [stdout] 2025-09-01T11:53:42.443720Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:53:42.443739Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:42.443903Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:42.444360Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:53:42.444955Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:42.445052Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:42.445171Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:42.445698Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, seq(4099) < min_allowed(27471) [INFO] [stdout] 2025-09-01T11:53:42.446292Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:42.446847Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(22): Added, allow_drop: true, next_sequence: 29519, min_allowed_seq: 27471 [INFO] [stdout] 2025-09-01T11:53:42.447327Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 29519) [INFO] [stdout] 2025-09-01T11:53:42.447410Z INFO Node{id=19}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:42.447432Z INFO Node{id=19}: sequenced_broadcast: 134|SequencedBroadcastWorker Started id=134 [INFO] [stdout] 2025-09-01T11:53:42.447538Z INFO Node{id=19}: sequenced_broadcast: 91|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:42.447555Z INFO Node{id=19}: sequenced_broadcast: 91|SequencedBroadcastWorker Stopped id=91 elapsed=81.13825262s iter=1799 [INFO] [stdout] 2025-09-01T11:53:42.487590Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:42.522659Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:42.523683Z INFO Node{id=29}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:42.529617Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:42.689470Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:42.769673Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:42.769718Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:53:42.770089Z INFO Node{id=17}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:42.770174Z INFO Node{id=17}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:42.770941Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:42.787732Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727622770) } [INFO] [stdout] 2025-09-01T11:53:42.788896Z INFO Node{id=17}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:53:42.788927Z INFO Node{id=17}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:42.806729Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:42.808960Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:53:42.808987Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13] peer=13 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:42.810245Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:42.810656Z INFO Node{id=17}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:42.810732Z INFO Node{id=17}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:42.811651Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:42.828347Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:42.921041Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:42.922812Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:43.063073Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:43.124578Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:43.126866Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13]) [INFO] [stdout] 2025-09-01T11:53:43.253358Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:43.458168Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:43.459457Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:43.529330Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:43.529375Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:53:43.529654Z INFO Node{id=29}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:43.529780Z INFO Node{id=29}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:43.529887Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:43.534022Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727623529) } [INFO] [stdout] 2025-09-01T11:53:43.534855Z INFO Node{id=29}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=5 [INFO] [stdout] 2025-09-01T11:53:43.534885Z INFO Node{id=29}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:43.537297Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:43.540595Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:43.544456Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:53:43.544491Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27] peer=27 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:43.545984Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:43.550690Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:43.550835Z INFO Node{id=29}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:43.550907Z INFO Node{id=29}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:43.551658Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:43.652695Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:43.653442Z INFO Node{id=30}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:43.653838Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:43.811473Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:43.811519Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:53:43.812361Z INFO Node{id=17}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:43.812472Z INFO Node{id=17}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:43.812627Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:43.818850Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727623812) } [INFO] [stdout] 2025-09-01T11:53:43.819855Z INFO Node{id=17}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=7 [INFO] [stdout] 2025-09-01T11:53:43.819884Z INFO Node{id=17}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:43.821621Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:43.823485Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:53:43.823511Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10, 32] peer=32 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:43.823718Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:43.823799Z INFO Node{id=17}:connect{target=32}:WritePeer{remote=32}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:43.824715Z INFO Node{id=17}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:43.825776Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:43.825898Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:44.052318Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727622290) } [INFO] [stdout] 2025-09-01T11:53:44.053663Z INFO Node{id=3}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=1763 [INFO] [stdout] 2025-09-01T11:53:44.053698Z INFO Node{id=3}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:44.288774Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:44.291267Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:44.299889Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:53:44.471699Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:44.474122Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:44.475207Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10, 9, 28]) [INFO] [stdout] 2025-09-01T11:53:44.484362Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:44.485875Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:44.485898Z ERROR Node{id=4}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:44.553854Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:44.553901Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:53:44.554412Z INFO Node{id=29}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:44.554510Z INFO Node{id=29}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:44.555357Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:44.559480Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727624554) } [INFO] [stdout] 2025-09-01T11:53:44.559569Z INFO Node{id=29}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=5 [INFO] [stdout] 2025-09-01T11:53:44.559592Z INFO Node{id=29}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:44.564164Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:44.567334Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:44.572273Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:53:44.572302Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22, 15] peer=15 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:44.573240Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:44.573866Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:44.574474Z INFO Node{id=29}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:44.574571Z INFO Node{id=29}:connect{target=15}:WritePeer{remote=15}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:44.574741Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:44.655925Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:44.655978Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:53:44.656903Z INFO Node{id=30}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:44.656987Z INFO Node{id=30}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:44.657116Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:53:44.661637Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727624656) } [INFO] [stdout] 2025-09-01T11:53:44.662646Z INFO Node{id=30}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=6 [INFO] [stdout] 2025-09-01T11:53:44.662674Z INFO Node{id=30}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:44.665081Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:44.665270Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:53:44.667663Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:44.668551Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:44.668695Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:44.674265Z INFO Node{id=26}: sequenced_broadcast: 115|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:44.675459Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:44.675520Z INFO Node{id=30}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:44.804674Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:44.807311Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:44.808159Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 16]) [INFO] [stdout] 2025-09-01T11:53:44.824343Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:44.824389Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:53:44.825247Z INFO Node{id=17}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:44.825328Z INFO Node{id=17}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:44.825471Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:44.832626Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727624825) } [INFO] [stdout] 2025-09-01T11:53:44.834554Z INFO Node{id=17}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:53:44.834586Z INFO Node{id=17}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:44.841147Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:44.842022Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:53:44.842046Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 18] peer=18 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:53:44.842131Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:44.842172Z INFO Node{id=17}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:44.842243Z INFO Node{id=17}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:44.843929Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:44.844002Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:44.913400Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:44.920267Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:44.921270Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22]) [INFO] [stdout] 2025-09-01T11:53:44.982386Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:45.049991Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:45.057921Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:45.058895Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 16, 6]) [INFO] [stdout] 2025-09-01T11:53:45.202484Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:45.207181Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:45.208344Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 22, 15]) [INFO] [stdout] 2025-09-01T11:53:45.421410Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:45.429474Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:45.430538Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 18]) [INFO] [stdout] 2025-09-01T11:53:45.575456Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:45.575502Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:53:45.576205Z INFO Node{id=29}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:45.576286Z INFO Node{id=29}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:45.576408Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:45.579582Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727625576) } [INFO] [stdout] 2025-09-01T11:53:45.580358Z INFO Node{id=29}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=4 [INFO] [stdout] 2025-09-01T11:53:45.580384Z INFO Node{id=29}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:45.582032Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:45.584820Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:45.587340Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:53:45.587362Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10, 8] peer=8 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:45.588161Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:45.588845Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:45.589287Z INFO Node{id=29}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:45.589363Z INFO Node{id=29}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:45.589526Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:45.765213Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:45.765897Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:45.765920Z ERROR Node{id=20}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:45.846371Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:45.846418Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:53:45.847199Z INFO Node{id=17}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:45.847308Z INFO Node{id=17}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:45.847456Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:53:45.851730Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727625847) } [INFO] [stdout] 2025-09-01T11:53:45.852199Z INFO Node{id=17}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=5 [INFO] [stdout] 2025-09-01T11:53:45.852231Z INFO Node{id=17}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:45.856075Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:45.857264Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:53:45.860896Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:45.861330Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:45.862493Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:45.864250Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, seq(4099) < min_allowed(30975) [INFO] [stdout] 2025-09-01T11:53:45.872665Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:45.872889Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(33036) < seq(33039) [INFO] [stdout] 2025-09-01T11:53:45.884750Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(23): Added, allow_drop: true, next_sequence: 33039, min_allowed_seq: 30997 [INFO] [stdout] 2025-09-01T11:53:45.885374Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 33039) [INFO] [stdout] 2025-09-01T11:53:45.885461Z INFO Node{id=17}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:45.885488Z INFO Node{id=17}: sequenced_broadcast: 135|SequencedBroadcastWorker Started id=135 [INFO] [stdout] 2025-09-01T11:53:45.885605Z INFO Node{id=17}: sequenced_broadcast: 85|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:45.885623Z INFO Node{id=17}: sequenced_broadcast: 85|SequencedBroadcastWorker Stopped id=85 elapsed=90.203917125s iter=1957 [INFO] [stdout] 2025-09-01T11:53:46.008005Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:46.011177Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:53:46.011220Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=29 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:53:46.012702Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:46.012752Z INFO Node{id=3}:connect{target=29}:WritePeer{remote=29}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:46.013601Z INFO Node{id=3}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:46.014441Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:46.030580Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:46.035366Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:46.040254Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10]) [INFO] [stdout] 2025-09-01T11:53:46.486673Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:46.589984Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:46.590030Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:53:46.590522Z INFO Node{id=29}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:46.590611Z INFO Node{id=29}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:46.590714Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:46.595300Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727626590) } [INFO] [stdout] 2025-09-01T11:53:46.597027Z INFO Node{id=29}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=7 [INFO] [stdout] 2025-09-01T11:53:46.597056Z INFO Node{id=29}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:46.600393Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:46.607300Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:46.610667Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:53:46.610705Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10, 9, 28, 31] peer=31 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:46.611502Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:46.611975Z INFO Node{id=29}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:46.612053Z INFO Node{id=29}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:46.612190Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:46.738458Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:46.741268Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:46.742558Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 18, 26]) [INFO] [stdout] 2025-09-01T11:53:46.951608Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:46.952979Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14]) [INFO] [stdout] 2025-09-01T11:53:47.013149Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:47.013194Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:53:47.014034Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:47.014119Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:47.014248Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:47.444428Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727627014) } [INFO] [stdout] 2025-09-01T11:53:47.445932Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=431 [INFO] [stdout] 2025-09-01T11:53:47.445964Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:47.509615Z INFO sharedstate::testing::fuzzy_test: node0 sequence updated: 0 => 34842 [INFO] [stdout] 2025-09-01T11:53:47.509635Z INFO sharedstate::testing::fuzzy_test: node1 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509641Z INFO sharedstate::testing::fuzzy_test: node2 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509645Z INFO sharedstate::testing::fuzzy_test: node3 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509649Z INFO sharedstate::testing::fuzzy_test: node4 sequence updated: 0 => 34724 [INFO] [stdout] 2025-09-01T11:53:47.509653Z INFO sharedstate::testing::fuzzy_test: node5 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509657Z INFO sharedstate::testing::fuzzy_test: node6 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509661Z INFO sharedstate::testing::fuzzy_test: node7 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509665Z INFO sharedstate::testing::fuzzy_test: node8 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509669Z INFO sharedstate::testing::fuzzy_test: node9 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509673Z INFO sharedstate::testing::fuzzy_test: node10 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509677Z INFO sharedstate::testing::fuzzy_test: node11 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509681Z INFO sharedstate::testing::fuzzy_test: node12 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509685Z INFO sharedstate::testing::fuzzy_test: node13 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509697Z INFO sharedstate::testing::fuzzy_test: node14 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509702Z INFO sharedstate::testing::fuzzy_test: node15 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509705Z INFO sharedstate::testing::fuzzy_test: node16 sequence updated: 0 => 34784 [INFO] [stdout] 2025-09-01T11:53:47.509709Z INFO sharedstate::testing::fuzzy_test: node17 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509713Z INFO sharedstate::testing::fuzzy_test: node18 sequence updated: 0 => 34708 [INFO] [stdout] 2025-09-01T11:53:47.509718Z INFO sharedstate::testing::fuzzy_test: node19 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509722Z INFO sharedstate::testing::fuzzy_test: node20 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509726Z INFO sharedstate::testing::fuzzy_test: node21 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509730Z INFO sharedstate::testing::fuzzy_test: node22 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509734Z INFO sharedstate::testing::fuzzy_test: node23 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509738Z INFO sharedstate::testing::fuzzy_test: node24 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509742Z INFO sharedstate::testing::fuzzy_test: node25 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509746Z INFO sharedstate::testing::fuzzy_test: node26 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509750Z INFO sharedstate::testing::fuzzy_test: node27 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509754Z INFO sharedstate::testing::fuzzy_test: node28 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509779Z INFO sharedstate::testing::fuzzy_test: node29 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509783Z INFO sharedstate::testing::fuzzy_test: node30 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.509787Z INFO sharedstate::testing::fuzzy_test: node31 sequence updated: 0 => 4098 [INFO] [stdout] 2025-09-01T11:53:47.534870Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:47.602986Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:47.604186Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24]) [INFO] [stdout] 2025-09-01T11:53:47.614687Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:47.614734Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:53:47.615298Z INFO Node{id=29}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:47.615394Z INFO Node{id=29}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:47.615521Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:47.620203Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727627615) } [INFO] [stdout] 2025-09-01T11:53:47.623139Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:47.623980Z INFO Node{id=29}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=8 [INFO] [stdout] 2025-09-01T11:53:47.624007Z INFO Node{id=29}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:47.626430Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:47.628580Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:53:47.628604Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 16, 6] peer=6 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:47.629946Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:47.631395Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:47.633158Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:47.633697Z INFO Node{id=29}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:47.633804Z INFO Node{id=29}:connect{target=6}:WritePeer{remote=6}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:47.633954Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:47.767860Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:47.810721Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:47.939240Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:47.946043Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:53:48.444629Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:48.446899Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:48.446924Z ERROR Node{id=4}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:48.525872Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:48.526331Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:48.526778Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:48.527364Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, seq(4099) < min_allowed(33692) [INFO] [stdout] 2025-09-01T11:53:48.634830Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:48.634877Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:53:48.635130Z INFO Node{id=29}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:48.635233Z INFO Node{id=29}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:48.635333Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:48.719321Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:48.721540Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:48.721687Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27]) [INFO] [stdout] 2025-09-01T11:53:48.757530Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:48.757901Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(36016) < seq(36052) [INFO] [stdout] 2025-09-01T11:53:48.769316Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, max_seq(36027) < seq(36052) [INFO] [stdout] 2025-09-01T11:53:48.770043Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 36052, max: 36027 } [INFO] [stdout] 2025-09-01T11:53:48.770067Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:53:48.770335Z ERROR Node{id=3}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:48.773123Z WARN Node{id=3}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:53:48.773145Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:48.773267Z INFO Node{id=3}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:48.821369Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:48.822606Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:48.822777Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 16, 6]) [INFO] [stdout] 2025-09-01T11:53:48.989143Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:48.990032Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:48.990134Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10]) [INFO] [stdout] 2025-09-01T11:53:49.243218Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:49.244154Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:49.244260Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:49.466344Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:49.466542Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14]) [INFO] [stdout] 2025-09-01T11:53:49.466726Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727628635) } [INFO] [stdout] 2025-09-01T11:53:49.467030Z INFO Node{id=29}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=832 [INFO] [stdout] 2025-09-01T11:53:49.467059Z INFO Node{id=29}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:49.467402Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:49.467613Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:53:49.467633Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14] peer=14 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:49.467730Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:49.467796Z INFO Node{id=29}:connect{target=14}:WritePeer{remote=14}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:49.467863Z INFO Node{id=29}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:49.468024Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:49.468251Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:49.472352Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:49.472457Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:49.472481Z ERROR Node{id=20}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:49.719409Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:49.731127Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:49.731852Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:49.731964Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10]) [INFO] [stdout] 2025-09-01T11:53:49.748052Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:49.748417Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([20]) [INFO] [stdout] 2025-09-01T11:53:49.774911Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:49.774958Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:53:49.775404Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:49.775553Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:49.775604Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:53:49.776802Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727629775) } [INFO] [stdout] 2025-09-01T11:53:49.776880Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:53:49.776901Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:49.777037Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:49.777412Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:53:49.777821Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:49.777939Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:49.778059Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:49.778432Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber rejected, seq(4099) < min_allowed(34006) [INFO] [stdout] 2025-09-01T11:53:49.778816Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:53:49.779168Z INFO Node{id=1}: sequenced_broadcast: 15|Subscriber(24): Added, allow_drop: true, next_sequence: 36054, min_allowed_seq: 34006 [INFO] [stdout] 2025-09-01T11:53:49.779579Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 36054) [INFO] [stdout] 2025-09-01T11:53:49.779658Z INFO Node{id=3}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:53:49.779684Z INFO Node{id=3}: sequenced_broadcast: 136|SequencedBroadcastWorker Started id=136 [INFO] [stdout] 2025-09-01T11:53:49.781834Z INFO Node{id=3}: sequenced_broadcast: 75|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:53:49.781854Z INFO Node{id=3}: sequenced_broadcast: 75|SequencedBroadcastWorker Stopped id=75 elapsed=99.240489825s iter=2074 [INFO] [stdout] 2025-09-01T11:53:50.447139Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:50.470728Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:50.470797Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:53:50.471190Z INFO Node{id=29}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:50.471276Z INFO Node{id=29}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:50.472177Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:50.478827Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727630471) } [INFO] [stdout] 2025-09-01T11:53:50.481167Z INFO Node{id=29}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=10 [INFO] [stdout] 2025-09-01T11:53:50.481198Z INFO Node{id=29}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:50.486105Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:50.491588Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:53:50.491616Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14, 18] peer=18 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:50.492668Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:50.493091Z INFO Node{id=29}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:50.493179Z INFO Node{id=29}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:50.493265Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:50.501475Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:50.548617Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:50.553450Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:50.555663Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([20]) [INFO] [stdout] 2025-09-01T11:53:50.562464Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:50.566809Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:50.568939Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10, 9]) [INFO] [stdout] 2025-09-01T11:53:51.474362Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:51.493872Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:51.493918Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:53:51.494229Z INFO Node{id=29}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:51.494348Z INFO Node{id=29}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:51.494459Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:51.496444Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727631494) } [INFO] [stdout] 2025-09-01T11:53:51.496624Z INFO Node{id=29}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:53:51.496649Z INFO Node{id=29}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:51.496925Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:51.498697Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:51.499033Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:53:51.499055Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13] peer=13 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:51.502135Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:51.502468Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:51.502968Z INFO Node{id=29}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:51.503038Z INFO Node{id=29}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:51.503213Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:51.749605Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:51.752947Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:51.753125Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([20]) [INFO] [stdout] 2025-09-01T11:53:52.184363Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:52.184929Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:52.184953Z ERROR Node{id=4}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:52.223465Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:52.224678Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:52.224882Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 16, 6, 27]) [INFO] [stdout] 2025-09-01T11:53:52.450498Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:52.451629Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:52.466556Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:53:52.505325Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:52.505373Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:53:52.505593Z INFO Node{id=29}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:52.505712Z INFO Node{id=29}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:52.505824Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:52.507957Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727632505) } [INFO] [stdout] 2025-09-01T11:53:52.508124Z INFO Node{id=29}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:53:52.508150Z INFO Node{id=29}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:52.508406Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:52.511462Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:52.515330Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:53:52.515356Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 16, 6, 22] peer=22 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:52.516430Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:52.518918Z INFO Node{id=29}:connect{target=22}:ReadPeer{remote=22}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:52.519002Z INFO Node{id=29}:connect{target=22}:WritePeer{remote=22}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:52.519186Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:52.558937Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:52.560131Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:52.560290Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([20]) [INFO] [stdout] 2025-09-01T11:53:52.689281Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:52.691923Z INFO Node{id=4}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:52.692216Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:52.923244Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:52.924450Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:52.924640Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:53.127142Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:53.128492Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:53.128734Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13]) [INFO] [stdout] 2025-09-01T11:53:53.185513Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:53.185642Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:53.185663Z ERROR Node{id=20}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:53.253609Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:53.253725Z INFO Node{id=20}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:53.253998Z ERROR ReadClient{remote=20}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:53.460680Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:53.460953Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:53.461068Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:53.510120Z INFO sharedstate::testing::fuzzy_test: node0 sequence updated: 34842 => 40150 [INFO] [stdout] 2025-09-01T11:53:53.510145Z INFO sharedstate::testing::fuzzy_test: node2 sequence updated: 4098 => 40150 [INFO] [stdout] 2025-09-01T11:53:53.510150Z INFO sharedstate::testing::fuzzy_test: node4 sequence updated: 34724 => 40150 [INFO] [stdout] 2025-09-01T11:53:53.510160Z INFO sharedstate::testing::fuzzy_test: node16 sequence updated: 34784 => 40150 [INFO] [stdout] 2025-09-01T11:53:53.510164Z INFO sharedstate::testing::fuzzy_test: node18 sequence updated: 34708 => 40150 [INFO] [stdout] 2025-09-01T11:53:53.517525Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:53.517590Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:53:53.517699Z INFO Node{id=29}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:53.517873Z INFO Node{id=29}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:53.517939Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:53.518452Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727633517) } [INFO] [stdout] 2025-09-01T11:53:53.518548Z INFO Node{id=29}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:53:53.518575Z INFO Node{id=29}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:53.518736Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:53.518915Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:53:53.518937Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 7] peer=7 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:53.519052Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:53.519107Z INFO Node{id=29}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:53.519172Z INFO Node{id=29}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:53.519262Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:53.519322Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:53.693859Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:53.693918Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:53:53.694045Z INFO Node{id=4}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:53.694132Z INFO Node{id=4}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:53.694186Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:53:53.694676Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727633694) } [INFO] [stdout] 2025-09-01T11:53:53.694789Z INFO Node{id=4}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:53:53.694812Z INFO Node{id=4}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:53.694949Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:53.695085Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:53:53.695261Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:53.695399Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:53.695507Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:53.695603Z INFO Node{id=15}: sequenced_broadcast: 105|Subscriber(2): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:53.695718Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:53.695786Z INFO Node{id=4}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:53.754909Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:53.755124Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:53.755228Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:53.755248Z ERROR Node{id=24}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:54.256081Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:54.256145Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:53:54.256261Z INFO Node{id=20}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:54.256359Z INFO Node{id=20}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:54.256423Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:53:54.256905Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727634256) } [INFO] [stdout] 2025-09-01T11:53:54.256987Z INFO Node{id=20}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:53:54.257009Z INFO Node{id=20}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:54.257138Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:54.257267Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:53:54.257428Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:54.257525Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:54.257624Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:54.257701Z INFO Node{id=7}: sequenced_broadcast: 131|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:54.257824Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:54.257874Z INFO Node{id=20}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:54.300566Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:54.300819Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:54.300928Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:53:54.477893Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:54.478161Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:54.478279Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10, 9, 28]) [INFO] [stdout] 2025-09-01T11:53:54.520963Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:54.521019Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:53:54.521118Z INFO Node{id=29}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:54.521195Z INFO Node{id=29}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:54.521251Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:54.521703Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727634521) } [INFO] [stdout] 2025-09-01T11:53:54.521822Z INFO Node{id=29}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:53:54.521848Z INFO Node{id=29}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:54.522007Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:54.522161Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:53:54.522184Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 16, 6, 27, 2] peer=2 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:54.522301Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:54.522350Z INFO Node{id=29}:connect{target=2}:WritePeer{remote=2}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:54.522428Z INFO Node{id=29}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:54.522524Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:54.522574Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:54.562127Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:54.562396Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:54.562503Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 7, 20]) [INFO] [stdout] 2025-09-01T11:53:54.562523Z WARN Node{id=12}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:53:54.562558Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:54.562609Z INFO Node{id=12}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:54.562846Z ERROR ReadClient{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:54.666814Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:54.667071Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:54.667193Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 18, 26]) [INFO] [stdout] 2025-09-01T11:53:54.809785Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:54.810049Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:54.810171Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14, 25, 16]) [INFO] [stdout] 2025-09-01T11:53:54.923039Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:54.923276Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:54.923394Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 16, 6, 22]) [INFO] [stdout] 2025-09-01T11:53:55.059463Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:55.059725Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:55.059861Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14, 25, 16, 6]) [INFO] [stdout] 2025-09-01T11:53:55.209472Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:55.209709Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:55.209832Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 16, 6, 22, 15]) [INFO] [stdout] 2025-09-01T11:53:55.432199Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:55.432464Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:55.432601Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14, 18]) [INFO] [stdout] 2025-09-01T11:53:55.523721Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:55.523803Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:53:55.523921Z INFO Node{id=29}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:55.524031Z INFO Node{id=29}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:55.524090Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:55.524651Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727635524) } [INFO] [stdout] 2025-09-01T11:53:55.524744Z INFO Node{id=29}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:53:55.524797Z INFO Node{id=29}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:55.524961Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:55.525111Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:53:55.525133Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10] peer=10 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:55.525232Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:55.525288Z INFO Node{id=29}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:55.525351Z INFO Node{id=29}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:55.525435Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:55.525486Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:55.563875Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:55.563934Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:53:55.564120Z INFO Node{id=12}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:55.564206Z INFO Node{id=12}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:55.564282Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:53:55.564705Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727635564) } [INFO] [stdout] 2025-09-01T11:53:55.564810Z INFO Node{id=12}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:53:55.564831Z INFO Node{id=12}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:55.564958Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:55.565144Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:53:55.565309Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:55.565414Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:55.565513Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:55.565604Z INFO Node{id=26}: sequenced_broadcast: 115|Subscriber(3): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:55.565706Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:55.565790Z INFO Node{id=12}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:55.757159Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:55.757440Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:55.757586Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 7, 20]) [INFO] [stdout] 2025-09-01T11:53:55.757611Z WARN Node{id=24}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:53:55.757658Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:55.757714Z INFO Node{id=24}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:55.757994Z ERROR ReadClient{remote=24}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:55.858602Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:55.858875Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:55.858984Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:53:56.042057Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:56.042331Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:56.042453Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10]) [INFO] [stdout] 2025-09-01T11:53:56.527149Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:56.527210Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:53:56.527365Z INFO Node{id=29}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:56.527465Z INFO Node{id=29}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:56.527569Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:56.528025Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727636527) } [INFO] [stdout] 2025-09-01T11:53:56.528108Z INFO Node{id=29}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:53:56.528132Z INFO Node{id=29}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:56.528274Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:56.528407Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:53:56.528428Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14, 25, 16] peer=16 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:56.528525Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:56.528583Z INFO Node{id=29}:connect{target=16}:WritePeer{remote=16}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:56.528640Z INFO Node{id=29}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:56.528723Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:56.528790Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:56.743515Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:56.744201Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:56.744325Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14, 18, 26]) [INFO] [stdout] 2025-09-01T11:53:56.760027Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:56.760090Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:53:56.760213Z INFO Node{id=24}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:56.760312Z INFO Node{id=24}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:56.760377Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:53:56.760946Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727636760) } [INFO] [stdout] 2025-09-01T11:53:56.761044Z INFO Node{id=24}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:53:56.761067Z INFO Node{id=24}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:56.761203Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:56.761355Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:53:56.761376Z ERROR Node{id=24}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 7] peer=7 leader=1 local=24 [INFO] [stdout] 2025-09-01T11:53:56.761461Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:56.761499Z INFO Node{id=24}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:56.761572Z INFO Node{id=24}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:56.761685Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:56.761747Z ERROR ReadClient{remote=24}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:56.954856Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:56.955102Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:56.955211Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14]) [INFO] [stdout] 2025-09-01T11:53:57.529561Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:57.529618Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:53:57.529719Z INFO Node{id=29}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:57.529812Z INFO Node{id=29}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:57.529866Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:57.530244Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727637529) } [INFO] [stdout] 2025-09-01T11:53:57.530324Z INFO Node{id=29}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:53:57.530344Z INFO Node{id=29}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:57.530468Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:57.530596Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:53:57.530615Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14, 25, 23] peer=23 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:57.530697Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:57.530739Z INFO Node{id=29}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:57.530833Z INFO Node{id=29}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:57.530915Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:57.530973Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:57.606145Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:57.606413Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:57.606522Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:57.606549Z ERROR Node{id=13}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:57.763274Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:57.763344Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:53:57.763454Z INFO Node{id=24}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:57.763547Z INFO Node{id=24}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:57.763609Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:53:57.764113Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727637763) } [INFO] [stdout] 2025-09-01T11:53:57.764209Z INFO Node{id=24}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:53:57.764235Z INFO Node{id=24}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:57.764401Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:57.764554Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:53:57.764721Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:57.764856Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:53:57.764971Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeRecovery(RecoverableStateDetails { id: 1769850849781770309, generation: 1, recover_accept_seq: 4099, state_accept_seq: 4098, history: [] }) } [INFO] [stdout] 2025-09-01T11:53:57.765061Z INFO Node{id=32}: sequenced_broadcast: 129|Subscriber(1): Added, allow_drop: true, next_sequence: 4099, min_allowed_seq: 4099 [INFO] [stdout] 2025-09-01T11:53:57.765185Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 4099) [INFO] [stdout] 2025-09-01T11:53:57.765239Z INFO Node{id=24}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:53:58.533872Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:58.533925Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:53:58.534064Z INFO Node{id=29}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:58.534139Z INFO Node{id=29}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:58.534221Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:58.534688Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727638534) } [INFO] [stdout] 2025-09-01T11:53:58.534800Z INFO Node{id=29}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:53:58.534823Z INFO Node{id=29}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:58.534957Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:58.535210Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:53:58.535230Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14, 18, 26] peer=26 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:58.535347Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:58.535386Z INFO Node{id=29}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:58.535439Z INFO Node{id=29}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:58.535507Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:58.535550Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:58.723221Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:58.723450Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:58.723576Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 16, 6, 27]) [INFO] [stdout] 2025-09-01T11:53:58.723600Z WARN Node{id=10}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:53:58.723643Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:58.723705Z INFO Node{id=10}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:58.723982Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:58.823885Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:58.824128Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:58.824237Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14, 25, 16, 6]) [INFO] [stdout] 2025-09-01T11:53:58.991598Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:58.991899Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:58.992019Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:58.992043Z ERROR Node{id=9}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:59.245893Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:59.246184Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:59.246329Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14, 25]) [INFO] [stdout] 2025-09-01T11:53:59.467428Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:59.467805Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:59.467918Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 3, 19, 10, 2, 28, 17, 29, 20, 24, 13, 14]) [INFO] [stdout] 2025-09-01T11:53:59.511909Z INFO sharedstate::testing::fuzzy_test: leader sequence stable [INFO] [stdout] 2025-09-01T11:53:59.536895Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:59.536987Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:53:59.537102Z INFO Node{id=29}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:59.537368Z INFO Node{id=29}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:59.537427Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:53:59.538053Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727639537) } [INFO] [stdout] 2025-09-01T11:53:59.538186Z INFO Node{id=29}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:53:59.538212Z INFO Node{id=29}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:59.538426Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:59.538728Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:53:59.538750Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[9] peer=9 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:53:59.538901Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:59.538946Z INFO Node{id=29}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:59.539045Z INFO Node{id=29}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:59.539141Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:59.539225Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:59.609879Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:59.610181Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:59.610350Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10, 32, 24]) [INFO] [stdout] 2025-09-01T11:53:59.726086Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:53:59.726150Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:53:59.726356Z INFO Node{id=10}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:53:59.726451Z INFO Node{id=10}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:53:59.726510Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:53:59.727025Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727639726) } [INFO] [stdout] 2025-09-01T11:53:59.727124Z INFO Node{id=10}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:53:59.727150Z INFO Node{id=10}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:53:59.727319Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:59.727460Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:53:59.727482Z ERROR Node{id=10}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 16, 6, 22, 15] peer=15 leader=1 local=10 [INFO] [stdout] 2025-09-01T11:53:59.727597Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:53:59.727734Z INFO Node{id=10}:connect{target=15}:WritePeer{remote=15}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:53:59.727819Z INFO Node{id=10}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:53:59.727902Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:53:59.727948Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:53:59.733850Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:59.734093Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:59.734199Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:53:59.734219Z ERROR Node{id=32}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:53:59.778386Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:53:59.778658Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:53:59.778795Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:54:00.539601Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:00.539658Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:54:00.539885Z INFO Node{id=29}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:00.539971Z INFO Node{id=29}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:00.540031Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:54:00.540502Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727640539) } [INFO] [stdout] 2025-09-01T11:54:00.540593Z INFO Node{id=29}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:54:00.540616Z INFO Node{id=29}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:00.540795Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:00.540952Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:54:00.540975Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 10, 24, 13, 14, 25, 16, 6, 22, 15, 21] peer=21 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:54:00.541179Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:00.541232Z INFO Node{id=29}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:00.541304Z INFO Node{id=29}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:00.541396Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:00.541445Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:00.570009Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:54:00.570264Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:00.570380Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([9]) [INFO] [stdout] 2025-09-01T11:54:00.728842Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:00.728897Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:54:00.729089Z INFO Node{id=10}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:00.729172Z INFO Node{id=10}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:00.729221Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:54:00.729672Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727640729) } [INFO] [stdout] 2025-09-01T11:54:00.729774Z INFO Node{id=10}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:54:00.729801Z INFO Node{id=10}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:00.729961Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:00.730123Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:54:00.730145Z ERROR Node{id=10}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 29, 12, 3, 19, 14, 25, 16, 6, 27, 10, 9, 28, 31] peer=31 leader=1 local=10 [INFO] [stdout] 2025-09-01T11:54:00.730242Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:00.730296Z INFO Node{id=10}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:00.730462Z INFO Node{id=10}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:00.730560Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:00.730612Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:00.994064Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:54:00.994276Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:00.994376Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:54:00.994394Z ERROR Node{id=9}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] [INFO] [stdout] thread 'testing::fuzzy_test::fuzzy_test' (21) panicked at src/testing/fuzzy_test.rs:173:13: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: TestState { sequence: 4098, numbers: [1378, 205, 205, 205, 0, 205, 206, 0, 0, 0, 203, 204, 204, 204, 204, 21, 22, 20, 0, 0, 0, 0, 0, 176, 99, 71, 55, 0, 0, 25, 0, 186] } [INFO] [stdout] right: TestState { sequence: 40150, numbers: [8786, 1117, 3502, 1121, 1525, 1063, 280, 187, 165, 622, 1140, 298, 1126, 1445, 1381, 728, 1618, 768, 3212, 937, 340, 376, 445, 1110, 1251, 761, 999, 206, 744, 878, 902, 1117] } [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x5e97b9557d52 - std::backtrace_rs::backtrace::libunwind::trace::h7cddb8376417e7cc [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x5e97b9557d52 - std::backtrace_rs::backtrace::trace_unsynchronized::hb4e41acf4b349ff1 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x5e97b9557d52 - std::sys::backtrace::_print_fmt::h1222b80910ba6eb5 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x5e97b9557d52 - ::fmt::h5bb8a979ba5db788 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x5e97b956977f - core::fmt::rt::Argument::fmt::h2ee2c138a50a7796 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x5e97b956977f - core::fmt::write::h1e0dbf07fe3990bd [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/fmt/mod.rs:1468:25 [INFO] [stdout] 6: 0x5e97b9524ee3 - std::io::default_write_fmt::h7d7ad5ed6a883d81 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x5e97b9524ee3 - std::io::Write::write_fmt::hc477d9325b345ece [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x5e97b9530e62 - std::sys::backtrace::BacktraceLock::print::h788d486777205086 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x5e97b9535b7f - std::panicking::default_hook::{{closure}}::ha7bdfeb5949fc0fa [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x5e97b9535a11 - std::panicking::default_hook::h6dad75ec721846f4 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x5e97b940225e - as core::ops::function::Fn>::call::h33799d04b0b96146 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 12: 0x5e97b940225e - test::test_main_with_exit_callback::{{closure}}::h880e31c829a2eb5d [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x5e97b95362ce - as core::ops::function::Fn>::call::h5e1b8c652ea49180 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 14: 0x5e97b95362ce - std::panicking::panic_with_hook::hba00e869ada17676 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x5e97b9535fea - std::panicking::panic_handler::{{closure}}::h3d21149c258e5ceb [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:707:13 [INFO] [stdout] 16: 0x5e97b9530f99 - std::sys::backtrace::__rust_end_short_backtrace::h4f5d9b9dfb3e6ec1 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x5e97b95190ed - __rustc[9a7a9f9af7564de1]::rust_begin_unwind [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x5e97b9571700 - core::panicking::panic_fmt::h78e817a90331d98b [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/panicking.rs:75:14 [INFO] [stdout] 19: 0x5e97b95714b3 - core::panicking::assert_failed_inner::he0a3934689115f42 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/panicking.rs:439:17 [INFO] [stdout] 20: 0x5e97b92c4d0a - core::panicking::assert_failed::h145430560fe517b9 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/panicking.rs:394:5 [INFO] [stdout] 21: 0x5e97b931f101 - sharedstate::testing::fuzzy_test::_fuzzy_test::{{closure}}::h8992b8b19f0e5290 [INFO] [stdout] at /opt/rustwide/workdir/src/testing/fuzzy_test.rs:173:13 [INFO] [stdout] 22: 0x5e97b931b74c - sharedstate::testing::fuzzy_test::fuzzy_test::{{closure}}::hfb7afd19cf1e6965 [INFO] [stdout] at /opt/rustwide/workdir/src/testing/fuzzy_test.rs:14:19 [INFO] [stdout] 23: 0x5e97b9360922 - as core::future::future::Future>::poll::hdc2cde685b1c9153 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/future/future.rs:133:9 [INFO] [stdout] 24: 0x5e97b936081d - as core::future::future::Future>::poll::ha7e441f4304e0948 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/future/future.rs:133:9 [INFO] [stdout] 25: 0x5e97b9385f6d - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}}::h0e229549dd3a4ed5 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:733:70 [INFO] [stdout] 26: 0x5e97b9385e9b - tokio::task::coop::with_budget::h82a11d48344b33ba [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 27: 0x5e97b9385e9b - tokio::task::coop::budget::h56b88984c5ce2455 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 28: 0x5e97b9385e9b - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::h8d9efced01555ee3 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:733:25 [INFO] [stdout] 29: 0x5e97b9382d40 - tokio::runtime::scheduler::current_thread::Context::enter::h10ebae8120046ff4 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 30: 0x5e97b93853dd - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::hfd9aca60d5395c04 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:732:44 [INFO] [stdout] 31: 0x5e97b9384614 - tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}}::h367a396a4648db5c [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:820:68 [INFO] [stdout] 32: 0x5e97b937a39b - tokio::runtime::context::scoped::Scoped::set::h4cec7548e67a6db8 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 33: 0x5e97b92fdeb9 - tokio::runtime::context::set_scheduler::{{closure}}::h3e010846485f5b56 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/context.rs:176:38 [INFO] [stdout] 34: 0x5e97b9398c62 - std::thread::local::LocalKey::try_with::h345a2d90251cbb6b [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 35: 0x5e97b939802e - std::thread::local::LocalKey::with::h225b682cef68382c [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 36: 0x5e97b92fde2d - tokio::runtime::context::set_scheduler::h3aa114d1cdeb3ad1 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/context.rs:176:17 [INFO] [stdout] 37: 0x5e97b9383f80 - tokio::runtime::scheduler::current_thread::CoreGuard::enter::h690900ccb2cef9df [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:820:27 [INFO] [stdout] 38: 0x5e97b9384693 - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::hb0e6b882d6778002 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:720:24 [INFO] [stdout] 39: 0x5e97b937b770 - tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}}::hc29be964be350afb [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 40: 0x5e97b937a910 - tokio::runtime::context::runtime::enter_runtime::he2f3043664243876 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 41: 0x5e97b937b131 - tokio::runtime::scheduler::current_thread::CurrentThread::block_on::h78c3d0d92fe243a7 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 42: 0x5e97b93593b4 - tokio::runtime::runtime::Runtime::block_on_inner::hfac5f2d2ba13bbff [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 43: 0x5e97b9359577 - tokio::runtime::runtime::Runtime::block_on::h4c2346d2a506f8e6 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 44: 0x5e97b931b563 - sharedstate::testing::fuzzy_test::fuzzy_test::h768d336becc798d3 [INFO] [stdout] at /opt/rustwide/workdir/src/testing/fuzzy_test.rs:14:24 [INFO] [stdout] 45: 0x5e97b931b5f7 - sharedstate::testing::fuzzy_test::fuzzy_test::{{closure}}::h2a1f4f6971413493 [INFO] [stdout] at /opt/rustwide/workdir/src/testing/fuzzy_test.rs:13:22 [INFO] [stdout] 46: 0x5e97b91ef816 - core::ops::function::FnOnce::call_once::h921374776a808dc5 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/ops/function.rs:253:5 [INFO] [stdout] 47: 0x5e97b940205b - core::ops::function::FnOnce::call_once::h7f4b4fba903e39d5 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/ops/function.rs:253:5 [INFO] [stdout] 48: 0x5e97b940205b - test::__rust_begin_short_backtrace::h9277cb6a2ccfc000 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/test/src/lib.rs:663:18 [INFO] [stdout] 49: 0x5e97b9417b05 - test::run_test_in_process::{{closure}}::h9aea5ca90d1f4423 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/test/src/lib.rs:686:74 [INFO] [stdout] 50: 0x5e97b9417b05 - as core::ops::function::FnOnce<()>>::call_once::h364f5fe6cc8afa85 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/panic/unwind_safe.rs:272:9 [INFO] [stdout] 51: 0x5e97b9417b05 - std::panicking::catch_unwind::do_call::h2b2bec3317fe53ec [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:590:40 [INFO] [stdout] 52: 0x5e97b9417b05 - std::panicking::catch_unwind::hc3763734156da4af [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:553:19 [INFO] [stdout] 53: 0x5e97b9417b05 - std::panic::catch_unwind::h28038391e867eabc [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panic.rs:359:14 [INFO] [stdout] 54: 0x5e97b9417b05 - test::run_test_in_process::had0273166695a036 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/test/src/lib.rs:686:27 [INFO] [stdout] 55: 0x5e97b9417b05 - test::run_test::{{closure}}::h743d09d4bb476605 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/test/src/lib.rs:607:43 [INFO] [stdout] 56: 0x5e97b93ee524 - test::run_test::{{closure}}::he950b8f9118d37e2 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/test/src/lib.rs:637:41 [INFO] [stdout] 57: 0x5e97b93ee524 - std::sys::backtrace::__rust_begin_short_backtrace::hbfaffa6539f6abb7 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 58: 0x5e97b93f1e0a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hcce3c2c65b9c3b20 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 59: 0x5e97b93f1e0a - as core::ops::function::FnOnce<()>>::call_once::hb66b30b6d37985a5 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/panic/unwind_safe.rs:272:9 [INFO] [stdout] 60: 0x5e97b93f1e0a - std::panicking::catch_unwind::do_call::h992bbe2c32dc1d79 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:590:40 [INFO] [stdout] 61: 0x5e97b93f1e0a - std::panicking::catch_unwind::h412991d5237de610 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panicking.rs:553:19 [INFO] [stdout] 62: 0x5e97b93f1e0a - std::panic::catch_unwind::ha82b139b3eb5840a [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/panic.rs:359:14 [INFO] [stdout] 63: 0x5e97b93f1e0a - std::thread::Builder::spawn_unchecked_::{{closure}}::ha823b36f5114938e [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 64: 0x5e97b93f1e0a - core::ops::function::FnOnce::call_once{{vtable.shim}}::hf120e7b1db22ac07 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/core/src/ops/function.rs:253:5 [INFO] [stdout] 65: 0x5e97b952b21f - as core::ops::function::FnOnce>::call_once::h3e049222c99298ac [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 66: 0x5e97b952b21f - std::sys::pal::unix::thread::Thread::new::thread_start::h942e336943ad5963 [INFO] [stdout] at /rustc/cdb45c87e2cd43495379f7e867e3cc15dcee9f93/library/std/src/sys/pal/unix/thread.rs:118:17 [INFO] [stdout] 67: 0x76916dec8aa4 - [INFO] [stdout] 68: 0x76916df55a34 - clone [INFO] [stdout] 69: 0x0 - [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] testing::fuzzy_test::fuzzy_test [INFO] [stdout] testing::state_tests::slow_state_deadlock_test [INFO] [stdout] [INFO] [stdout] test result: FAILED. 14 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out; finished in 111.26s [INFO] [stdout] [INFO] [stderr] error: test failed, to rerun pass `--lib` [INFO] running `Command { std: "docker" "inspect" "08a46c804b7ab9f4457424298e45448a67a968771879dee1db9c4cab6c0a232c", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "08a46c804b7ab9f4457424298e45448a67a968771879dee1db9c4cab6c0a232c", kill_on_drop: false }` [INFO] [stdout] 08a46c804b7ab9f4457424298e45448a67a968771879dee1db9c4cab6c0a232c