[INFO] fetching crate sharedstate 0.4.2... [INFO] testing sharedstate-0.4.2 against try#b83b707f97d809763b7861afa7638871f3339a33 for pr-145838-1 [INFO] extracting crate sharedstate 0.4.2 into /workspace/builds/worker-4-tc2/source [INFO] started tweaking crates.io crate sharedstate 0.4.2 [INFO] finished tweaking crates.io crate sharedstate 0.4.2 [INFO] tweaked toml for crates.io crate sharedstate 0.4.2 written to /workspace/builds/worker-4-tc2/source/Cargo.toml [INFO] validating manifest of crates.io crate sharedstate 0.4.2 on toolchain b83b707f97d809763b7861afa7638871f3339a33 [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+b83b707f97d809763b7861afa7638871f3339a33" "metadata" "--manifest-path" "Cargo.toml" "--no-deps", kill_on_drop: false }` [INFO] crate crates.io crate sharedstate 0.4.2 already has a lockfile, it will not be regenerated [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+b83b707f97d809763b7861afa7638871f3339a33" "fetch" "--manifest-path" "Cargo.toml", kill_on_drop: false }` [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7ad1b28ee6f5f7f699f6cf7015098d6ccdd96d6f2d78dd06228f5b4c9faf309c" "/opt/rustwide/cargo-home/bin/cargo" "+b83b707f97d809763b7861afa7638871f3339a33" "metadata" "--no-deps" "--format-version=1", kill_on_drop: false }` [INFO] [stdout] 070c15d581c3d18dd6cce1da90ae339d228aebb682d29d280473172f8eb26698 [INFO] running `Command { std: "docker" "start" "-a" "070c15d581c3d18dd6cce1da90ae339d228aebb682d29d280473172f8eb26698", kill_on_drop: false }` [INFO] running `Command { std: "docker" "inspect" "070c15d581c3d18dd6cce1da90ae339d228aebb682d29d280473172f8eb26698", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "070c15d581c3d18dd6cce1da90ae339d228aebb682d29d280473172f8eb26698", kill_on_drop: false }` [INFO] [stdout] 070c15d581c3d18dd6cce1da90ae339d228aebb682d29d280473172f8eb26698 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7ad1b28ee6f5f7f699f6cf7015098d6ccdd96d6f2d78dd06228f5b4c9faf309c" "/opt/rustwide/cargo-home/bin/cargo" "+b83b707f97d809763b7861afa7638871f3339a33" "build" "--frozen" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] 07e766ef524a726ea9f330d297f0f26aa7033a6053b1e2f36b88b578f578d9eb [INFO] running `Command { std: "docker" "start" "-a" "07e766ef524a726ea9f330d297f0f26aa7033a6053b1e2f36b88b578f578d9eb", kill_on_drop: false }` [INFO] [stderr] Compiling syn v2.0.101 [INFO] [stderr] Compiling lock_api v0.4.13 [INFO] [stderr] Compiling smallvec v1.15.0 [INFO] [stderr] Compiling mio v1.0.4 [INFO] [stderr] Compiling socket2 v0.5.10 [INFO] [stderr] Compiling signal-hook-registry v1.4.5 [INFO] [stderr] Compiling pkg-details v0.1.2 [INFO] [stderr] Compiling ppv-lite86 v0.2.21 [INFO] [stderr] Compiling arc-metrics v0.1.2 [INFO] [stderr] Compiling parking_lot_core v0.9.11 [INFO] [stderr] Compiling rand_core v0.6.4 [INFO] [stderr] Compiling message-encoding v0.2.7 [INFO] [stderr] Compiling parking_lot v0.12.4 [INFO] [stderr] Compiling rand_chacha v0.3.1 [INFO] [stderr] Compiling tokio-macros v2.5.0 [INFO] [stderr] Compiling tracing-attributes v0.1.28 [INFO] [stderr] Compiling futures-macro v0.3.31 [INFO] [stderr] Compiling tokio v1.45.1 [INFO] [stderr] Compiling tracing v0.1.41 [INFO] [stderr] Compiling futures-util v0.3.31 [INFO] [stderr] Compiling tokio-util v0.7.15 [INFO] [stderr] Compiling sequenced-broadcast v0.2.6 [INFO] [stderr] Compiling sharedstate v0.4.2 (/opt/rustwide/workdir) [INFO] [stdout] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:1:32 [INFO] [stdout] | [INFO] [stdout] 1 | use std::{hash::{Hash, Hasher, SipHasher}, time::{Duration, SystemTime, UNIX_EPOCH}}; [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(deprecated)]` on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:491:20 [INFO] [stdout] | [INFO] [stdout] 491 | let mut hash = SipHasher::new(); [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated associated function `std::hash::SipHasher::new`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:491:31 [INFO] [stdout] | [INFO] [stdout] 491 | let mut hash = SipHasher::new(); [INFO] [stdout] | ^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: field `last_client_msg_epoch` is never read [INFO] [stdout] --> src/worker/sync_manager.rs:1370:5 [INFO] [stdout] | [INFO] [stdout] 1367 | struct DiscoveredPeer { [INFO] [stdout] | -------------- field in this struct [INFO] [stdout] ... [INFO] [stdout] 1370 | last_client_msg_epoch: u64, [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: method `log` is never used [INFO] [stdout] --> src/utils.rs:4:8 [INFO] [stdout] | [INFO] [stdout] 3 | pub trait LogHelper { [INFO] [stdout] | --------- method in this trait [INFO] [stdout] 4 | fn log(self) -> Self; [INFO] [stdout] | ^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: trait `TimeoutPanicHelper` is never used [INFO] [stdout] --> src/utils.rs:110:11 [INFO] [stdout] | [INFO] [stdout] 110 | pub trait TimeoutPanicHelper: Sized { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:136:17 [INFO] [stdout] | [INFO] [stdout] 136 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] = note: `#[warn(mismatched_lifetime_syntaxes)]` on by default [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 136 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:142:17 [INFO] [stdout] | [INFO] [stdout] 142 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 142 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:266:23 [INFO] [stdout] | [INFO] [stdout] 266 | pub fn read_state(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 266 | pub fn read_state(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:384:17 [INFO] [stdout] | [INFO] [stdout] 384 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 384 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stderr] Finished `dev` profile [unoptimized + debuginfo] target(s) in 26.54s [INFO] running `Command { std: "docker" "inspect" "07e766ef524a726ea9f330d297f0f26aa7033a6053b1e2f36b88b578f578d9eb", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "07e766ef524a726ea9f330d297f0f26aa7033a6053b1e2f36b88b578f578d9eb", kill_on_drop: false }` [INFO] [stdout] 07e766ef524a726ea9f330d297f0f26aa7033a6053b1e2f36b88b578f578d9eb [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7ad1b28ee6f5f7f699f6cf7015098d6ccdd96d6f2d78dd06228f5b4c9faf309c" "/opt/rustwide/cargo-home/bin/cargo" "+b83b707f97d809763b7861afa7638871f3339a33" "test" "--frozen" "--no-run" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] 571154267aaffe549c9cca39fb90c2e15e66614a2fb7184a7a45e02b0a68bd2b [INFO] running `Command { std: "docker" "start" "-a" "571154267aaffe549c9cca39fb90c2e15e66614a2fb7184a7a45e02b0a68bd2b", kill_on_drop: false }` [INFO] [stderr] Compiling tracing v0.1.41 [INFO] [stderr] Compiling tracing-subscriber v0.3.19 [INFO] [stderr] Compiling sequenced-broadcast v0.2.6 [INFO] [stderr] Compiling sharedstate v0.4.2 (/opt/rustwide/workdir) [INFO] [stdout] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:1:32 [INFO] [stdout] | [INFO] [stdout] 1 | use std::{hash::{Hash, Hasher, SipHasher}, time::{Duration, SystemTime, UNIX_EPOCH}}; [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(deprecated)]` on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:491:20 [INFO] [stdout] | [INFO] [stdout] 491 | let mut hash = SipHasher::new(); [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated associated function `std::hash::SipHasher::new`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:491:31 [INFO] [stdout] | [INFO] [stdout] 491 | let mut hash = SipHasher::new(); [INFO] [stdout] | ^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: field `last_client_msg_epoch` is never read [INFO] [stdout] --> src/worker/sync_manager.rs:1370:5 [INFO] [stdout] | [INFO] [stdout] 1367 | struct DiscoveredPeer { [INFO] [stdout] | -------------- field in this struct [INFO] [stdout] ... [INFO] [stdout] 1370 | last_client_msg_epoch: u64, [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: method `log` is never used [INFO] [stdout] --> src/utils.rs:4:8 [INFO] [stdout] | [INFO] [stdout] 3 | pub trait LogHelper { [INFO] [stdout] | --------- method in this trait [INFO] [stdout] 4 | fn log(self) -> Self; [INFO] [stdout] | ^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: trait `TimeoutPanicHelper` is never used [INFO] [stdout] --> src/utils.rs:110:11 [INFO] [stdout] | [INFO] [stdout] 110 | pub trait TimeoutPanicHelper: Sized { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:136:17 [INFO] [stdout] | [INFO] [stdout] 136 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] = note: `#[warn(mismatched_lifetime_syntaxes)]` on by default [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 136 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:142:17 [INFO] [stdout] | [INFO] [stdout] 142 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 142 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:266:23 [INFO] [stdout] | [INFO] [stdout] 266 | pub fn read_state(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 266 | pub fn read_state(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:384:17 [INFO] [stdout] | [INFO] [stdout] 384 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 384 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: unused import: `worker::sync_updater::FollowTarget` [INFO] [stdout] --> src/worker/sync_updater.rs:502:133 [INFO] [stdout] | [INFO] [stdout] 502 | ...TestState, TestStateAction}}, utils::PanicHelper, worker::sync_updater::FollowTarget}; [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: unused import: `tokio::sync::mpsc::channel` [INFO] [stdout] --> src/testing/fuzzy_test.rs:4:5 [INFO] [stdout] | [INFO] [stdout] 4 | use tokio::sync::mpsc::channel; [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:1:32 [INFO] [stdout] | [INFO] [stdout] 1 | use std::{hash::{Hash, Hasher, SipHasher}, time::{Duration, SystemTime, UNIX_EPOCH}}; [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(deprecated)]` on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:491:20 [INFO] [stdout] | [INFO] [stdout] 491 | let mut hash = SipHasher::new(); [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: use of deprecated associated function `std::hash::SipHasher::new`: use `std::hash::DefaultHasher` instead [INFO] [stdout] --> src/worker/sync_updater.rs:491:31 [INFO] [stdout] | [INFO] [stdout] 491 | let mut hash = SipHasher::new(); [INFO] [stdout] | ^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: field `last_client_msg_epoch` is never read [INFO] [stdout] --> src/worker/sync_manager.rs:1370:5 [INFO] [stdout] | [INFO] [stdout] 1367 | struct DiscoveredPeer { [INFO] [stdout] | -------------- field in this struct [INFO] [stdout] ... [INFO] [stdout] 1370 | last_client_msg_epoch: u64, [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: method `log` is never used [INFO] [stdout] --> src/utils.rs:4:8 [INFO] [stdout] | [INFO] [stdout] 3 | pub trait LogHelper { [INFO] [stdout] | --------- method in this trait [INFO] [stdout] 4 | fn log(self) -> Self; [INFO] [stdout] | ^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: trait `TimeoutPanicHelper` is never used [INFO] [stdout] --> src/utils.rs:110:11 [INFO] [stdout] | [INFO] [stdout] 110 | pub trait TimeoutPanicHelper: Sized { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: field `rate_limit_bps` is never read [INFO] [stdout] --> src/testing/test_sync_io.rs:99:5 [INFO] [stdout] | [INFO] [stdout] 96 | struct ConnManage { [INFO] [stdout] | ---------- field in this struct [INFO] [stdout] ... [INFO] [stdout] 99 | rate_limit_bps: Arc, [INFO] [stdout] | ^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:136:17 [INFO] [stdout] | [INFO] [stdout] 136 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] = note: `#[warn(mismatched_lifetime_syntaxes)]` on by default [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 136 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:142:17 [INFO] [stdout] | [INFO] [stdout] 142 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 142 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:266:23 [INFO] [stdout] | [INFO] [stdout] 266 | pub fn read_state(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 266 | pub fn read_state(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/state.rs:384:17 [INFO] [stdout] | [INFO] [stdout] 384 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stdout] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 384 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stderr] Finished `test` profile [unoptimized + debuginfo] target(s) in 14.74s [INFO] running `Command { std: "docker" "inspect" "571154267aaffe549c9cca39fb90c2e15e66614a2fb7184a7a45e02b0a68bd2b", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "571154267aaffe549c9cca39fb90c2e15e66614a2fb7184a7a45e02b0a68bd2b", kill_on_drop: false }` [INFO] [stdout] 571154267aaffe549c9cca39fb90c2e15e66614a2fb7184a7a45e02b0a68bd2b [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7ad1b28ee6f5f7f699f6cf7015098d6ccdd96d6f2d78dd06228f5b4c9faf309c" "/opt/rustwide/cargo-home/bin/cargo" "+b83b707f97d809763b7861afa7638871f3339a33" "test" "--frozen", kill_on_drop: false }` [INFO] [stdout] 593d2bde7e94d6db5f1f22cb023b34dfa28ed30f62328215421da1d66ab0e72f [INFO] running `Command { std: "docker" "start" "-a" "593d2bde7e94d6db5f1f22cb023b34dfa28ed30f62328215421da1d66ab0e72f", kill_on_drop: false }` [INFO] [stderr] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stderr] --> src/worker/sync_updater.rs:1:32 [INFO] [stderr] | [INFO] [stderr] 1 | use std::{hash::{Hash, Hasher, SipHasher}, time::{Duration, SystemTime, UNIX_EPOCH}}; [INFO] [stderr] | ^^^^^^^^^ [INFO] [stderr] | [INFO] [stderr] = note: `#[warn(deprecated)]` on by default [INFO] [stderr] [INFO] [stderr] warning: use of deprecated struct `std::hash::SipHasher`: use `std::hash::DefaultHasher` instead [INFO] [stderr] --> src/worker/sync_updater.rs:491:20 [INFO] [stderr] | [INFO] [stderr] 491 | let mut hash = SipHasher::new(); [INFO] [stderr] | ^^^^^^^^^ [INFO] [stderr] [INFO] [stderr] warning: use of deprecated associated function `std::hash::SipHasher::new`: use `std::hash::DefaultHasher` instead [INFO] [stderr] --> src/worker/sync_updater.rs:491:31 [INFO] [stderr] | [INFO] [stderr] 491 | let mut hash = SipHasher::new(); [INFO] [stderr] | ^^^ [INFO] [stderr] [INFO] [stderr] warning: field `last_client_msg_epoch` is never read [INFO] [stderr] --> src/worker/sync_manager.rs:1370:5 [INFO] [stderr] | [INFO] [stderr] 1367 | struct DiscoveredPeer { [INFO] [stderr] | -------------- field in this struct [INFO] [stderr] ... [INFO] [stderr] 1370 | last_client_msg_epoch: u64, [INFO] [stderr] | ^^^^^^^^^^^^^^^^^^^^^ [INFO] [stderr] | [INFO] [stderr] = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default [INFO] [stderr] [INFO] [stderr] warning: method `log` is never used [INFO] [stderr] --> src/utils.rs:4:8 [INFO] [stderr] | [INFO] [stderr] 3 | pub trait LogHelper { [INFO] [stderr] | --------- method in this trait [INFO] [stderr] 4 | fn log(self) -> Self; [INFO] [stderr] | ^^^ [INFO] [stderr] [INFO] [stderr] warning: trait `TimeoutPanicHelper` is never used [INFO] [stderr] --> src/utils.rs:110:11 [INFO] [stderr] | [INFO] [stderr] 110 | pub trait TimeoutPanicHelper: Sized { [INFO] [stderr] | ^^^^^^^^^^^^^^^^^^ [INFO] [stderr] [INFO] [stderr] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stderr] --> src/state.rs:136:17 [INFO] [stderr] | [INFO] [stderr] 136 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stderr] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stderr] | | [INFO] [stderr] | the lifetime is elided here [INFO] [stderr] | [INFO] [stderr] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stderr] = note: `#[warn(mismatched_lifetime_syntaxes)]` on by default [INFO] [stderr] help: use `'_` for type paths [INFO] [stderr] | [INFO] [stderr] 136 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stderr] | +++ [INFO] [stderr] [INFO] [stderr] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stderr] --> src/state.rs:142:17 [INFO] [stderr] | [INFO] [stderr] 142 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stderr] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stderr] | | [INFO] [stderr] | the lifetime is elided here [INFO] [stderr] | [INFO] [stderr] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stderr] help: use `'_` for type paths [INFO] [stderr] | [INFO] [stderr] 142 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stderr] | +++ [INFO] [stderr] [INFO] [stderr] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stderr] --> src/state.rs:266:23 [INFO] [stderr] | [INFO] [stderr] 266 | pub fn read_state(&self) -> RwLockReadGuard { [INFO] [stderr] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stderr] | | [INFO] [stderr] | the lifetime is elided here [INFO] [stderr] | [INFO] [stderr] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stderr] help: use `'_` for type paths [INFO] [stderr] | [INFO] [stderr] 266 | pub fn read_state(&self) -> RwLockReadGuard<'_, D> { [INFO] [stderr] | +++ [INFO] [stderr] [INFO] [stderr] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stderr] --> src/state.rs:384:17 [INFO] [stderr] | [INFO] [stderr] 384 | pub fn read(&self) -> RwLockReadGuard { [INFO] [stderr] | ^^^^^ ^^^^^^^^^^^^^^^^^^ the same lifetime is hidden here [INFO] [stderr] | | [INFO] [stderr] | the lifetime is elided here [INFO] [stderr] | [INFO] [stderr] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stderr] help: use `'_` for type paths [INFO] [stderr] | [INFO] [stderr] 384 | pub fn read(&self) -> RwLockReadGuard<'_, D> { [INFO] [stderr] | +++ [INFO] [stderr] [INFO] [stderr] warning: `sharedstate` (lib) generated 10 warnings (run `cargo fix --lib -p sharedstate` to apply 4 suggestions) [INFO] [stderr] warning: unused import: `worker::sync_updater::FollowTarget` [INFO] [stderr] --> src/worker/sync_updater.rs:502:133 [INFO] [stderr] | [INFO] [stderr] 502 | ...TestState, TestStateAction}}, utils::PanicHelper, worker::sync_updater::FollowTarget}; [INFO] [stderr] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stderr] | [INFO] [stderr] = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default [INFO] [stderr] [INFO] [stderr] warning: unused import: `tokio::sync::mpsc::channel` [INFO] [stderr] --> src/testing/fuzzy_test.rs:4:5 [INFO] [stderr] | [INFO] [stderr] 4 | use tokio::sync::mpsc::channel; [INFO] [stderr] | ^^^^^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stderr] [INFO] [stderr] warning: field `rate_limit_bps` is never read [INFO] [stderr] --> src/testing/test_sync_io.rs:99:5 [INFO] [stderr] | [INFO] [stderr] 96 | struct ConnManage { [INFO] [stderr] | ---------- field in this struct [INFO] [stderr] ... [INFO] [stderr] 99 | rate_limit_bps: Arc, [INFO] [stderr] | ^^^^^^^^^^^^^^ [INFO] [stderr] [INFO] [stderr] warning: `sharedstate` (lib test) generated 13 warnings (10 duplicates) (run `cargo fix --lib -p sharedstate --tests` to apply 2 suggestions) [INFO] [stderr] Finished `test` profile [unoptimized + debuginfo] target(s) in 0.15s [INFO] [stderr] Running unittests src/lib.rs (/opt/rustwide/target/debug/deps/sharedstate-ee8aadbf8182d277) [INFO] [stdout] [INFO] [stdout] running 16 tests [INFO] [stdout] test recoverable_state::test::recoverable_state_encoding_test ... ok [INFO] [stdout] test state::test::state_flush_tests ... ok [INFO] [stdout] test testing::state_tests::test_state_action_encoding_test ... ok [INFO] [stdout] test testing::message_io_tests::message_io_send_receive ... ok [INFO] [stdout] test testing::message_io_tests::message_io_zero_send ... ok [INFO] [stdout] test net::message_io::test::message_io_test ... ok [INFO] [stdout] test recoverable_state::test::state_recovery_check_test ... ok [INFO] [stdout] test worker::sync_updater::test::sync_updater_recover_state_fork_same_seq_reject_test ... ok [INFO] [stdout] test worker::sync_updater::test::sync_updater_recover_state_fork_reject_test ... ok [INFO] [stdout] test worker::sync_updater::test::sync_updater_recover_state_test ... ok [INFO] [stdout] test worker::sync_updater::test::sync_updater_simple_follower_test ... ok [INFO] [stdout] test worker::sync_updater::test::sync_updater_simple_leader_test ... ok [INFO] [stdout] test worker::sync_manager::test::sync_manager_connects_through_peer_test ... ok [INFO] [stdout] test testing::state_tests::slow_state_deadlock_test ... FAILED [INFO] [stdout] test worker::sync_manager::test::sync_manager_doesnt_allow_peer_with_different_leader_test ... ok [INFO] [stdout] test testing::fuzzy_test::fuzzy_test has been running for over 60 seconds [INFO] [stdout] test testing::fuzzy_test::fuzzy_test ... FAILED [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] [INFO] [stdout] ---- testing::state_tests::slow_state_deadlock_test stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'testing::state_tests::slow_state_deadlock_test' (24) panicked at src/testing/state_tests.rs:215:5: [INFO] [stdout] assertion failed: !failed.load(Ordering::SeqCst) [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x622e8134e652 - std::backtrace_rs::backtrace::libunwind::trace::h589a96ef7638b383 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x622e8134e652 - std::backtrace_rs::backtrace::trace_unsynchronized::h23a30a7548c40de4 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x622e8134e652 - std::sys::backtrace::_print_fmt::h87dabd6535c8c07a [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x622e8134e652 - ::fmt::hffd20ad4e5eca8ab [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x622e8136007f - core::fmt::rt::Argument::fmt::h75c83e3fd5ce2419 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x622e8136007f - core::fmt::write::h6d9d3a7cfd7b84f5 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/fmt/mod.rs:1468:25 [INFO] [stdout] 6: 0x622e8131b7e3 - std::io::default_write_fmt::he11a713685e2464d [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x622e8131b7e3 - std::io::Write::write_fmt::h9d08f7e050bd2612 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x622e81327762 - std::sys::backtrace::BacktraceLock::print::hb28797143397220e [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x622e8132c47f - std::panicking::default_hook::{{closure}}::h7555113b62983743 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x622e8132c311 - std::panicking::default_hook::h2714b564abe8d914 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x622e811fb94e - as core::ops::function::Fn>::call::h0e9de8cceb22406e [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 12: 0x622e811fb94e - test::test_main_with_exit_callback::{{closure}}::h8125bd66cf4739d0 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x622e8132cbce - as core::ops::function::Fn>::call::h1fd0a0802eaec16b [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 14: 0x622e8132cbce - std::panicking::panic_with_hook::h190dc82263685ec5 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x622e8132c8b6 - std::panicking::panic_handler::{{closure}}::h8b036a1b366643ac [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:700:13 [INFO] [stdout] 16: 0x622e81327899 - std::sys::backtrace::__rust_end_short_backtrace::h45affcfc0c830da8 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x622e8130f9ed - __rustc[9b67c8562bba447b]::rust_begin_unwind [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x622e81368000 - core::panicking::panic_fmt::hc084f85b1e76c16d [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/panicking.rs:75:14 [INFO] [stdout] 19: 0x622e81367fdc - core::panicking::panic::h019d7f9cc862af37 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/panicking.rs:145:5 [INFO] [stdout] 20: 0x622e81039d3b - sharedstate::testing::state_tests::slow_state_deadlock_test::h6ac9f1e4e1772da2 [INFO] [stdout] at /opt/rustwide/workdir/src/testing/state_tests.rs:215:5 [INFO] [stdout] 21: 0x622e8103a447 - sharedstate::testing::state_tests::slow_state_deadlock_test::{{closure}}::ha62370b6700db6bf [INFO] [stdout] at /opt/rustwide/workdir/src/testing/state_tests.rs:150:30 [INFO] [stdout] 22: 0x622e80fe9996 - core::ops::function::FnOnce::call_once::h9d5604c282174f80 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/ops/function.rs:253:5 [INFO] [stdout] 23: 0x622e811fb70b - core::ops::function::FnOnce::call_once::hdbf42be2a49fb464 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/ops/function.rs:253:5 [INFO] [stdout] 24: 0x622e811fb70b - test::__rust_begin_short_backtrace::h8ae08814d38cb356 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/test/src/lib.rs:663:18 [INFO] [stdout] 25: 0x622e812111f5 - test::run_test_in_process::{{closure}}::h54574ee4f414d690 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/test/src/lib.rs:686:74 [INFO] [stdout] 26: 0x622e812111f5 - as core::ops::function::FnOnce<()>>::call_once::h01a0b7e7aad1a501 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/panic/unwind_safe.rs:272:9 [INFO] [stdout] 27: 0x622e812111f5 - std::panicking::catch_unwind::do_call::h23820e817c5612cd [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:590:40 [INFO] [stdout] 28: 0x622e812111f5 - std::panicking::catch_unwind::h89c73a1febce5587 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:553:19 [INFO] [stdout] 29: 0x622e812111f5 - std::panic::catch_unwind::h6f55caf410861914 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panic.rs:359:14 [INFO] [stdout] 30: 0x622e812111f5 - test::run_test_in_process::h4db851878e5d3983 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/test/src/lib.rs:686:27 [INFO] [stdout] 31: 0x622e812111f5 - test::run_test::{{closure}}::h57e8c5acaceacd24 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/test/src/lib.rs:607:43 [INFO] [stdout] 32: 0x622e811e7be4 - test::run_test::{{closure}}::he775ccc3a9d3b97e [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/test/src/lib.rs:637:41 [INFO] [stdout] 33: 0x622e811e7be4 - std::sys::backtrace::__rust_begin_short_backtrace::hcd94d85d7765225a [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 34: 0x622e811eb42a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::ha814de8393f07830 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 35: 0x622e811eb42a - as core::ops::function::FnOnce<()>>::call_once::h8deb00af0abdeaf1 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/panic/unwind_safe.rs:272:9 [INFO] [stdout] 36: 0x622e811eb42a - std::panicking::catch_unwind::do_call::h1917979b0c279ea2 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:590:40 [INFO] [stdout] 37: 0x622e811eb42a - std::panicking::catch_unwind::h791ad91fee6ce34e [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:553:19 [INFO] [stdout] 38: 0x622e811eb42a - std::panic::catch_unwind::h982b8f40ed9c3cce [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panic.rs:359:14 [INFO] [stdout] 39: 0x622e811eb42a - std::thread::Builder::spawn_unchecked_::{{closure}}::h545e54700c945db4 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 40: 0x622e811eb42a - core::ops::function::FnOnce::call_once{{vtable.shim}}::hb427481a5227d78a [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/ops/function.rs:253:5 [INFO] [stdout] 41: 0x622e81321b1f - as core::ops::function::FnOnce>::call_once::h1d452951a81e9d0a [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 42: 0x622e81321b1f - std::sys::pal::unix::thread::Thread::new::thread_start::h29c84e109630689a [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/sys/pal/unix/thread.rs:118:17 [INFO] [stdout] 43: 0x7bfdf2a7caa4 - [INFO] [stdout] 44: 0x7bfdf2b09a34 - clone [INFO] [stdout] 45: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- testing::fuzzy_test::fuzzy_test stdout ---- [INFO] [stdout] 2025-09-01T11:54:46.434742Z INFO Node{id=1}: sequenced_broadcast: 20|SequencedBroadcastWorker Started id=20 [INFO] [stdout] 2025-09-01T11:54:46.449322Z INFO Node{id=2}: sequenced_broadcast: 34|SequencedBroadcastWorker Started id=34 [INFO] [stdout] 2025-09-01T11:54:46.449394Z INFO Node{id=3}: sequenced_broadcast: 35|SequencedBroadcastWorker Started id=35 [INFO] [stdout] 2025-09-01T11:54:46.449506Z INFO Node{id=4}: sequenced_broadcast: 36|SequencedBroadcastWorker Started id=36 [INFO] [stdout] 2025-09-01T11:54:46.449600Z INFO Node{id=5}: sequenced_broadcast: 37|SequencedBroadcastWorker Started id=37 [INFO] [stdout] 2025-09-01T11:54:46.449662Z INFO Node{id=6}: sequenced_broadcast: 38|SequencedBroadcastWorker Started id=38 [INFO] [stdout] 2025-09-01T11:54:46.449715Z INFO Node{id=7}: sequenced_broadcast: 39|SequencedBroadcastWorker Started id=39 [INFO] [stdout] 2025-09-01T11:54:46.455477Z INFO Node{id=8}: sequenced_broadcast: 40|SequencedBroadcastWorker Started id=40 [INFO] [stdout] 2025-09-01T11:54:46.455598Z INFO Node{id=9}: sequenced_broadcast: 41|SequencedBroadcastWorker Started id=41 [INFO] [stdout] 2025-09-01T11:54:46.455684Z INFO Node{id=10}: sequenced_broadcast: 42|SequencedBroadcastWorker Started id=42 [INFO] [stdout] 2025-09-01T11:54:46.455774Z INFO Node{id=11}: sequenced_broadcast: 43|SequencedBroadcastWorker Started id=43 [INFO] [stdout] 2025-09-01T11:54:46.458884Z INFO Node{id=12}: sequenced_broadcast: 44|SequencedBroadcastWorker Started id=44 [INFO] [stdout] 2025-09-01T11:54:46.458947Z INFO Node{id=13}: sequenced_broadcast: 45|SequencedBroadcastWorker Started id=45 [INFO] [stdout] 2025-09-01T11:54:46.459025Z INFO Node{id=14}: sequenced_broadcast: 46|SequencedBroadcastWorker Started id=46 [INFO] [stdout] 2025-09-01T11:54:46.459090Z INFO Node{id=15}: sequenced_broadcast: 47|SequencedBroadcastWorker Started id=47 [INFO] [stdout] 2025-09-01T11:54:46.459143Z INFO Node{id=16}: sequenced_broadcast: 48|SequencedBroadcastWorker Started id=48 [INFO] [stdout] 2025-09-01T11:54:46.459258Z INFO Node{id=17}: sequenced_broadcast: 49|SequencedBroadcastWorker Started id=49 [INFO] [stdout] 2025-09-01T11:54:46.459317Z INFO Node{id=18}: sequenced_broadcast: 50|SequencedBroadcastWorker Started id=50 [INFO] [stdout] 2025-09-01T11:54:46.459365Z INFO Node{id=19}: sequenced_broadcast: 51|SequencedBroadcastWorker Started id=51 [INFO] [stdout] 2025-09-01T11:54:46.459415Z INFO Node{id=20}: sequenced_broadcast: 52|SequencedBroadcastWorker Started id=52 [INFO] [stdout] 2025-09-01T11:54:46.459469Z INFO Node{id=21}: sequenced_broadcast: 53|SequencedBroadcastWorker Started id=53 [INFO] [stdout] 2025-09-01T11:54:46.459529Z INFO Node{id=22}: sequenced_broadcast: 54|SequencedBroadcastWorker Started id=54 [INFO] [stdout] 2025-09-01T11:54:46.459592Z INFO Node{id=23}: sequenced_broadcast: 55|SequencedBroadcastWorker Started id=55 [INFO] [stdout] 2025-09-01T11:54:46.459664Z INFO Node{id=24}: sequenced_broadcast: 56|SequencedBroadcastWorker Started id=56 [INFO] [stdout] 2025-09-01T11:54:46.459721Z INFO Node{id=25}: sequenced_broadcast: 57|SequencedBroadcastWorker Started id=57 [INFO] [stdout] 2025-09-01T11:54:46.467882Z INFO Node{id=26}: sequenced_broadcast: 58|SequencedBroadcastWorker Started id=58 [INFO] [stdout] 2025-09-01T11:54:46.467977Z INFO Node{id=27}: sequenced_broadcast: 59|SequencedBroadcastWorker Started id=59 [INFO] [stdout] 2025-09-01T11:54:46.468033Z INFO Node{id=28}: sequenced_broadcast: 60|SequencedBroadcastWorker Started id=60 [INFO] [stdout] 2025-09-01T11:54:46.468084Z INFO Node{id=29}: sequenced_broadcast: 61|SequencedBroadcastWorker Started id=61 [INFO] [stdout] 2025-09-01T11:54:46.468136Z INFO Node{id=30}: sequenced_broadcast: 62|SequencedBroadcastWorker Started id=62 [INFO] [stdout] 2025-09-01T11:54:46.468185Z INFO Node{id=31}: sequenced_broadcast: 63|SequencedBroadcastWorker Started id=63 [INFO] [stdout] 2025-09-01T11:54:46.468308Z INFO Node{id=32}: sequenced_broadcast: 64|SequencedBroadcastWorker Started id=64 [INFO] [stdout] 2025-09-01T11:54:46.468500Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.468636Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.468776Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.468853Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.468911Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.468966Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469036Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469061Z INFO Node{id=1}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:54:46.469123Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469174Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469225Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469296Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469360Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469419Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469473Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469533Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469578Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469625Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469674Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.469722Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.487828Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.487915Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.487984Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.488041Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.488250Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.488313Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.488370Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.488428Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.488531Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.488617Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.488680Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.488844Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.488903Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:54:46.518107Z INFO Node{id=7}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518182Z INFO Node{id=6}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518208Z INFO Node{id=5}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518233Z INFO Node{id=4}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518257Z INFO Node{id=3}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518280Z INFO Node{id=2}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518307Z INFO Node{id=15}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518332Z INFO Node{id=14}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518355Z INFO Node{id=13}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518395Z INFO Node{id=12}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518422Z INFO Node{id=11}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518446Z INFO Node{id=10}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518468Z INFO Node{id=9}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518490Z INFO Node{id=8}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518514Z INFO Node{id=23}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518538Z INFO Node{id=22}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518572Z INFO Node{id=21}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518593Z INFO Node{id=20}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518614Z INFO Node{id=19}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518637Z INFO Node{id=18}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.518659Z INFO Node{id=17}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.527106Z INFO Node{id=16}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.527154Z INFO Node{id=30}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.527179Z INFO Node{id=29}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.527202Z INFO Node{id=28}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.527225Z INFO Node{id=27}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.527248Z INFO Node{id=26}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.527270Z INFO Node{id=25}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.527297Z INFO Node{id=24}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.527321Z INFO Node{id=32}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.527345Z INFO Node{id=31}: sharedstate::worker::sync_manager: need leader connection [INFO] [stdout] 2025-09-01T11:54:46.527492Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.527510Z INFO Node{id=17}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.527521Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.527628Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.527645Z INFO Node{id=18}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.527652Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.527690Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.527702Z INFO Node{id=19}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.527709Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.527742Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.527752Z INFO Node{id=20}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.544833Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.544950Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.544969Z INFO Node{id=21}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.544977Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545018Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545029Z INFO Node{id=22}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545036Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545072Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545092Z INFO Node{id=23}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545100Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545142Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545156Z INFO Node{id=8}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545164Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545206Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545219Z INFO Node{id=9}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545226Z INFO Node{id=9}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545265Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545278Z INFO Node{id=10}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545320Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545368Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545381Z INFO Node{id=11}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545388Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545431Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545443Z INFO Node{id=12}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545450Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545489Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545510Z INFO Node{id=13}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545517Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545608Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545635Z INFO Node{id=14}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545643Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545701Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545719Z INFO Node{id=15}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545728Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545790Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545806Z INFO Node{id=2}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545815Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545865Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545879Z INFO Node{id=3}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545887Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545930Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.545942Z INFO Node{id=4}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.545950Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.545998Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.546013Z INFO Node{id=5}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.546022Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.546064Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.546078Z INFO Node{id=6}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.546087Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.546129Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.546143Z INFO Node{id=7}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.546152Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.546316Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.546335Z INFO Node{id=31}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.546343Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.546386Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.546397Z INFO Node{id=32}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.546405Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.546438Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.546448Z INFO Node{id=24}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.546460Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.546494Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.546505Z INFO Node{id=25}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.546512Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.546560Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.546574Z INFO Node{id=26}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.546582Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.546620Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.546632Z INFO Node{id=27}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.546640Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.546682Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.546694Z INFO Node{id=28}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.546701Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.546739Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.546750Z INFO Node{id=29}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.558613Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.558734Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.558750Z INFO Node{id=30}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.558776Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.558943Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:46.558961Z INFO Node{id=16}: sharedstate::worker::sync_manager: peer info missing for leader so use as default [INFO] [stdout] 2025-09-01T11:54:46.558968Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=0 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:46.559088Z INFO Node{id=17}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.559158Z INFO Node{id=17}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.559204Z INFO Node{id=18}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.559260Z INFO Node{id=18}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.559300Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.559363Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.559408Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.559465Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.559504Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.559595Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.559636Z INFO Node{id=22}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.559689Z INFO Node{id=22}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.559726Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.563846Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.563928Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.564005Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.564059Z INFO Node{id=9}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.564133Z INFO Node{id=9}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.564181Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.564247Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.564296Z INFO Node{id=11}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.564371Z INFO Node{id=11}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.564424Z INFO Node{id=12}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.564507Z INFO Node{id=12}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.564577Z INFO Node{id=13}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.564653Z INFO Node{id=13}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.564704Z INFO Node{id=14}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.573595Z INFO Node{id=14}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.573724Z INFO Node{id=15}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.573832Z INFO Node{id=15}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.573897Z INFO Node{id=2}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.573990Z INFO Node{id=2}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.574047Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.574173Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.574223Z INFO Node{id=4}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.574293Z INFO Node{id=4}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.574342Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.574411Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.574460Z INFO Node{id=6}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.574569Z INFO Node{id=6}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.574628Z INFO Node{id=7}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.574722Z INFO Node{id=7}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.583895Z INFO Node{id=31}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.584037Z INFO Node{id=31}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.584156Z INFO Node{id=32}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.584214Z INFO Node{id=32}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.584266Z INFO Node{id=24}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.584334Z INFO Node{id=24}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.584385Z INFO Node{id=25}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.584479Z INFO Node{id=25}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.584564Z INFO Node{id=26}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.584650Z INFO Node{id=26}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.584710Z INFO Node{id=27}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.591917Z INFO Node{id=27}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.592021Z INFO Node{id=28}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.592095Z INFO Node{id=28}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.592167Z INFO Node{id=29}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.592226Z INFO Node{id=29}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.592265Z INFO Node{id=30}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.592329Z INFO Node{id=30}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.608102Z INFO Node{id=16}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:46.608228Z INFO Node{id=16}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:46.608279Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:54:46.609347Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:54:46.618178Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:54:46.632474Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:54:46.639825Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:54:46.640397Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:54:46.640530Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727686559) } [INFO] [stdout] 2025-09-01T11:54:46.640565Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=18 [INFO] [stdout] 2025-09-01T11:54:46.654097Z INFO Node{id=18}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=95 [INFO] [stdout] 2025-09-01T11:54:46.654148Z INFO Node{id=18}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.654695Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727686559) } [INFO] [stdout] 2025-09-01T11:54:46.654726Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=17 [INFO] [stdout] 2025-09-01T11:54:46.654824Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:54:46.655329Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:54:46.655375Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:46.655672Z INFO Node{id=17}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=96 [INFO] [stdout] 2025-09-01T11:54:46.655698Z INFO Node{id=17}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.677082Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:46.677654Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727686559) } [INFO] [stdout] 2025-09-01T11:54:46.677680Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=19 [INFO] [stdout] 2025-09-01T11:54:46.677728Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=9 [INFO] [stdout] 2025-09-01T11:54:46.678171Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:54:46.678212Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:46.678436Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=119 [INFO] [stdout] 2025-09-01T11:54:46.678459Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.678656Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:46.695130Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727686559) } [INFO] [stdout] 2025-09-01T11:54:46.695191Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=22 [INFO] [stdout] 2025-09-01T11:54:46.695299Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:54:46.701032Z INFO Node{id=22}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=142 [INFO] [stdout] 2025-09-01T11:54:46.701072Z INFO Node{id=22}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.701441Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:54:46.701491Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727686559) } [INFO] [stdout] 2025-09-01T11:54:46.701509Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=21 [INFO] [stdout] 2025-09-01T11:54:46.702023Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727686559) } [INFO] [stdout] 2025-09-01T11:54:46.702043Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=20 [INFO] [stdout] 2025-09-01T11:54:46.702077Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:54:46.702300Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=143 [INFO] [stdout] 2025-09-01T11:54:46.702324Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.702426Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=143 [INFO] [stdout] 2025-09-01T11:54:46.702448Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.702663Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:54:46.702707Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727686563) } [INFO] [stdout] 2025-09-01T11:54:46.702721Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=23 [INFO] [stdout] 2025-09-01T11:54:46.719611Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=156 [INFO] [stdout] 2025-09-01T11:54:46.719648Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.722196Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727686564) } [INFO] [stdout] 2025-09-01T11:54:46.722230Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=8 [INFO] [stdout] 2025-09-01T11:54:46.722271Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:54:46.722608Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:54:46.722658Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:46.722851Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=158 [INFO] [stdout] 2025-09-01T11:54:46.722876Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.722951Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:46.723135Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:46.723174Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:54:46.723512Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:46.723679Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:54:46.723726Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SubscribeRecovery(RecoverableStateDetails { id: 18228676017188899823, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:46.724086Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:46.724150Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:54:46.724601Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:46.750027Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:54:46.750102Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:46.750510Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:46.759113Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeRecovery(RecoverableStateDetails { id: 12636613181629258007, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:46.759198Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:54:46.759754Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:54:46.768918Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727686564) } [INFO] [stdout] 2025-09-01T11:54:46.768939Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=10 [INFO] [stdout] 2025-09-01T11:54:46.769412Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: Ping(1756727686564) } [INFO] [stdout] 2025-09-01T11:54:46.769440Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=9 [INFO] [stdout] 2025-09-01T11:54:46.769484Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:54:46.769821Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:54:46.769867Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:46.769952Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=205 [INFO] [stdout] 2025-09-01T11:54:46.770003Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.770226Z INFO Node{id=9}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=206 [INFO] [stdout] 2025-09-01T11:54:46.770253Z INFO Node{id=9}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.770465Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:46.770637Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727686564) } [INFO] [stdout] 2025-09-01T11:54:46.770654Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=11 [INFO] [stdout] 2025-09-01T11:54:46.770683Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:54:46.788314Z INFO Node{id=11}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=224 [INFO] [stdout] 2025-09-01T11:54:46.788362Z INFO Node{id=11}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.796143Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:54:46.796231Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:46.802635Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:46.802728Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:54:46.810716Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:54:46.816203Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:46.816447Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:46.816996Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727686573) } [INFO] [stdout] 2025-09-01T11:54:46.817018Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=14 [INFO] [stdout] 2025-09-01T11:54:46.817062Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:54:46.817584Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:54:46.817627Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727686564) } [INFO] [stdout] 2025-09-01T11:54:46.817640Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=13 [INFO] [stdout] 2025-09-01T11:54:46.817821Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:46.818100Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:46.818145Z INFO Node{id=14}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=245 [INFO] [stdout] 2025-09-01T11:54:46.832477Z INFO Node{id=14}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.837121Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727686564) } [INFO] [stdout] 2025-09-01T11:54:46.837163Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=12 [INFO] [stdout] 2025-09-01T11:54:46.837222Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:54:46.837631Z INFO Node{id=13}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=273 [INFO] [stdout] 2025-09-01T11:54:46.837662Z INFO Node{id=13}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.849336Z INFO Node{id=12}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=285 [INFO] [stdout] 2025-09-01T11:54:46.849375Z INFO Node{id=12}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.850004Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:46.850749Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(16) < seq(18) [INFO] [stdout] 2025-09-01T11:54:46.875411Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber(1): Added, allow_drop: true, next_sequence: 18, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:54:46.875699Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 18) [INFO] [stdout] 2025-09-01T11:54:46.878533Z INFO Node{id=18}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:46.878584Z INFO Node{id=18}: sequenced_broadcast: 69|SequencedBroadcastWorker Started id=69 [INFO] [stdout] 2025-09-01T11:54:46.878676Z INFO Node{id=18}: sequenced_broadcast: 50|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:46.878694Z INFO Node{id=18}: sequenced_broadcast: 50|SequencedBroadcastWorker Stopped id=50 elapsed=419.362592ms iter=2 [INFO] [stdout] 2025-09-01T11:54:46.878875Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727686573) } [INFO] [stdout] 2025-09-01T11:54:46.878925Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=2 [INFO] [stdout] 2025-09-01T11:54:46.879118Z INFO Node{id=2}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=306 [INFO] [stdout] 2025-09-01T11:54:46.879147Z INFO Node{id=2}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.879416Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727686573) } [INFO] [stdout] 2025-09-01T11:54:46.879438Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=15 [INFO] [stdout] 2025-09-01T11:54:46.879652Z INFO Node{id=15}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=306 [INFO] [stdout] 2025-09-01T11:54:46.879687Z INFO Node{id=15}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.880030Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:46.880295Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:46.880513Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727686574) } [INFO] [stdout] 2025-09-01T11:54:46.880537Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=3 [INFO] [stdout] 2025-09-01T11:54:46.880737Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=306 [INFO] [stdout] 2025-09-01T11:54:46.899826Z INFO Node{id=3}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.900721Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:46.901091Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeRecovery(RecoverableStateDetails { id: 12964949283331415817, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:46.901240Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:46.901460Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727686574) } [INFO] [stdout] 2025-09-01T11:54:46.901483Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=4 [INFO] [stdout] 2025-09-01T11:54:46.901776Z INFO Node{id=4}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=327 [INFO] [stdout] 2025-09-01T11:54:46.901815Z INFO Node{id=4}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.902072Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:46.902282Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber(2): Added, allow_drop: true, next_sequence: 23, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:54:46.902528Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 23) [INFO] [stdout] 2025-09-01T11:54:46.902653Z INFO Node{id=17}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:46.902682Z INFO Node{id=17}: sequenced_broadcast: 70|SequencedBroadcastWorker Started id=70 [INFO] [stdout] 2025-09-01T11:54:46.918592Z INFO Node{id=17}: sequenced_broadcast: 49|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:46.918634Z INFO Node{id=17}: sequenced_broadcast: 49|SequencedBroadcastWorker Stopped id=49 elapsed=459.358712ms iter=2 [INFO] [stdout] 2025-09-01T11:54:46.918963Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeRecovery(RecoverableStateDetails { id: 11977939416554638530, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:46.919794Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727686574) } [INFO] [stdout] 2025-09-01T11:54:46.919826Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=5 [INFO] [stdout] 2025-09-01T11:54:46.920180Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727686574) } [INFO] [stdout] 2025-09-01T11:54:46.920237Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=6 [INFO] [stdout] 2025-09-01T11:54:46.920535Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=346 [INFO] [stdout] 2025-09-01T11:54:46.920606Z INFO Node{id=5}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.920718Z INFO Node{id=6}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=346 [INFO] [stdout] 2025-09-01T11:54:46.920750Z INFO Node{id=6}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.938540Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:46.945391Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:46.945773Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:46.945886Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:46.946343Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:46.946865Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727686574) } [INFO] [stdout] 2025-09-01T11:54:46.946888Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=7 [INFO] [stdout] 2025-09-01T11:54:46.946980Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:46.947123Z INFO Node{id=7}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=373 [INFO] [stdout] 2025-09-01T11:54:46.947150Z INFO Node{id=7}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.947403Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:46.947725Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:46.976561Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727686584) } [INFO] [stdout] 2025-09-01T11:54:46.976605Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=24 [INFO] [stdout] 2025-09-01T11:54:46.977971Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727686584) } [INFO] [stdout] 2025-09-01T11:54:46.977994Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=32 [INFO] [stdout] 2025-09-01T11:54:46.978168Z INFO Node{id=24}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=394 [INFO] [stdout] 2025-09-01T11:54:46.978196Z INFO Node{id=24}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.978266Z INFO Node{id=32}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=394 [INFO] [stdout] 2025-09-01T11:54:46.978289Z INFO Node{id=32}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.978504Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727686584) } [INFO] [stdout] 2025-09-01T11:54:46.978523Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=31 [INFO] [stdout] 2025-09-01T11:54:46.980935Z INFO Node{id=31}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=396 [INFO] [stdout] 2025-09-01T11:54:46.980960Z INFO Node{id=31}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.981194Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727686584) } [INFO] [stdout] 2025-09-01T11:54:46.981214Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=25 [INFO] [stdout] 2025-09-01T11:54:46.981456Z INFO Node{id=25}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=397 [INFO] [stdout] 2025-09-01T11:54:46.981478Z INFO Node{id=25}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.981729Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeRecovery(RecoverableStateDetails { id: 15966214309367731050, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:46.982732Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:46.983000Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727686584) } [INFO] [stdout] 2025-09-01T11:54:46.983020Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=26 [INFO] [stdout] 2025-09-01T11:54:46.983170Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:46.983219Z INFO Node{id=26}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=399 [INFO] [stdout] 2025-09-01T11:54:46.983238Z INFO Node{id=26}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:46.983467Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:46.995398Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:46.995883Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeRecovery(RecoverableStateDetails { id: 9391859352936412299, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:46.996297Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:46.996594Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:47.011026Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727686591) } [INFO] [stdout] 2025-09-01T11:54:47.011068Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=27 [INFO] [stdout] 2025-09-01T11:54:47.011365Z INFO Node{id=27}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=420 [INFO] [stdout] 2025-09-01T11:54:47.011394Z INFO Node{id=27}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:47.012047Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.012389Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeRecovery(RecoverableStateDetails { id: 634985037238024698, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:47.012586Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.025068Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727686592) } [INFO] [stdout] 2025-09-01T11:54:47.025113Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=29 [INFO] [stdout] 2025-09-01T11:54:47.025573Z INFO Node{id=29}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=433 [INFO] [stdout] 2025-09-01T11:54:47.025606Z INFO Node{id=29}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:47.026365Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727686592) } [INFO] [stdout] 2025-09-01T11:54:47.026402Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=28 [INFO] [stdout] 2025-09-01T11:54:47.037605Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.039992Z INFO Node{id=28}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=447 [INFO] [stdout] 2025-09-01T11:54:47.040027Z INFO Node{id=28}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:47.040193Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.040785Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727686592) } [INFO] [stdout] 2025-09-01T11:54:47.040809Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=30 [INFO] [stdout] 2025-09-01T11:54:47.041185Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.041431Z INFO Node{id=30}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=449 [INFO] [stdout] 2025-09-01T11:54:47.041458Z INFO Node{id=30}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:47.041722Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.041938Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeRecovery(RecoverableStateDetails { id: 15928016797218250794, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:47.042243Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727686608) } [INFO] [stdout] 2025-09-01T11:54:47.042262Z INFO Node{id=1}: sharedstate::worker::sync_manager: Discovered Peer client_id=16 [INFO] [stdout] 2025-09-01T11:54:47.042501Z INFO Node{id=16}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=434 [INFO] [stdout] 2025-09-01T11:54:47.042523Z INFO Node{id=16}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:47.061539Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.061998Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.062647Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.072506Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.073514Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.085231Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.092211Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:47.092419Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(33) < seq(35) [INFO] [stdout] 2025-09-01T11:54:47.116043Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber(3): Added, allow_drop: true, next_sequence: 35, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:54:47.116733Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 35) [INFO] [stdout] 2025-09-01T11:54:47.127204Z INFO Node{id=19}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:47.127598Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.127704Z INFO Node{id=19}: sequenced_broadcast: 71|SequencedBroadcastWorker Started id=71 [INFO] [stdout] 2025-09-01T11:54:47.127984Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.128551Z INFO Node{id=19}: sequenced_broadcast: 51|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:47.128569Z INFO Node{id=19}: sequenced_broadcast: 51|SequencedBroadcastWorker Stopped id=51 elapsed=669.190003ms iter=2 [INFO] [stdout] 2025-09-01T11:54:47.165126Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:47.165846Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(42) < seq(45) [INFO] [stdout] 2025-09-01T11:54:47.190714Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber(4): Added, allow_drop: true, next_sequence: 45, min_allowed_seq: 1 [INFO] [stdout] 2025-09-01T11:54:47.214792Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.218978Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.219437Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 45) [INFO] [stdout] 2025-09-01T11:54:47.219563Z INFO Node{id=22}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:47.220070Z INFO Node{id=22}: sequenced_broadcast: 72|SequencedBroadcastWorker Started id=72 [INFO] [stdout] 2025-09-01T11:54:47.220684Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.241124Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:47.241752Z INFO Node{id=22}: sequenced_broadcast: 54|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:47.241797Z INFO Node{id=22}: sequenced_broadcast: 54|SequencedBroadcastWorker Stopped id=54 elapsed=782.246486ms iter=2 [INFO] [stdout] 2025-09-01T11:54:47.242188Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.242632Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:47.255870Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:47.257110Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:47.276214Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.298396Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.298496Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeRecovery(RecoverableStateDetails { id: 15335810314635339748, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:47.298902Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeRecovery(RecoverableStateDetails { id: 1675523409979011756, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:47.318531Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:47.328791Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:47.350900Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeRecovery(RecoverableStateDetails { id: 13252726253926303898, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:47.352490Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.377922Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.387593Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.402855Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.408837Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.409040Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.423013Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.424316Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:47.424554Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.442736Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(74) < seq(78) [INFO] [stdout] 2025-09-01T11:54:47.494011Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(77) < seq(78) [INFO] [stdout] 2025-09-01T11:54:47.505251Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 78, max: 77 } [INFO] [stdout] 2025-09-01T11:54:47.505302Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:47.520841Z ERROR Node{id=20}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:47.521246Z WARN Node{id=20}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:47.521265Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:47.529392Z INFO Node{id=20}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:47.553911Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.601807Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.604986Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:47.606551Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(89) < seq(93) [INFO] [stdout] 2025-09-01T11:54:47.659872Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(92) < seq(93) [INFO] [stdout] 2025-09-01T11:54:47.660972Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 93, max: 92 } [INFO] [stdout] 2025-09-01T11:54:47.660996Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:47.690181Z ERROR Node{id=21}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:47.691588Z WARN Node{id=21}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:47.691609Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:47.705549Z INFO Node{id=21}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:47.706178Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.740929Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.774875Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:47.775362Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:54:47.776095Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(100) < seq(105) [INFO] [stdout] 2025-09-01T11:54:47.809852Z ERROR Node{id=10}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:47.810080Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(103) < seq(105) [INFO] [stdout] 2025-09-01T11:54:47.810416Z WARN Node{id=10}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:47.810433Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:47.811347Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 105, max: 103 } [INFO] [stdout] 2025-09-01T11:54:47.811384Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:47.811455Z INFO Node{id=10}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:47.821410Z ERROR Node{id=23}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:47.833250Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:47.833666Z WARN Node{id=23}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:47.833683Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:47.838422Z INFO Node{id=23}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:47.842832Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:47.843559Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeRecovery(RecoverableStateDetails { id: 637232509615178270, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:47.907160Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.922844Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.937170Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:47.961272Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:47.962045Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(120) < seq(127) [INFO] [stdout] 2025-09-01T11:54:47.962998Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:47.986196Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(122) < seq(127) [INFO] [stdout] 2025-09-01T11:54:47.987581Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 127, max: 122 } [INFO] [stdout] 2025-09-01T11:54:47.987610Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:48.002685Z ERROR Node{id=8}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:48.010732Z WARN Node{id=8}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:48.010785Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:48.011060Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.011674Z INFO Node{id=8}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:48.011972Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:48.012489Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeRecovery(RecoverableStateDetails { id: 7729601842349720361, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.027187Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.028618Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:48.029533Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:48.049137Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:54:48.049218Z INFO Node{id=1}: sharedstate::worker::sync_manager: Broadcasted 31 peers to 31 peers [INFO] [stdout] 2025-09-01T11:54:48.050149Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:48.066492Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.066553Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.066568Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.066577Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.066588Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.066596Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.066604Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.066612Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.066624Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.066633Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.066641Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.066650Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.066659Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.066666Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.066674Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.066693Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.066701Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.066710Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.066719Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.066727Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.066735Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.066744Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.066752Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.072040Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.072055Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.072065Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.072072Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.072079Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.072088Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.072109Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.072116Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.072122Z INFO Node{id=32}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.072268Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.072286Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.072295Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.072303Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.072311Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.072318Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.072325Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.072331Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.072341Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.072347Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.072354Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.072361Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.072368Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.072374Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.072381Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.072392Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.072399Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.072406Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.072412Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.072419Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.072425Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.072431Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.072438Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.072445Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.072452Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.072458Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.072465Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.072471Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.072478Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.072493Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.072500Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.072506Z INFO Node{id=16}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.081434Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.081482Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.081494Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.081503Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.081514Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.081530Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.081538Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.081554Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.081565Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.081572Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.081580Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.081588Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.081596Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.081603Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.081610Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.081623Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.081630Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.081637Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.081644Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.081651Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.081658Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.081665Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.081672Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.081680Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.081687Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.081695Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.081702Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.081710Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.081717Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.081783Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.081809Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.081818Z INFO Node{id=26}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.081853Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.081868Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.081876Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.081884Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.081892Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.081899Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.081907Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.081915Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.081926Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.081939Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.081947Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.081954Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.081962Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.081969Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.081977Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.081990Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.081998Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.082005Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.082012Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.082021Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.082028Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.082036Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.082043Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.082050Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.082057Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.082064Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.082072Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.082080Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.082088Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.082106Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.082114Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.082122Z INFO Node{id=29}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.082164Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.082179Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.082192Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.082199Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.082209Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.082217Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.082225Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.082233Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.082243Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.082250Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.082258Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.082266Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.082301Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.082310Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.082328Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.082342Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.082350Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.082357Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.082366Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.082373Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.082381Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.082388Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.082396Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.082404Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.082412Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.082420Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.082427Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.082435Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.082443Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.082460Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.082468Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.082476Z INFO Node{id=30}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.082511Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.082525Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.082534Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.082550Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.082559Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.082568Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.082575Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.082583Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.082594Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.082601Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.082620Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.082629Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.082636Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.082643Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.082656Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.082669Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.082677Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.082687Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.082695Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.082720Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.082729Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.082736Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.082748Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.082756Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.092747Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.092786Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.092797Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.092806Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.092814Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.092839Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.092850Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.092858Z INFO Node{id=2}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.092942Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.092962Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.092970Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.092978Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.092988Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.092996Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.093003Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.093010Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.093022Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.093030Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.093038Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.093046Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.093054Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.093063Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.093072Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.093086Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.093094Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.093102Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.093110Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.093118Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.093126Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.093134Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.093142Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.093150Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.093166Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.093175Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.093183Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.093190Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.093197Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.093216Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.093225Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.093233Z INFO Node{id=14}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.093272Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.093291Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.093302Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.093311Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.093321Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.093329Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.093337Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.093345Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.093355Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.093364Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.093372Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.093379Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.093387Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.093395Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.093403Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.093417Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.093426Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.093435Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.093443Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.093451Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.093460Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.093468Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.093476Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.093483Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.093490Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.093497Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.093504Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.093512Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.093519Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.093552Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.093563Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.093571Z INFO Node{id=12}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.093606Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.093621Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.093630Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.093638Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.093648Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.093655Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.093663Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.093671Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.093682Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.093690Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.093698Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.093705Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.093713Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.093721Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.093729Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.093741Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.102831Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.102857Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.102866Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.102875Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.102882Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.102890Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.102897Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.102904Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.102912Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.102919Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.102927Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.102934Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.102941Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.102963Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.102971Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.102979Z INFO Node{id=3}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.103053Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.103071Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.103080Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.103089Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.103099Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.103107Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.103114Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.103122Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.103132Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.103140Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.103147Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.103155Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.103162Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.103169Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.103177Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.103190Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.103198Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.103205Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.103212Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.103219Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.103227Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.103234Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.103241Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.103249Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.103256Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.103264Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.103271Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.103280Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.103288Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.103318Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.103329Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.103338Z INFO Node{id=4}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.103373Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.103388Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.103397Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.103405Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.103415Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.103505Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.103515Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.103522Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.103532Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.103540Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.103554Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.103562Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.103570Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.103579Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.103587Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.103601Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.103609Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.103617Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.103625Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.103633Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.103641Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.103649Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.103657Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.103665Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.103673Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.103681Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.103689Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.103696Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.103705Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.103723Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.103734Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.103742Z INFO Node{id=7}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.103801Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.103818Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.103827Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.103835Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.103844Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.103852Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.103859Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.103867Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.103876Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.103884Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.103891Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.103899Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.103907Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.103915Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.103923Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.103936Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.103944Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.103952Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.103959Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.103966Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.103974Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.103981Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.103988Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.103995Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.104003Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.104010Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.104018Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.104025Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.104032Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.104050Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.104057Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.104064Z INFO Node{id=25}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.104092Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.104104Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.104112Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.104125Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.104135Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.104143Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.104151Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.104159Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.104170Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.104179Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.104187Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.104195Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.104203Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.104211Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.104220Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.104233Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.104241Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.104249Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.104257Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.104264Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.104271Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.104279Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.104287Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.104295Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.104303Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.104310Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.104318Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.104326Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.104333Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.104353Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.104362Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.104369Z INFO Node{id=31}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.104405Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.104422Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.104431Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.104440Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.104451Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.104459Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.104467Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.104475Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.104492Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.104501Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.104509Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.104517Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.104525Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.104533Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.104541Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.104563Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.104571Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.104578Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.104586Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.104594Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.104601Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.104609Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.104616Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.104624Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.104631Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.104639Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.104647Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.104655Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.104663Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.104683Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.104690Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.104698Z INFO Node{id=13}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.104735Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.104749Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.117816Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.117856Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.117871Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.117880Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.117888Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.117896Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.117907Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.117915Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.117923Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.117931Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.117938Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.117955Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.117963Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.117978Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.117986Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.117993Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.118000Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.118007Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.118016Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.118023Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.118031Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.118039Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.118047Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.118055Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.118063Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.118071Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.118079Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.118102Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.118111Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.118119Z INFO Node{id=28}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.118177Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.118197Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.118207Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.118215Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.118226Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.118233Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.118241Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.118249Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.118260Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.118268Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.118276Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.118283Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.118291Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.118300Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.118308Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.118321Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.118329Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.118337Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.118360Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.118370Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.118377Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.118385Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.118402Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.118409Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.118417Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.118425Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.118437Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.118445Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.118453Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.118474Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.118483Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.118494Z INFO Node{id=6}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.119342Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.119384Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.119402Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.119411Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.119421Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.119437Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.119446Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.119453Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.119463Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.119485Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.119493Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.119500Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.119507Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.119531Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.119538Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.119559Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.119566Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.119575Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.119582Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.119589Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.119596Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.119604Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.119612Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.119624Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.119631Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.119638Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.119645Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.119652Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.119659Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.119677Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.119685Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.119692Z INFO Node{id=27}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.119720Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.119733Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.119741Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.119748Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.119773Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.119783Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.119790Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.119800Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.119810Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.119818Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.119825Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.119833Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.119841Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.119848Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.119855Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.119867Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.119874Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.119881Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.119889Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.119896Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.119909Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.119918Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.119925Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.119932Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.119939Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.119950Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.119957Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.119965Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.119976Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.119998Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.120007Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.120015Z INFO Node{id=5}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.120049Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.120062Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.120069Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.120077Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.120086Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.120093Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.120100Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.120107Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.120116Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.120123Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.120130Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.120138Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.120145Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.120152Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.120159Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.120171Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.120179Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.120186Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.120193Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.120201Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.120208Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.120216Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.120223Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.120230Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.120238Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.120245Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.120252Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.120259Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.120266Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.120283Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.120290Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.120299Z INFO Node{id=11}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.120329Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.120348Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.120356Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.120364Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.120373Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.120381Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.120388Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.120395Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.120406Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.120414Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.120423Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.120431Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.120438Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.120445Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.120452Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.120463Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.120471Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.120478Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.120485Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.120492Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.120500Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.120507Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.120514Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.120522Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.120530Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.120537Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.120553Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.120561Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.120568Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.120588Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.120595Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.120603Z INFO Node{id=9}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.120631Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.120643Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.120650Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.120657Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.120666Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.120674Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.120687Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.120694Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.120704Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.120712Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.120719Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.120727Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.120734Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.120741Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.120749Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.137347Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.137400Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.137409Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.137417Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.137432Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.137440Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.137448Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.137456Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.137469Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.137478Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.137487Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.137495Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.137504Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.137512Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.137532Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.137549Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.137557Z INFO Node{id=24}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.137618Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.137637Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.137647Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.137655Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.137666Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.137674Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.137682Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.137689Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.137700Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.137708Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.137727Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.137737Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.137745Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.137753Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.137784Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.137798Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.137807Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.137816Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.137823Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.137832Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.137840Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.137849Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.137857Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.137865Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.137874Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.137882Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.137891Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.137899Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.137908Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.137928Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.137937Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.137945Z INFO Node{id=15}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.139930Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.140663Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.140685Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.140695Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.140702Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.140711Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.140718Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.140725Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.140732Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.140741Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.140748Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.140755Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.164193Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.164204Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.164211Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.164226Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.164245Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.164253Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.164260Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.164267Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.164274Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.164281Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.164288Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.164296Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.164304Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.164311Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.164318Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.164326Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.164334Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.164342Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.164363Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.164372Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.164380Z INFO Node{id=18}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.165047Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.177146Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.177502Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeRecovery(RecoverableStateDetails { id: 3338079291511305984, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.192097Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.200495Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.200552Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.200569Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.200579Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.200591Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.200600Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.200609Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.200617Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.200629Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.200638Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.200646Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.200654Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.200662Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.200679Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.200688Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.200703Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.200712Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.200720Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.200728Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.200736Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.200744Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.200752Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.200780Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.200788Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.200795Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.200802Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.200809Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.200817Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.200824Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.200843Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.200850Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.200857Z INFO Node{id=17}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.214879Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.214931Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.214977Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.214987Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.215002Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.215011Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.215019Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.215028Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.215040Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.215049Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.215057Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.215064Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.215071Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.215079Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.215094Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.215108Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.215116Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.215128Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.215145Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.215153Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.215161Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.215169Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.215178Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.215186Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.215194Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.215202Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.215210Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.215218Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.215226Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.215244Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.215253Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.215262Z INFO Node{id=22}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.215444Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.238589Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.239360Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeRecovery(RecoverableStateDetails { id: 6495069325016322389, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.239445Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:54:48.239466Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 20 [INFO] [stdout] 2025-09-01T11:54:48.239479Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 26 [INFO] [stdout] 2025-09-01T11:54:48.239488Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 29 [INFO] [stdout] 2025-09-01T11:54:48.239499Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 32 [INFO] [stdout] 2025-09-01T11:54:48.239507Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 8 [INFO] [stdout] 2025-09-01T11:54:48.239514Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 10 [INFO] [stdout] 2025-09-01T11:54:48.239521Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 30 [INFO] [stdout] 2025-09-01T11:54:48.239530Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 17 [INFO] [stdout] 2025-09-01T11:54:48.239538Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 2 [INFO] [stdout] 2025-09-01T11:54:48.239559Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 14 [INFO] [stdout] 2025-09-01T11:54:48.239568Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 12 [INFO] [stdout] 2025-09-01T11:54:48.239575Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 3 [INFO] [stdout] 2025-09-01T11:54:48.239582Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 4 [INFO] [stdout] 2025-09-01T11:54:48.239590Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 7 [INFO] [stdout] 2025-09-01T11:54:48.239606Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 25 [INFO] [stdout] 2025-09-01T11:54:48.239614Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 23 [INFO] [stdout] 2025-09-01T11:54:48.239622Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 31 [INFO] [stdout] 2025-09-01T11:54:48.239641Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 16 [INFO] [stdout] 2025-09-01T11:54:48.239650Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 13 [INFO] [stdout] 2025-09-01T11:54:48.239658Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 19 [INFO] [stdout] 2025-09-01T11:54:48.239667Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 28 [INFO] [stdout] 2025-09-01T11:54:48.239675Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 6 [INFO] [stdout] 2025-09-01T11:54:48.239684Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 27 [INFO] [stdout] 2025-09-01T11:54:48.239692Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 18 [INFO] [stdout] 2025-09-01T11:54:48.239700Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 21 [INFO] [stdout] 2025-09-01T11:54:48.239708Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 5 [INFO] [stdout] 2025-09-01T11:54:48.239715Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 11 [INFO] [stdout] 2025-09-01T11:54:48.239723Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 22 [INFO] [stdout] 2025-09-01T11:54:48.239741Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 9 [INFO] [stdout] 2025-09-01T11:54:48.239750Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 24 [INFO] [stdout] 2025-09-01T11:54:48.239777Z INFO Node{id=19}: sharedstate::worker::sync_manager: Informed of new new peer: 15 [INFO] [stdout] 2025-09-01T11:54:48.279255Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeRecovery(RecoverableStateDetails { id: 1133459967848790104, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.298511Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.298987Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.299381Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeRecovery(RecoverableStateDetails { id: 259493686016308194, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.352461Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.353066Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeRecovery(RecoverableStateDetails { id: 3967747982285532032, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.353660Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.418460Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.424223Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.443326Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.445070Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.445745Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeRecovery(RecoverableStateDetails { id: 2956772458635842773, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.453653Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeRecovery(RecoverableStateDetails { id: 3001348910760355552, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.516713Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:48.517684Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:48.539559Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:48.539601Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=1 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:48.539734Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(200) < seq(213) [INFO] [stdout] 2025-09-01T11:54:48.540271Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:48.540342Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:48.540661Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:54:48.552532Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(203) < seq(213) [INFO] [stdout] 2025-09-01T11:54:48.552682Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 213, max: 203 } [INFO] [stdout] 2025-09-01T11:54:48.552702Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:48.563040Z ERROR Node{id=11}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:48.574988Z WARN Node{id=11}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:48.575022Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:48.575573Z INFO Node{id=11}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:48.733919Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:48.733956Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=1 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:48.739164Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:48.739247Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:48.740121Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.740377Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:54:48.753928Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeRecovery(RecoverableStateDetails { id: 15849201951862108824, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.754678Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.773266Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:48.773527Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(226) < seq(242) [INFO] [stdout] 2025-09-01T11:54:48.825523Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:48.825573Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=1 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:48.825667Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(232) < seq(242) [INFO] [stdout] 2025-09-01T11:54:48.827260Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:48.827346Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:48.827384Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 242, max: 232 } [INFO] [stdout] 2025-09-01T11:54:48.827408Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:48.827865Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:54:48.828563Z ERROR Node{id=9}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:48.832297Z WARN Node{id=9}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:48.832320Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:48.835974Z INFO Node{id=9}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:48.836534Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:48.836567Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=1 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:48.837207Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:48.837275Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:48.837698Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:54:48.863887Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.868564Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.873982Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SubscribeRecovery(RecoverableStateDetails { id: 4930385600716210758, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.893025Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.895900Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.896039Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.896434Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeRecovery(RecoverableStateDetails { id: 1864775603426703921, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.896807Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.908812Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.909176Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:48.911627Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SubscribeRecovery(RecoverableStateDetails { id: 4080161324815589637, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.920414Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeRecovery(RecoverableStateDetails { id: 14315515704630917242, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:48.996718Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:48.999008Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:49.008643Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeRecovery(RecoverableStateDetails { id: 13147879088706776850, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:49.012510Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:49.012544Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=0 peers=1 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:49.016939Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:49.017033Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:49.017308Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:54:49.083199Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:49.086336Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:49.093093Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeRecovery(RecoverableStateDetails { id: 13739817974832384570, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:49.150776Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:49.151037Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(373) < seq(391) [INFO] [stdout] 2025-09-01T11:54:49.167651Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(378) < seq(391) [INFO] [stdout] 2025-09-01T11:54:49.169302Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 391, max: 378 } [INFO] [stdout] 2025-09-01T11:54:49.169327Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:49.170430Z ERROR Node{id=14}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:49.175028Z WARN Node{id=14}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:49.175057Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:49.175352Z INFO Node{id=14}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:49.245164Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:49.246616Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:49.246752Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeRecovery(RecoverableStateDetails { id: 13528030838573203821, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:49.271733Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:49.278719Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:49.284096Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 16584620148520749951, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:49.297625Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:49.298117Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(447) < seq(458) [INFO] [stdout] 2025-09-01T11:54:49.311497Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(454) < seq(458) [INFO] [stdout] 2025-09-01T11:54:49.314229Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 458, max: 454 } [INFO] [stdout] 2025-09-01T11:54:49.314266Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:49.314599Z ERROR Node{id=13}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:49.314709Z WARN Node{id=13}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:49.314723Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:49.315202Z INFO Node{id=13}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:49.333930Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:49.334490Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:49.344642Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SubscribeRecovery(RecoverableStateDetails { id: 16571668472844679484, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:49.354612Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:49.355471Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeRecovery(RecoverableStateDetails { id: 1069456004009129247, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:49.355665Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:49.382461Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:49.382747Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(481) < seq(501) [INFO] [stdout] 2025-09-01T11:54:49.404667Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(490) < seq(501) [INFO] [stdout] 2025-09-01T11:54:49.411889Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 501, max: 490 } [INFO] [stdout] 2025-09-01T11:54:49.411932Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:49.416685Z ERROR Node{id=12}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:49.417357Z WARN Node{id=12}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:49.417378Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:49.425043Z INFO Node{id=12}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:49.490364Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:49.491499Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(506) < seq(523) [INFO] [stdout] 2025-09-01T11:54:49.520048Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(510) < seq(523) [INFO] [stdout] 2025-09-01T11:54:49.520590Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 523, max: 510 } [INFO] [stdout] 2025-09-01T11:54:49.520616Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:49.521637Z ERROR Node{id=2}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:49.524256Z WARN Node{id=2}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:49.524281Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:49.524732Z INFO Node{id=2}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:49.543087Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:49.544143Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(522) < seq(544) [INFO] [stdout] 2025-09-01T11:54:49.555066Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(529) < seq(544) [INFO] [stdout] 2025-09-01T11:54:49.555482Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 544, max: 529 } [INFO] [stdout] 2025-09-01T11:54:49.555507Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:49.555782Z ERROR Node{id=15}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:49.555859Z WARN Node{id=15}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:49.555872Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:49.555910Z INFO Node{id=15}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:49.579304Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:49.579354Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=4 leader=1 [INFO] [stdout] 2025-09-01T11:54:49.580197Z INFO Node{id=11}:connect{target=4}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:49.580268Z INFO Node{id=11}:connect{target=4}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:49.580580Z INFO Node{id=4}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:54:49.581429Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:49.581598Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(546) < seq(570) [INFO] [stdout] 2025-09-01T11:54:49.592596Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(550) < seq(570) [INFO] [stdout] 2025-09-01T11:54:49.598207Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727689580) } [INFO] [stdout] 2025-09-01T11:54:49.598266Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 570, max: 550 } [INFO] [stdout] 2025-09-01T11:54:49.598284Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:49.599360Z ERROR Node{id=3}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:49.599804Z INFO Node{id=11}:connect{target=4}: sharedstate::worker::sync_manager: got pong from peer latency=19 [INFO] [stdout] 2025-09-01T11:54:49.599832Z INFO Node{id=11}:connect{target=4}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:49.599863Z WARN Node{id=3}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:49.599873Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:49.600428Z INFO Node{id=3}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:49.603751Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:49.608257Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 4 } [INFO] [stdout] 2025-09-01T11:54:49.608291Z ERROR Node{id=11}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=4 leader=1 local=11 [INFO] [stdout] 2025-09-01T11:54:49.608900Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:49.608945Z INFO Node{id=11}:connect{target=4}:WritePeer{remote=4}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:49.609142Z INFO Node{id=11}:connect{target=4}:ReadPeer{remote=4}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:49.609464Z ERROR ReadClient{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:49.613860Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:49.639439Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:49.642458Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(580) < seq(604) [INFO] [stdout] 2025-09-01T11:54:49.656074Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(586) < seq(604) [INFO] [stdout] 2025-09-01T11:54:49.656582Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 604, max: 586 } [INFO] [stdout] 2025-09-01T11:54:49.656617Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:49.657551Z ERROR Node{id=4}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:49.657738Z WARN Node{id=4}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:49.657773Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:49.657888Z INFO Node{id=4}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:49.730108Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:49.730461Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(627) < seq(646) [INFO] [stdout] 2025-09-01T11:54:49.747989Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(634) < seq(646) [INFO] [stdout] 2025-09-01T11:54:49.748233Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 646, max: 634 } [INFO] [stdout] 2025-09-01T11:54:49.748257Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:49.748741Z ERROR Node{id=6}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:49.748945Z WARN Node{id=6}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:49.748962Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:49.749019Z INFO Node{id=6}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:49.771698Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:49.778318Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(643) < seq(660) [INFO] [stdout] 2025-09-01T11:54:49.789631Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(649) < seq(660) [INFO] [stdout] 2025-09-01T11:54:49.790316Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 660, max: 649 } [INFO] [stdout] 2025-09-01T11:54:49.790344Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:49.794233Z ERROR Node{id=5}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:49.794964Z WARN Node{id=5}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:49.794983Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:49.795477Z INFO Node{id=5}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:49.833307Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:49.833354Z INFO Node{id=9}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:54:49.833857Z INFO Node{id=9}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:49.833945Z INFO Node{id=9}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:49.834073Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=9 [INFO] [stdout] 2025-09-01T11:54:49.842404Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: Ping(1756727689833) } [INFO] [stdout] 2025-09-01T11:54:49.846099Z INFO Node{id=9}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:54:49.846138Z INFO Node{id=9}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:49.857498Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:49.857795Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:54:49.858065Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:49.864240Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:49.865120Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeRecovery(RecoverableStateDetails { id: 13252726253926303898, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:49.865647Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:49.866728Z INFO Node{id=22}: sequenced_broadcast: 72|Subscriber rejected, max_seq(666) < seq(668) [INFO] [stdout] 2025-09-01T11:54:49.874754Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727688540) } [INFO] [stdout] 2025-09-01T11:54:49.875901Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=1335 [INFO] [stdout] 2025-09-01T11:54:49.875934Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:49.886731Z INFO Node{id=22}: sequenced_broadcast: 72|Subscriber(1): Added, allow_drop: true, next_sequence: 668, min_allowed_seq: 45 [INFO] [stdout] 2025-09-01T11:54:49.887443Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 668) [INFO] [stdout] 2025-09-01T11:54:49.887551Z INFO Node{id=9}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:49.887581Z INFO Node{id=9}: sequenced_broadcast: 73|SequencedBroadcastWorker Started id=73 [INFO] [stdout] 2025-09-01T11:54:49.887670Z INFO Node{id=9}: sequenced_broadcast: 41|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:49.887687Z INFO Node{id=9}: sequenced_broadcast: 41|SequencedBroadcastWorker Stopped id=41 elapsed=3.432051683s iter=2 [INFO] [stdout] 2025-09-01T11:54:49.925240Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:49.925393Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(718) < seq(733) [INFO] [stdout] 2025-09-01T11:54:49.949069Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(728) < seq(733) [INFO] [stdout] 2025-09-01T11:54:49.952996Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 733, max: 728 } [INFO] [stdout] 2025-09-01T11:54:49.953026Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:49.954028Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727688739) } [INFO] [stdout] 2025-09-01T11:54:49.954344Z ERROR Node{id=7}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:49.954908Z WARN Node{id=7}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:49.954926Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:49.955617Z INFO Node{id=7}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:49.956162Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=1217 [INFO] [stdout] 2025-09-01T11:54:49.956189Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.026910Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727688827) } [INFO] [stdout] 2025-09-01T11:54:50.030656Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=1203 [INFO] [stdout] 2025-09-01T11:54:50.030692Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.043446Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727688837) } [INFO] [stdout] 2025-09-01T11:54:50.054958Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=1217 [INFO] [stdout] 2025-09-01T11:54:50.054999Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.079228Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:50.079550Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(775) < seq(789) [INFO] [stdout] 2025-09-01T11:54:50.092468Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(780) < seq(789) [INFO] [stdout] 2025-09-01T11:54:50.093190Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 789, max: 780 } [INFO] [stdout] 2025-09-01T11:54:50.093215Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:50.103345Z ERROR Node{id=32}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:50.103566Z WARN Node{id=32}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:50.103582Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:50.104401Z INFO Node{id=32}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:50.147069Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:50.165138Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(796) < seq(810) [INFO] [stdout] 2025-09-01T11:54:50.177792Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:50.177837Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:54:50.178274Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(799) < seq(810) [INFO] [stdout] 2025-09-01T11:54:50.178678Z INFO Node{id=14}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:50.178742Z INFO Node{id=14}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:50.179325Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 810, max: 799 } [INFO] [stdout] 2025-09-01T11:54:50.179348Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:50.179700Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:54:50.191805Z ERROR Node{id=24}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:50.192480Z WARN Node{id=24}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:50.192498Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:50.198046Z INFO Node{id=24}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:50.213219Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727690178) } [INFO] [stdout] 2025-09-01T11:54:50.224334Z INFO Node{id=14}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=46 [INFO] [stdout] 2025-09-01T11:54:50.224376Z INFO Node{id=14}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.226290Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:50.226495Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(810) < seq(825) [INFO] [stdout] 2025-09-01T11:54:50.238312Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:50.243220Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:54:50.243252Z ERROR Node{id=14}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=7 leader=1 local=14 [INFO] [stdout] 2025-09-01T11:54:50.248246Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:50.248321Z INFO Node{id=14}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:50.248806Z INFO Node{id=14}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:50.255052Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:50.255399Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(815) < seq(825) [INFO] [stdout] 2025-09-01T11:54:50.255507Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:50.263920Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 825, max: 815 } [INFO] [stdout] 2025-09-01T11:54:50.263958Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:50.265096Z ERROR Node{id=25}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:50.265788Z WARN Node{id=25}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:50.265807Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:50.266288Z INFO Node{id=25}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:50.279288Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:50.280497Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(822) < seq(835) [INFO] [stdout] 2025-09-01T11:54:50.309097Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(831) < seq(835) [INFO] [stdout] 2025-09-01T11:54:50.309704Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 835, max: 831 } [INFO] [stdout] 2025-09-01T11:54:50.309733Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:50.315367Z ERROR Node{id=31}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:50.315748Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:50.316242Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:54:50.316647Z WARN Node{id=31}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:50.316666Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:50.317216Z INFO Node{id=13}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:50.317290Z INFO Node{id=13}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:50.317330Z INFO Node{id=31}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:50.318078Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:54:50.337428Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727690317) } [INFO] [stdout] 2025-09-01T11:54:50.343664Z INFO Node{id=13}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=26 [INFO] [stdout] 2025-09-01T11:54:50.343701Z INFO Node{id=13}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.354908Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:50.361223Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:54:50.361256Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=15 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:54:50.362536Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:50.362598Z INFO Node{id=13}:connect{target=15}:WritePeer{remote=15}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:50.363416Z INFO Node{id=13}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:50.364097Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:50.364209Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:50.421956Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:50.422021Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:54:50.422525Z INFO Node{id=12}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:50.422620Z INFO Node{id=12}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:50.430887Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:54:50.443381Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727690422) } [INFO] [stdout] 2025-09-01T11:54:50.443420Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=12 [INFO] [stdout] 2025-09-01T11:54:50.452580Z INFO Node{id=12}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=30 [INFO] [stdout] 2025-09-01T11:54:50.452620Z INFO Node{id=12}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.459813Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:50.466427Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:50.466947Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(879) < seq(894) [INFO] [stdout] 2025-09-01T11:54:50.467126Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:54:50.467146Z ERROR Node{id=12}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=20 leader=1 local=12 [INFO] [stdout] 2025-09-01T11:54:50.467600Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:50.467647Z INFO Node{id=12}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:50.470086Z INFO Node{id=12}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:50.470579Z ERROR ReadClient{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:50.470977Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:50.480773Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(885) < seq(894) [INFO] [stdout] 2025-09-01T11:54:50.481324Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 894, max: 885 } [INFO] [stdout] 2025-09-01T11:54:50.481347Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:50.487247Z ERROR Node{id=26}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:50.488227Z ERROR Node{id=22}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:54:50.488336Z WARN Node{id=26}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:50.488352Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:50.491833Z INFO Node{id=26}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:50.916034Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:50.916079Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=11 leader=1 [INFO] [stdout] 2025-09-01T11:54:50.916436Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:50.916463Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:54:50.916537Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:50.916566Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:54:50.916616Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:50.916635Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:54:50.916715Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:50.916736Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:54:50.922991Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:50.923053Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:54:50.923148Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:50.923170Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:54:50.923692Z INFO Node{id=2}:connect{target=11}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:50.923773Z INFO Node{id=2}:connect{target=11}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:50.924257Z INFO Node{id=15}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:50.924313Z INFO Node{id=15}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:50.924372Z INFO Node{id=3}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:50.924415Z INFO Node{id=3}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:50.924481Z INFO Node{id=11}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:50.924525Z INFO Node{id=11}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:50.924609Z INFO Node{id=4}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:50.924656Z INFO Node{id=4}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:50.931801Z INFO Node{id=6}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:50.931942Z INFO Node{id=6}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:50.932063Z INFO Node{id=5}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:50.932112Z INFO Node{id=5}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:50.932277Z INFO Node{id=11}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:54:50.932847Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727689017) } [INFO] [stdout] 2025-09-01T11:54:50.933176Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:54:50.933245Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:54:50.933734Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:54:50.935842Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:54:50.935902Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:54:50.936601Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:54:50.946970Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=1929 [INFO] [stdout] 2025-09-01T11:54:50.947011Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.954430Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727690923) } [INFO] [stdout] 2025-09-01T11:54:50.962062Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727690924) } [INFO] [stdout] 2025-09-01T11:54:50.962130Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727690924) } [INFO] [stdout] 2025-09-01T11:54:50.962166Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727690924) } [INFO] [stdout] 2025-09-01T11:54:50.962895Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:50.962926Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:54:50.970253Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727690932) } [INFO] [stdout] 2025-09-01T11:54:50.970849Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727690924) } [INFO] [stdout] 2025-09-01T11:54:50.970886Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727690931) } [INFO] [stdout] 2025-09-01T11:54:50.970902Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=6 [INFO] [stdout] 2025-09-01T11:54:50.971058Z INFO Node{id=7}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:50.971146Z INFO Node{id=7}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:50.972078Z INFO Node{id=2}:connect{target=11}: sharedstate::worker::sync_manager: got pong from peer latency=49 [INFO] [stdout] 2025-09-01T11:54:50.972104Z INFO Node{id=2}:connect{target=11}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.972128Z INFO Node{id=15}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=48 [INFO] [stdout] 2025-09-01T11:54:50.972144Z INFO Node{id=15}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.972165Z INFO Node{id=3}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=48 [INFO] [stdout] 2025-09-01T11:54:50.972180Z INFO Node{id=3}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.972367Z INFO Node{id=11}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=48 [INFO] [stdout] 2025-09-01T11:54:50.972385Z INFO Node{id=11}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.972439Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:54:50.973597Z INFO Node{id=5}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=41 [INFO] [stdout] 2025-09-01T11:54:50.973625Z INFO Node{id=5}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.973657Z INFO Node{id=4}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=49 [INFO] [stdout] 2025-09-01T11:54:50.973676Z INFO Node{id=4}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.973704Z INFO Node{id=6}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=42 [INFO] [stdout] 2025-09-01T11:54:50.973723Z INFO Node{id=6}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:50.984210Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:50.984255Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:50.984284Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:50.984309Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:50.991057Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:50.991125Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:50.991387Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:50.992013Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727690971) } [INFO] [stdout] 2025-09-01T11:54:50.993318Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:54:50.993338Z ERROR Node{id=11}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=13 leader=1 local=11 [INFO] [stdout] 2025-09-01T11:54:50.993382Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:54:50.993395Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=2 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:54:50.993431Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:54:50.993444Z ERROR Node{id=15}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=27 leader=1 local=15 [INFO] [stdout] 2025-09-01T11:54:50.993480Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 11 } [INFO] [stdout] 2025-09-01T11:54:50.993491Z ERROR Node{id=2}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=11 leader=1 local=2 [INFO] [stdout] 2025-09-01T11:54:51.001289Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:54:51.001327Z ERROR Node{id=6}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=10 leader=1 local=6 [INFO] [stdout] 2025-09-01T11:54:51.001366Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:54:51.001377Z ERROR Node{id=4}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=25 leader=1 local=4 [INFO] [stdout] 2025-09-01T11:54:51.001409Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:54:51.001420Z ERROR Node{id=5}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=2 leader=1 local=5 [INFO] [stdout] 2025-09-01T11:54:51.001441Z INFO Node{id=7}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=30 [INFO] [stdout] 2025-09-01T11:54:51.001465Z INFO Node{id=7}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:51.001509Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.001608Z INFO Node{id=2}:connect{target=11}:WritePeer{remote=11}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.001676Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.001714Z INFO Node{id=15}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.001777Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.001826Z INFO Node{id=3}:connect{target=2}:WritePeer{remote=2}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.001880Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.001916Z INFO Node{id=11}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.002455Z INFO Node{id=2}:connect{target=11}:ReadPeer{remote=11}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.007839Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.007914Z INFO Node{id=5}:connect{target=2}:WritePeer{remote=2}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.007988Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.008035Z INFO Node{id=4}:connect{target=25}:WritePeer{remote=25}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.008088Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.008128Z INFO Node{id=6}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.008191Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.008779Z INFO Node{id=15}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.008855Z INFO Node{id=3}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.008915Z INFO Node{id=11}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.009596Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.016826Z INFO Node{id=5}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.016967Z INFO Node{id=4}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.017016Z INFO Node{id=6}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.017704Z ERROR ReadClient{remote=15}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.017795Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.017846Z ERROR ReadClient{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.017959Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.018020Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.018049Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.018653Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.018714Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.021802Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.022517Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.022590Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.022936Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.029974Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:51.032300Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:54:51.044239Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.044460Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:51.045785Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeRecovery(RecoverableStateDetails { id: 15849201951862108824, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:51.046898Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:51.047325Z INFO Node{id=9}: sequenced_broadcast: 73|Subscriber rejected, max_seq(794) < seq(795) [INFO] [stdout] 2025-09-01T11:54:51.075193Z INFO Node{id=9}: sequenced_broadcast: 73|Subscriber(1): Added, allow_drop: true, next_sequence: 795, min_allowed_seq: 668 [INFO] [stdout] 2025-09-01T11:54:51.092047Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 795) [INFO] [stdout] 2025-09-01T11:54:51.092209Z INFO Node{id=7}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:51.092261Z INFO Node{id=7}: sequenced_broadcast: 74|SequencedBroadcastWorker Started id=74 [INFO] [stdout] 2025-09-01T11:54:51.092604Z INFO Node{id=7}: sequenced_broadcast: 39|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:51.092623Z INFO Node{id=7}: sequenced_broadcast: 39|SequencedBroadcastWorker Stopped id=39 elapsed=4.642894179s iter=2 [INFO] [stdout] 2025-09-01T11:54:51.094654Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:51.099113Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(943) < seq(954) [INFO] [stdout] 2025-09-01T11:54:51.104572Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:51.104614Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:54:51.112715Z INFO Node{id=32}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:51.112820Z INFO Node{id=32}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:51.113353Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:54:51.124946Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(949) < seq(954) [INFO] [stdout] 2025-09-01T11:54:51.125958Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 954, max: 949 } [INFO] [stdout] 2025-09-01T11:54:51.125984Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:51.129654Z ERROR Node{id=27}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.137935Z WARN Node{id=27}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:51.137966Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.138007Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727691112) } [INFO] [stdout] 2025-09-01T11:54:51.138366Z INFO Node{id=27}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.148717Z INFO Node{id=32}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=36 [INFO] [stdout] 2025-09-01T11:54:51.148779Z INFO Node{id=32}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:51.151406Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:51.163565Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:54:51.163601Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=3 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:54:51.169638Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.169716Z INFO Node{id=32}:connect{target=3}:WritePeer{remote=3}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.172975Z INFO Node{id=32}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.173575Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.179611Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.202580Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:51.202628Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:54:51.203687Z INFO Node{id=24}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:51.203751Z INFO Node{id=24}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:51.204213Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:54:51.223779Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727691203) } [INFO] [stdout] 2025-09-01T11:54:51.223830Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=24 [INFO] [stdout] 2025-09-01T11:54:51.227889Z INFO Node{id=24}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=24 [INFO] [stdout] 2025-09-01T11:54:51.227924Z INFO Node{id=24}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:51.239699Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:51.248216Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:54:51.248252Z ERROR Node{id=24}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=23 leader=1 local=24 [INFO] [stdout] 2025-09-01T11:54:51.252287Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:51.252331Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:54:51.253115Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.253165Z INFO Node{id=24}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.253382Z INFO Node{id=14}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:51.253442Z INFO Node{id=14}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:51.259105Z INFO Node{id=24}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.259224Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:54:51.259898Z ERROR ReadClient{remote=24}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.260559Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.267573Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:51.267615Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:54:51.273862Z INFO Node{id=25}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:51.273930Z INFO Node{id=25}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:51.274477Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:54:51.275944Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727691253) } [INFO] [stdout] 2025-09-01T11:54:51.290028Z INFO Node{id=14}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=37 [INFO] [stdout] 2025-09-01T11:54:51.290067Z INFO Node{id=14}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:51.298741Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727691273) } [INFO] [stdout] 2025-09-01T11:54:51.298799Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=25 [INFO] [stdout] 2025-09-01T11:54:51.300394Z INFO Node{id=25}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=27 [INFO] [stdout] 2025-09-01T11:54:51.300422Z INFO Node{id=25}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:51.311713Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:51.320708Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:54:51.320792Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:51.320820Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:54:51.321718Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:51.321852Z INFO Node{id=31}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:51.321913Z INFO Node{id=31}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:51.322809Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:54:51.333054Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:54:51.333093Z ERROR Node{id=25}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=10 leader=1 local=25 [INFO] [stdout] 2025-09-01T11:54:51.334541Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.334607Z INFO Node{id=25}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.340883Z INFO Node{id=25}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.347036Z ERROR ReadClient{remote=25}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.347600Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727691321) } [INFO] [stdout] 2025-09-01T11:54:51.347651Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.347867Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.349332Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeRecovery(RecoverableStateDetails { id: 637232509615178270, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:51.361102Z INFO Node{id=31}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=40 [INFO] [stdout] 2025-09-01T11:54:51.361144Z INFO Node{id=31}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:51.361172Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:51.365619Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:51.365655Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:54:51.366321Z INFO Node{id=13}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:51.366383Z INFO Node{id=13}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:51.366420Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:51.373113Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:54:51.379262Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:51.382026Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:54:51.382049Z ERROR Node{id=31}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=13 leader=1 local=31 [INFO] [stdout] 2025-09-01T11:54:51.382729Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.382796Z INFO Node{id=31}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.382939Z INFO Node{id=17}: sequenced_broadcast: 70|Subscriber rejected, max_seq(972) < seq(974) [INFO] [stdout] 2025-09-01T11:54:51.383419Z INFO Node{id=31}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.383626Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.394932Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.401411Z INFO Node{id=17}: sequenced_broadcast: 70|Subscriber(1): Added, allow_drop: true, next_sequence: 974, min_allowed_seq: 23 [INFO] [stdout] 2025-09-01T11:54:51.411861Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 974) [INFO] [stdout] 2025-09-01T11:54:51.411964Z INFO Node{id=14}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:51.412693Z INFO Node{id=14}: sequenced_broadcast: 75|SequencedBroadcastWorker Started id=75 [INFO] [stdout] 2025-09-01T11:54:51.417579Z INFO Node{id=14}: sequenced_broadcast: 46|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:51.417605Z INFO Node{id=14}: sequenced_broadcast: 46|SequencedBroadcastWorker Stopped id=46 elapsed=4.958563264s iter=2 [INFO] [stdout] 2025-09-01T11:54:51.445778Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:54:51.445817Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 2 peers to 1 peers [INFO] [stdout] 2025-09-01T11:54:51.470944Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:51.470996Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:54:51.475942Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:51.476392Z INFO Node{id=12}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:51.476477Z INFO Node{id=12}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:51.485939Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:54:51.486390Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(1032) < seq(1041) [INFO] [stdout] 2025-09-01T11:54:51.495873Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:51.495902Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:54:51.504130Z INFO Node{id=26}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:51.504219Z INFO Node{id=26}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:51.505092Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:54:51.526947Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(1036) < seq(1041) [INFO] [stdout] 2025-09-01T11:54:51.528010Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 1041, max: 1036 } [INFO] [stdout] 2025-09-01T11:54:51.528037Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:51.528071Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727691476) } [INFO] [stdout] 2025-09-01T11:54:51.540419Z ERROR Node{id=29}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.541744Z WARN Node{id=29}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:51.546814Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.546925Z INFO Node{id=12}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=70 [INFO] [stdout] 2025-09-01T11:54:51.546952Z INFO Node{id=12}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:51.547704Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727691504) } [INFO] [stdout] 2025-09-01T11:54:51.547728Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=26 [INFO] [stdout] 2025-09-01T11:54:51.553797Z INFO Node{id=29}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.564388Z INFO Node{id=26}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=60 [INFO] [stdout] 2025-09-01T11:54:51.564429Z INFO Node{id=26}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:51.565422Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:51.581366Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:54:51.581403Z ERROR Node{id=12}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=30 leader=1 local=12 [INFO] [stdout] 2025-09-01T11:54:51.581440Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:51.585681Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.588777Z INFO Node{id=12}:connect{target=30}:WritePeer{remote=30}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.590399Z INFO Node{id=12}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.591649Z ERROR ReadClient{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.591743Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:54:51.598850Z ERROR Node{id=26}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=23 leader=1 local=26 [INFO] [stdout] 2025-09-01T11:54:51.598992Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.600288Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.600340Z INFO Node{id=26}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.609137Z INFO Node{id=26}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:51.614715Z ERROR ReadClient{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.616072Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:51.668913Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:51.669533Z WARN Node{id=22}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:51.669561Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.670614Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(1058) < seq(1070) [INFO] [stdout] 2025-09-01T11:54:51.681309Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.688221Z INFO Node{id=1}: sequenced_broadcast: 4|Subscriber(20): channel closed, dropping [INFO] [stdout] 2025-09-01T11:54:51.697063Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(1064) < seq(1070) [INFO] [stdout] 2025-09-01T11:54:51.703884Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 1070, max: 1064 } [INFO] [stdout] 2025-09-01T11:54:51.703923Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:51.707573Z ERROR Node{id=28}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.708248Z WARN Node{id=28}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:51.708267Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.708810Z INFO Node{id=28}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.852468Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:51.857347Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(1103) < seq(1115) [INFO] [stdout] 2025-09-01T11:54:51.873280Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(1108) < seq(1115) [INFO] [stdout] 2025-09-01T11:54:51.874148Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 1115, max: 1108 } [INFO] [stdout] 2025-09-01T11:54:51.874173Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:51.877277Z ERROR Node{id=30}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.877736Z WARN Node{id=30}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:51.877754Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.877891Z INFO Node{id=30}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:51.890494Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:51.890970Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(1112) < seq(1125) [INFO] [stdout] 2025-09-01T11:54:51.905306Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(1116) < seq(1125) [INFO] [stdout] 2025-09-01T11:54:51.905920Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 1125, max: 1116 } [INFO] [stdout] 2025-09-01T11:54:51.905945Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:51.910312Z ERROR Node{id=16}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:51.911054Z WARN Node{id=16}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:51.911078Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:51.911672Z INFO Node{id=16}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.009503Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.009565Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.009619Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.009639Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.009771Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.009802Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.009851Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.009869Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.010090Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.010120Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.010161Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.010179Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.010212Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.010230Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.014060Z INFO Node{id=15}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.014143Z INFO Node{id=15}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.014193Z INFO Node{id=2}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.014251Z INFO Node{id=2}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.014328Z INFO Node{id=11}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.014393Z INFO Node{id=11}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.014449Z INFO Node{id=3}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.014502Z INFO Node{id=3}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.014870Z INFO Node{id=6}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.014933Z INFO Node{id=6}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.014984Z INFO Node{id=4}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.015036Z INFO Node{id=4}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.015084Z INFO Node{id=5}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.015143Z INFO Node{id=5}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.016002Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:54:52.016061Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:54:52.016110Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:54:52.016154Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:54:52.016251Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:54:52.016297Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:54:52.031326Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:54:52.042517Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727692014) } [INFO] [stdout] 2025-09-01T11:54:52.043161Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727692015) } [INFO] [stdout] 2025-09-01T11:54:52.049033Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727692014) } [INFO] [stdout] 2025-09-01T11:54:52.049105Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727692014) } [INFO] [stdout] 2025-09-01T11:54:52.049140Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727692014) } [INFO] [stdout] 2025-09-01T11:54:52.049179Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727692014) } [INFO] [stdout] 2025-09-01T11:54:52.049211Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=3 [INFO] [stdout] 2025-09-01T11:54:52.053193Z INFO Node{id=6}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=39 [INFO] [stdout] 2025-09-01T11:54:52.053224Z INFO Node{id=6}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.054679Z INFO Node{id=4}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=39 [INFO] [stdout] 2025-09-01T11:54:52.054706Z INFO Node{id=4}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.057568Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727692015) } [INFO] [stdout] 2025-09-01T11:54:52.057749Z INFO Node{id=15}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=43 [INFO] [stdout] 2025-09-01T11:54:52.057796Z INFO Node{id=15}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.057828Z INFO Node{id=2}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=43 [INFO] [stdout] 2025-09-01T11:54:52.057847Z INFO Node{id=2}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.057879Z INFO Node{id=11}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=43 [INFO] [stdout] 2025-09-01T11:54:52.057896Z INFO Node{id=11}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.057921Z INFO Node{id=3}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=43 [INFO] [stdout] 2025-09-01T11:54:52.057937Z INFO Node{id=3}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.064255Z INFO Node{id=5}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=49 [INFO] [stdout] 2025-09-01T11:54:52.064295Z INFO Node{id=5}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.064664Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.065319Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.073084Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.073153Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.073191Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.073223Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.078978Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:54:52.081301Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:54:52.081326Z ERROR Node{id=4}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=31 leader=1 local=4 [INFO] [stdout] 2025-09-01T11:54:52.081687Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.082731Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:54:52.085815Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=10 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:54:52.085923Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:54:52.085940Z ERROR Node{id=11}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=6 leader=1 local=11 [INFO] [stdout] 2025-09-01T11:54:52.086001Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:54:52.086017Z ERROR Node{id=2}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=25 leader=1 local=2 [INFO] [stdout] 2025-09-01T11:54:52.086073Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:54:52.086090Z ERROR Node{id=15}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=32 leader=1 local=15 [INFO] [stdout] 2025-09-01T11:54:52.086745Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.086817Z INFO Node{id=4}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.093957Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.094032Z INFO Node{id=15}:connect{target=32}:WritePeer{remote=32}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.094103Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.094152Z INFO Node{id=2}:connect{target=25}:WritePeer{remote=25}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.094222Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.094273Z INFO Node{id=11}:connect{target=6}:WritePeer{remote=6}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.094333Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.094384Z INFO Node{id=3}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.094508Z INFO Node{id=4}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.094856Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.095504Z INFO Node{id=15}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.095618Z INFO Node{id=2}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.095703Z INFO Node{id=11}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.100779Z INFO Node{id=3}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.100939Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.101047Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:54:52.101796Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.101953Z ERROR ReadClient{remote=15}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.102016Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.102073Z ERROR ReadClient{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.102131Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.108103Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.108176Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.108220Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.108260Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.112954Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:52.128357Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeRecovery(RecoverableStateDetails { id: 2956772458635842773, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:52.129384Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.132988Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeRecovery(RecoverableStateDetails { id: 3001348910760355552, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:52.133460Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:52.140616Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.140661Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.141754Z INFO Node{id=27}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.147906Z INFO Node{id=27}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.149222Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:52.149285Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:54:52.153081Z INFO Node{id=14}: sequenced_broadcast: 75|Subscriber rejected, max_seq(1066) < seq(1068) [INFO] [stdout] 2025-09-01T11:54:52.164345Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727692147) } [INFO] [stdout] 2025-09-01T11:54:52.164382Z INFO Node{id=8}: sharedstate::worker::sync_manager: Discovered Peer client_id=27 [INFO] [stdout] 2025-09-01T11:54:52.168351Z INFO Node{id=27}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=21 [INFO] [stdout] 2025-09-01T11:54:52.168389Z INFO Node{id=27}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.174252Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.174300Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.179446Z INFO Node{id=32}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.179536Z INFO Node{id=32}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.184112Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:54:52.184500Z INFO Node{id=14}: sequenced_broadcast: 75|Subscriber(1): Added, allow_drop: true, next_sequence: 1068, min_allowed_seq: 974 [INFO] [stdout] 2025-09-01T11:54:52.193027Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.205914Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1068) [INFO] [stdout] 2025-09-01T11:54:52.206048Z INFO Node{id=6}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:52.207276Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:54:52.207300Z ERROR Node{id=27}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=8 leader=1 local=27 [INFO] [stdout] 2025-09-01T11:54:52.207327Z INFO Node{id=6}: sequenced_broadcast: 76|SequencedBroadcastWorker Started id=76 [INFO] [stdout] 2025-09-01T11:54:52.207391Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727692179) } [INFO] [stdout] 2025-09-01T11:54:52.207468Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=32 [INFO] [stdout] 2025-09-01T11:54:52.217249Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:52.217830Z INFO Node{id=14}: sequenced_broadcast: 75|Subscriber rejected, max_seq(1068) < seq(1071) [INFO] [stdout] 2025-09-01T11:54:52.217885Z INFO Node{id=6}: sequenced_broadcast: 38|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:52.217898Z INFO Node{id=6}: sequenced_broadcast: 38|SequencedBroadcastWorker Stopped id=38 elapsed=5.768221114s iter=2 [INFO] [stdout] 2025-09-01T11:54:52.218000Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.218052Z INFO Node{id=27}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.224079Z INFO Node{id=27}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.226930Z ERROR ReadClient{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.227195Z INFO Node{id=32}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=48 [INFO] [stdout] 2025-09-01T11:54:52.227228Z INFO Node{id=32}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.230815Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.236830Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.250821Z INFO Node{id=14}: sequenced_broadcast: 75|Subscriber(2): Added, allow_drop: true, next_sequence: 1071, min_allowed_seq: 974 [INFO] [stdout] 2025-09-01T11:54:52.255930Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:54:52.255961Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=23 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:54:52.256570Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.256599Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.256744Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.256805Z INFO Node{id=32}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.257361Z INFO Node{id=24}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.257437Z INFO Node{id=24}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.265083Z INFO Node{id=32}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.265217Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1071) [INFO] [stdout] 2025-09-01T11:54:52.265324Z INFO Node{id=5}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:52.265366Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:54:52.266040Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.266096Z INFO Node{id=5}: sequenced_broadcast: 77|SequencedBroadcastWorker Started id=77 [INFO] [stdout] 2025-09-01T11:54:52.266183Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.267198Z INFO Node{id=5}: sequenced_broadcast: 37|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:52.267215Z INFO Node{id=5}: sequenced_broadcast: 37|SequencedBroadcastWorker Stopped id=37 elapsed=5.817595942s iter=2 [INFO] [stdout] 2025-09-01T11:54:52.282245Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727692257) } [INFO] [stdout] 2025-09-01T11:54:52.284260Z INFO Node{id=24}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=27 [INFO] [stdout] 2025-09-01T11:54:52.284306Z INFO Node{id=24}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.304600Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.317071Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:54:52.317109Z ERROR Node{id=24}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=16 leader=1 local=24 [INFO] [stdout] 2025-09-01T11:54:52.319146Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.319196Z INFO Node{id=24}:connect{target=16}:WritePeer{remote=16}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.324594Z INFO Node{id=24}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.325238Z ERROR ReadClient{remote=24}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.328697Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.336943Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.336986Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.340918Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727691366) } [INFO] [stdout] 2025-09-01T11:54:52.341238Z INFO Node{id=25}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.341309Z INFO Node{id=25}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.342486Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:54:52.350647Z INFO Node{id=13}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=984 [INFO] [stdout] 2025-09-01T11:54:52.350747Z INFO Node{id=13}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.374254Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727692341) } [INFO] [stdout] 2025-09-01T11:54:52.388387Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.388431Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.388999Z INFO Node{id=25}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=47 [INFO] [stdout] 2025-09-01T11:54:52.389037Z INFO Node{id=25}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.389719Z INFO Node{id=31}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.392846Z INFO Node{id=31}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.394006Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:54:52.405664Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.422343Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:54:52.422377Z ERROR Node{id=25}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=26 leader=1 local=25 [INFO] [stdout] 2025-09-01T11:54:52.428456Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.428526Z INFO Node{id=25}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.433898Z INFO Node{id=25}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.434412Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727692392) } [INFO] [stdout] 2025-09-01T11:54:52.443066Z ERROR ReadClient{remote=25}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.443918Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.447091Z INFO Node{id=31}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=55 [INFO] [stdout] 2025-09-01T11:54:52.447127Z INFO Node{id=31}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.463990Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.477358Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:54:52.477394Z ERROR Node{id=31}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=25 leader=1 local=31 [INFO] [stdout] 2025-09-01T11:54:52.478473Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.478522Z INFO Node{id=31}:connect{target=25}:WritePeer{remote=25}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.479676Z INFO Node{id=31}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.488094Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.488265Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.555695Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.555739Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.556797Z INFO Node{id=29}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.556880Z INFO Node{id=29}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.562196Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:54:52.582540Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727692556) } [INFO] [stdout] 2025-09-01T11:54:52.588953Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.588999Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.598309Z INFO Node{id=12}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.598408Z INFO Node{id=12}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.599620Z INFO Node{id=29}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=43 [INFO] [stdout] 2025-09-01T11:54:52.599647Z INFO Node{id=29}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.599686Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:54:52.602714Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.602743Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.603657Z INFO Node{id=26}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.603725Z INFO Node{id=26}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.613992Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:54:52.616603Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.630031Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727692598) } [INFO] [stdout] 2025-09-01T11:54:52.633343Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:54:52.633374Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=15 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:54:52.634571Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727692603) } [INFO] [stdout] 2025-09-01T11:54:52.635378Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.635432Z INFO Node{id=12}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=37 [INFO] [stdout] 2025-09-01T11:54:52.635457Z INFO Node{id=12}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.640198Z INFO Node{id=29}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.640294Z INFO Node{id=29}:connect{target=15}:WritePeer{remote=15}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.641599Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.641702Z INFO Node{id=26}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=38 [INFO] [stdout] 2025-09-01T11:54:52.641729Z INFO Node{id=26}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.648401Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.653295Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.661618Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.665553Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:54:52.665581Z ERROR Node{id=12}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=32 leader=1 local=12 [INFO] [stdout] 2025-09-01T11:54:52.669889Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.669963Z INFO Node{id=12}:connect{target=32}:WritePeer{remote=32}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.670608Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:54:52.674379Z INFO Node{id=12}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.677950Z ERROR ReadClient{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.678879Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.687726Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.688919Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeRecovery(RecoverableStateDetails { id: 13147879088706776850, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:52.695453Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:52.695753Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.695801Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.696010Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:52.701985Z INFO Node{id=22}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.702088Z INFO Node{id=22}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.703413Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:54:52.706508Z INFO Node{id=7}: sequenced_broadcast: 74|Subscriber(1): Added, allow_drop: true, next_sequence: 876, min_allowed_seq: 795 [INFO] [stdout] 2025-09-01T11:54:52.717168Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.717211Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.718335Z INFO Node{id=28}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.718408Z INFO Node{id=28}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.724362Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 876) [INFO] [stdout] 2025-09-01T11:54:52.724498Z INFO Node{id=26}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:52.725094Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:54:52.725896Z INFO Node{id=26}: sequenced_broadcast: 78|SequencedBroadcastWorker Started id=78 [INFO] [stdout] 2025-09-01T11:54:52.731071Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727692702) } [INFO] [stdout] 2025-09-01T11:54:52.731112Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=22 [INFO] [stdout] 2025-09-01T11:54:52.731459Z INFO Node{id=26}: sequenced_broadcast: 58|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:52.731479Z INFO Node{id=26}: sequenced_broadcast: 58|SequencedBroadcastWorker Stopped id=58 elapsed=6.263558977s iter=2 [INFO] [stdout] 2025-09-01T11:54:52.745145Z INFO Node{id=22}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=43 [INFO] [stdout] 2025-09-01T11:54:52.745181Z INFO Node{id=22}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.755336Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727692718) } [INFO] [stdout] 2025-09-01T11:54:52.755390Z INFO Node{id=8}: sharedstate::worker::sync_manager: Discovered Peer client_id=28 [INFO] [stdout] 2025-09-01T11:54:52.769471Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.769552Z INFO Node{id=28}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=51 [INFO] [stdout] 2025-09-01T11:54:52.769573Z INFO Node{id=28}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.795899Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.796568Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:54:52.796590Z ERROR Node{id=22}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=10 leader=1 local=22 [INFO] [stdout] 2025-09-01T11:54:52.800170Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.801231Z INFO Node{id=22}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.801300Z INFO Node{id=22}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.802012Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.819280Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.819496Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:54:52.819519Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=8 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:54:52.821533Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.821584Z INFO Node{id=28}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:52.826906Z INFO Node{id=28}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:52.829799Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:52.830579Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:52.880879Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.880922Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.881992Z INFO Node{id=30}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.882054Z INFO Node{id=30}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.889412Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:54:52.914920Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:52.914966Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:54:52.915530Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727692882) } [INFO] [stdout] 2025-09-01T11:54:52.915561Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=30 [INFO] [stdout] 2025-09-01T11:54:52.916099Z INFO Node{id=16}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:52.916169Z INFO Node{id=16}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:52.916733Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:54:52.917418Z INFO Node{id=30}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=35 [INFO] [stdout] 2025-09-01T11:54:52.917440Z INFO Node{id=30}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.954422Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:52.966184Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727692916) } [INFO] [stdout] 2025-09-01T11:54:52.988364Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:54:52.988401Z ERROR Node{id=30}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=10 leader=1 local=30 [INFO] [stdout] 2025-09-01T11:54:52.989373Z INFO Node{id=16}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=73 [INFO] [stdout] 2025-09-01T11:54:52.989401Z INFO Node{id=16}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:52.990173Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:52.990227Z INFO Node{id=30}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:53.003447Z INFO Node{id=30}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:53.003862Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:53.029434Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.038852Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.066619Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:54:53.066656Z ERROR Node{id=16}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=29 leader=1 local=16 [INFO] [stdout] 2025-09-01T11:54:53.081872Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:53.081940Z INFO Node{id=16}:connect{target=29}:WritePeer{remote=29}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:53.083666Z INFO Node{id=16}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:53.093197Z ERROR ReadClient{remote=16}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:53.093316Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.093367Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.093392Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.101132Z INFO Node{id=4}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.101214Z INFO Node{id=4}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.101249Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.101273Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.101342Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.101365Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.101410Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.101431Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.101472Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.101492Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.108488Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:54:53.108595Z INFO Node{id=3}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.108668Z INFO Node{id=3}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.108718Z INFO Node{id=11}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.113594Z INFO Node{id=11}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.113693Z INFO Node{id=2}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.113794Z INFO Node{id=2}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.113847Z INFO Node{id=15}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.113909Z INFO Node{id=15}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.121268Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:54:53.121369Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:54:53.121422Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:54:53.121544Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:54:53.149316Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727693113) } [INFO] [stdout] 2025-09-01T11:54:53.153153Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727693108) } [INFO] [stdout] 2025-09-01T11:54:53.153204Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727693113) } [INFO] [stdout] 2025-09-01T11:54:53.153580Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727693101) } [INFO] [stdout] 2025-09-01T11:54:53.154461Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727693113) } [INFO] [stdout] 2025-09-01T11:54:53.154483Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=15 [INFO] [stdout] 2025-09-01T11:54:53.167913Z INFO Node{id=2}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=54 [INFO] [stdout] 2025-09-01T11:54:53.167953Z INFO Node{id=2}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.168607Z INFO Node{id=3}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=60 [INFO] [stdout] 2025-09-01T11:54:53.168630Z INFO Node{id=3}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.168654Z INFO Node{id=11}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=55 [INFO] [stdout] 2025-09-01T11:54:53.168668Z INFO Node{id=11}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.170998Z INFO Node{id=4}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=69 [INFO] [stdout] 2025-09-01T11:54:53.171033Z INFO Node{id=4}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.177206Z INFO Node{id=15}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=64 [INFO] [stdout] 2025-09-01T11:54:53.177249Z INFO Node{id=15}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.186687Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.187335Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.187371Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.194926Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.195174Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.213313Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:54:53.214163Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:54:53.214186Z ERROR Node{id=11}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=29 leader=1 local=11 [INFO] [stdout] 2025-09-01T11:54:53.214244Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:54:53.214260Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=27 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:54:53.214436Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:54:53.214461Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 4 peers to 3 peers [INFO] [stdout] 2025-09-01T11:54:53.222833Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:53.222896Z INFO Node{id=3}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:53.222956Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:53.222993Z INFO Node{id=11}:connect{target=29}:WritePeer{remote=29}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:53.223276Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:54:53.223304Z ERROR Node{id=15}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=10 leader=1 local=15 [INFO] [stdout] 2025-09-01T11:54:53.223345Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:54:53.223412Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.223434Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.224136Z INFO Node{id=3}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:53.224198Z INFO Node{id=11}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:53.224522Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:53.224579Z ERROR ReadClient{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:53.231668Z INFO Node{id=27}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.232329Z INFO Node{id=27}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.232750Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:53.232828Z INFO Node{id=15}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:53.233642Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:54:53.233704Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.234686Z INFO Node{id=15}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:53.234829Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.234915Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.241117Z ERROR ReadClient{remote=15}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:53.242211Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.242268Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.246911Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:53.247180Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeRecovery(RecoverableStateDetails { id: 3967747982285532032, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:53.253522Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:53.261173Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:53.264014Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.264049Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.264890Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeRecovery(RecoverableStateDetails { id: 6495069325016322389, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:53.265014Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.265291Z INFO Node{id=6}: sequenced_broadcast: 76|Subscriber rejected, max_seq(1165) < seq(1166) [INFO] [stdout] 2025-09-01T11:54:53.265549Z INFO Node{id=32}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.265610Z INFO Node{id=32}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.266456Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:54:53.281555Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:53.300121Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:53.303834Z INFO Node{id=6}: sequenced_broadcast: 76|Subscriber(1): Added, allow_drop: true, next_sequence: 1166, min_allowed_seq: 1068 [INFO] [stdout] 2025-09-01T11:54:53.304282Z INFO Node{id=18}: sequenced_broadcast: 69|Subscriber rejected, max_seq(1345) < seq(1349) [INFO] [stdout] 2025-09-01T11:54:53.310490Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727693232) } [INFO] [stdout] 2025-09-01T11:54:53.315533Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1166) [INFO] [stdout] 2025-09-01T11:54:53.315646Z INFO Node{id=4}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:53.316086Z INFO Node{id=4}: sequenced_broadcast: 79|SequencedBroadcastWorker Started id=79 [INFO] [stdout] 2025-09-01T11:54:53.316180Z INFO Node{id=27}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=84 [INFO] [stdout] 2025-09-01T11:54:53.316202Z INFO Node{id=27}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.316372Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727693265) } [INFO] [stdout] 2025-09-01T11:54:53.317111Z INFO Node{id=4}: sequenced_broadcast: 36|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:53.317127Z INFO Node{id=4}: sequenced_broadcast: 36|SequencedBroadcastWorker Stopped id=36 elapsed=6.867591505s iter=2 [INFO] [stdout] 2025-09-01T11:54:53.324151Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.324198Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.325043Z INFO Node{id=32}:connect{target=19}: sharedstate::worker::sync_manager: got pong from peer latency=60 [INFO] [stdout] 2025-09-01T11:54:53.325071Z INFO Node{id=32}:connect{target=19}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.325833Z INFO Node{id=24}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.325911Z INFO Node{id=24}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.330131Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:54:53.330499Z INFO Node{id=18}: sequenced_broadcast: 69|Subscriber(1): Added, allow_drop: true, next_sequence: 1349, min_allowed_seq: 18 [INFO] [stdout] 2025-09-01T11:54:53.334273Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.339196Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.343111Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:54:53.348223Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1349) [INFO] [stdout] 2025-09-01T11:54:53.348371Z INFO Node{id=2}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:53.353192Z INFO Node{id=2}: sequenced_broadcast: 80|SequencedBroadcastWorker Started id=80 [INFO] [stdout] 2025-09-01T11:54:53.353949Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 19 } [INFO] [stdout] 2025-09-01T11:54:53.355170Z INFO Node{id=2}: sequenced_broadcast: 34|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:53.355188Z INFO Node{id=2}: sequenced_broadcast: 34|SequencedBroadcastWorker Stopped id=34 elapsed=6.905845725s iter=2 [INFO] [stdout] 2025-09-01T11:54:53.355444Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727693325) } [INFO] [stdout] 2025-09-01T11:54:53.367432Z INFO Node{id=24}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=42 [INFO] [stdout] 2025-09-01T11:54:53.367480Z INFO Node{id=24}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.381536Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.382959Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.387106Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.387575Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeRecovery(RecoverableStateDetails { id: 13739817974832384570, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:53.388647Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:53.390216Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:53.397291Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 5 } [INFO] [stdout] 2025-09-01T11:54:53.406335Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SubscribeRecovery(RecoverableStateDetails { id: 4930385600716210758, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:53.407558Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:53.412456Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.414247Z INFO Node{id=19}: sequenced_broadcast: 71|Subscriber rejected, max_seq(1359) < seq(1363) [INFO] [stdout] 2025-09-01T11:54:53.423152Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeRecovery(RecoverableStateDetails { id: 1864775603426703921, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:53.423436Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:53.430888Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:53.431164Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.431194Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.432175Z INFO Node{id=5}: sequenced_broadcast: 77|Subscriber rejected, max_seq(1181) < seq(1183) [INFO] [stdout] 2025-09-01T11:54:53.432680Z INFO Node{id=25}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.435985Z INFO Node{id=25}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.437498Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:54:53.444434Z INFO Node{id=19}: sequenced_broadcast: 71|Subscriber(1): Added, allow_drop: true, next_sequence: 1363, min_allowed_seq: 35 [INFO] [stdout] 2025-09-01T11:54:53.465513Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1363) [INFO] [stdout] 2025-09-01T11:54:53.465640Z INFO Node{id=32}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:53.465953Z INFO Node{id=5}: sequenced_broadcast: 77|Subscriber(1): Added, allow_drop: true, next_sequence: 1183, min_allowed_seq: 1071 [INFO] [stdout] 2025-09-01T11:54:53.466515Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727693435) } [INFO] [stdout] 2025-09-01T11:54:53.466611Z INFO Node{id=32}: sequenced_broadcast: 81|SequencedBroadcastWorker Started id=81 [INFO] [stdout] 2025-09-01T11:54:53.477157Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:53.477453Z INFO Node{id=32}: sequenced_broadcast: 64|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:53.477473Z INFO Node{id=32}: sequenced_broadcast: 64|SequencedBroadcastWorker Stopped id=64 elapsed=7.009131102s iter=2 [INFO] [stdout] 2025-09-01T11:54:53.478222Z INFO Node{id=6}: sequenced_broadcast: 76|Subscriber rejected, max_seq(1189) < seq(1191) [INFO] [stdout] 2025-09-01T11:54:53.487789Z INFO Node{id=25}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=52 [INFO] [stdout] 2025-09-01T11:54:53.487840Z INFO Node{id=25}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.488675Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.488708Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.495791Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1183) [INFO] [stdout] 2025-09-01T11:54:53.495922Z INFO Node{id=24}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:53.495989Z INFO Node{id=31}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.496065Z INFO Node{id=31}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.501088Z INFO Node{id=24}: sequenced_broadcast: 82|SequencedBroadcastWorker Started id=82 [INFO] [stdout] 2025-09-01T11:54:53.501161Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:54:53.507170Z INFO Node{id=24}: sequenced_broadcast: 56|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:53.507188Z INFO Node{id=24}: sequenced_broadcast: 56|SequencedBroadcastWorker Stopped id=56 elapsed=7.047510213s iter=2 [INFO] [stdout] 2025-09-01T11:54:53.519012Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.520232Z INFO Node{id=6}: sequenced_broadcast: 76|Subscriber(2): Added, allow_drop: true, next_sequence: 1191, min_allowed_seq: 1068 [INFO] [stdout] 2025-09-01T11:54:53.540692Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.542182Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:54:53.542205Z ERROR Node{id=25}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=12 leader=1 local=25 [INFO] [stdout] 2025-09-01T11:54:53.542398Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.547815Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727693496) } [INFO] [stdout] 2025-09-01T11:54:53.547847Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=31 [INFO] [stdout] 2025-09-01T11:54:53.549033Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:53.549084Z INFO Node{id=25}:connect{target=12}:WritePeer{remote=12}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:53.557277Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1191) [INFO] [stdout] 2025-09-01T11:54:53.557412Z INFO Node{id=27}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:53.557463Z INFO Node{id=25}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:53.563527Z ERROR ReadClient{remote=25}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:53.566814Z INFO Node{id=27}: sequenced_broadcast: 83|SequencedBroadcastWorker Started id=83 [INFO] [stdout] 2025-09-01T11:54:53.567573Z INFO Node{id=31}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=71 [INFO] [stdout] 2025-09-01T11:54:53.567602Z INFO Node{id=31}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.567669Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:53.568993Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.569546Z INFO Node{id=27}: sequenced_broadcast: 59|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:53.569565Z INFO Node{id=27}: sequenced_broadcast: 59|SequencedBroadcastWorker Stopped id=59 elapsed=7.10157111s iter=2 [INFO] [stdout] 2025-09-01T11:54:53.575216Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:54:53.575249Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=22 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:54:53.583538Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:53.583603Z INFO Node{id=13}:connect{target=22}:WritePeer{remote=22}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:53.590822Z INFO Node{id=13}:connect{target=22}:ReadPeer{remote=22}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:53.599511Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:53.604687Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.634671Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:54:53.634716Z ERROR Node{id=31}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=20 leader=1 local=31 [INFO] [stdout] 2025-09-01T11:54:53.641686Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:53.641752Z INFO Node{id=31}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:53.644047Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.644077Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.651051Z INFO Node{id=31}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:53.651674Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:53.653809Z INFO Node{id=29}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.653883Z INFO Node{id=29}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.664249Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:54:53.667603Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.680425Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.680473Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.687478Z INFO Node{id=12}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.687572Z INFO Node{id=12}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.692284Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:54:53.716517Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727693653) } [INFO] [stdout] 2025-09-01T11:54:53.716564Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=29 [INFO] [stdout] 2025-09-01T11:54:53.734419Z INFO Node{id=29}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=81 [INFO] [stdout] 2025-09-01T11:54:53.734464Z INFO Node{id=29}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.743518Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727693687) } [INFO] [stdout] 2025-09-01T11:54:53.759333Z INFO Node{id=12}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=72 [INFO] [stdout] 2025-09-01T11:54:53.759376Z INFO Node{id=12}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.760155Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:54:53.760180Z INFO Node{id=8}: sharedstate::worker::sync_manager: Broadcasted 3 peers to 2 peers [INFO] [stdout] 2025-09-01T11:54:53.761609Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.784709Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:54:53.784746Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=10 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:54:53.790706Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:53.797005Z INFO Node{id=29}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:53.800474Z INFO Node{id=29}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:53.801328Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:53.807257Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.807310Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.808545Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.808995Z INFO Node{id=22}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.809061Z INFO Node{id=22}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.810752Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.822567Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:54:53.830609Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.830650Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:54:53.839137Z INFO Node{id=28}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:53.839244Z INFO Node{id=28}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:53.845388Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:54:53.845464Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:54:53.871461Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:53.881844Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.890660Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:53.891059Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeRecovery(RecoverableStateDetails { id: 3338079291511305984, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:53.900287Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727693839) } [INFO] [stdout] 2025-09-01T11:54:53.901939Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:53.902394Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:53.903103Z INFO Node{id=24}: sequenced_broadcast: 82|Subscriber rejected, max_seq(1209) < seq(1211) [INFO] [stdout] 2025-09-01T11:54:53.912820Z INFO Node{id=28}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=73 [INFO] [stdout] 2025-09-01T11:54:53.912866Z INFO Node{id=28}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:53.941169Z INFO Node{id=24}: sequenced_broadcast: 82|Subscriber(1): Added, allow_drop: true, next_sequence: 1211, min_allowed_seq: 1183 [INFO] [stdout] 2025-09-01T11:54:53.958283Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:53.976161Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1211) [INFO] [stdout] 2025-09-01T11:54:53.976323Z INFO Node{id=12}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:53.983207Z INFO Node{id=12}: sequenced_broadcast: 84|SequencedBroadcastWorker Started id=84 [INFO] [stdout] 2025-09-01T11:54:53.983306Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:54:53.995103Z INFO Node{id=12}: sequenced_broadcast: 44|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:53.995138Z INFO Node{id=12}: sequenced_broadcast: 44|SequencedBroadcastWorker Stopped id=44 elapsed=7.536237738s iter=2 [INFO] [stdout] 2025-09-01T11:54:53.996402Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:53.996432Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:54:54.000647Z INFO Node{id=30}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:54.000740Z INFO Node{id=30}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:54.007863Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:54.010741Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:54:54.028369Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:54:54.047928Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:54.061142Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727694000) } [INFO] [stdout] 2025-09-01T11:54:54.068854Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:54.080403Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 16584620148520749951, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:54.085833Z INFO Node{id=30}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=85 [INFO] [stdout] 2025-09-01T11:54:54.085874Z INFO Node{id=30}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:54.095080Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:54.095133Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:54:54.097835Z INFO Node{id=16}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:54.097913Z INFO Node{id=16}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:54.105549Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:54:54.124466Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:54.150325Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:54:54.180723Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:54.195193Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:54.195478Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SubscribeRecovery(RecoverableStateDetails { id: 16571668472844679484, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:54.216117Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:54.227978Z INFO Node{id=32}: sequenced_broadcast: 81|Subscriber rejected, max_seq(1421) < seq(1423) [INFO] [stdout] 2025-09-01T11:54:54.228226Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:54.228260Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:54:54.228331Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:54.228356Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:54:54.238061Z INFO Node{id=11}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:54.238160Z INFO Node{id=11}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:54.238202Z INFO Node{id=3}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:54.238253Z INFO Node{id=3}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:54.243172Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:54.243208Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:54:54.244254Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:54:54.244350Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:54:54.250124Z INFO Node{id=15}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:54.250215Z INFO Node{id=15}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:54.262009Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:54:54.270775Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:54.270959Z INFO Node{id=32}: sequenced_broadcast: 81|Subscriber(1): Added, allow_drop: true, next_sequence: 1423, min_allowed_seq: 1363 [INFO] [stdout] 2025-09-01T11:54:54.286038Z INFO Node{id=24}: sequenced_broadcast: 82|Subscriber rejected, max_seq(1238) < seq(1242) [INFO] [stdout] 2025-09-01T11:54:54.303642Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727694238) } [INFO] [stdout] 2025-09-01T11:54:54.303687Z INFO Node{id=8}: sharedstate::worker::sync_manager: Discovered Peer client_id=11 [INFO] [stdout] 2025-09-01T11:54:54.303723Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727694238) } [INFO] [stdout] 2025-09-01T11:54:54.303744Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=3 [INFO] [stdout] 2025-09-01T11:54:54.312140Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1423) [INFO] [stdout] 2025-09-01T11:54:54.312280Z INFO Node{id=30}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:54.313463Z INFO Node{id=24}: sequenced_broadcast: 82|Subscriber rejected, max_seq(1241) < seq(1242) [INFO] [stdout] 2025-09-01T11:54:54.320404Z INFO Node{id=30}: sequenced_broadcast: 85|SequencedBroadcastWorker Started id=85 [INFO] [stdout] 2025-09-01T11:54:54.326345Z ERROR Node{id=24}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 1242, max: 1241 } [INFO] [stdout] 2025-09-01T11:54:54.326385Z ERROR Node{id=24}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:54:54.332097Z INFO Node{id=11}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=94 [INFO] [stdout] 2025-09-01T11:54:54.332138Z INFO Node{id=11}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:54.332164Z INFO Node{id=3}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=94 [INFO] [stdout] 2025-09-01T11:54:54.332181Z INFO Node{id=3}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:54.332353Z INFO Node{id=30}: sequenced_broadcast: 62|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:54.332370Z INFO Node{id=30}: sequenced_broadcast: 62|SequencedBroadcastWorker Stopped id=62 elapsed=7.864221121s iter=2 [INFO] [stdout] 2025-09-01T11:54:54.344744Z ERROR Node{id=28}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:54.354336Z WARN Node{id=28}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:54.354368Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:54.358422Z INFO Node{id=28}:connect{target=24}:WritePeer{remote=24}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:54.370234Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:54.370304Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:54.405142Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:54:54.405180Z ERROR Node{id=3}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=20 leader=1 local=3 [INFO] [stdout] 2025-09-01T11:54:54.405238Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:54:54.405255Z ERROR Node{id=11}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=8 leader=1 local=11 [INFO] [stdout] 2025-09-01T11:54:54.412661Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:54.412733Z INFO Node{id=11}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:54.416496Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:54.416557Z INFO Node{id=3}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:54.423093Z INFO Node{id=11}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:54.423187Z INFO Node{id=3}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:54.423405Z ERROR ReadClient{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:54.423479Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:54.437885Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:54.437984Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:54.559421Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:54.559467Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:54:54.564355Z INFO Node{id=25}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:54.564437Z INFO Node{id=25}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:54.574414Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:54:54.587999Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:54.588046Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:54:54.598441Z INFO Node{id=13}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:54.598550Z INFO Node{id=13}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:54.610597Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:54:54.634992Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727694564) } [INFO] [stdout] 2025-09-01T11:54:54.648305Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:54.648349Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:54:54.658920Z INFO Node{id=31}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:54.659035Z INFO Node{id=31}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:54.659546Z INFO Node{id=25}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=95 [INFO] [stdout] 2025-09-01T11:54:54.659573Z INFO Node{id=25}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:54.669863Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:54:54.671255Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727694598) } [INFO] [stdout] 2025-09-01T11:54:54.706978Z INFO Node{id=13}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=108 [INFO] [stdout] 2025-09-01T11:54:54.707023Z INFO Node{id=13}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:54.708053Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727694097) } [INFO] [stdout] 2025-09-01T11:54:54.713118Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:54.728814Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:54:54.728858Z INFO Node{id=10}: sharedstate::worker::sync_manager: Broadcasted 8 peers to 7 peers [INFO] [stdout] 2025-09-01T11:54:54.729456Z INFO Node{id=16}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=632 [INFO] [stdout] 2025-09-01T11:54:54.729486Z INFO Node{id=16}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:54.743517Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:54.745744Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:54:54.745786Z ERROR Node{id=25}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=13 leader=1 local=25 [INFO] [stdout] 2025-09-01T11:54:54.755384Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:54.764082Z INFO Node{id=25}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:54.764182Z INFO Node{id=25}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:54.766493Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:54.766583Z ERROR ReadClient{remote=25}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:54.775092Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:54:54.775130Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=25 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:54:54.788942Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:54.794666Z INFO Node{id=13}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:54.794743Z INFO Node{id=13}:connect{target=25}:WritePeer{remote=25}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:54.799705Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:54.799740Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:54:54.803915Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:54.804486Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:54.805169Z INFO Node{id=29}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:54.805228Z INFO Node{id=29}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:54.819044Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:54:55.250807Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:55.281626Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:54:55.300728Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727694805) } [INFO] [stdout] 2025-09-01T11:54:55.328466Z INFO Node{id=29}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=523 [INFO] [stdout] 2025-09-01T11:54:55.328514Z INFO Node{id=29}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:55.365705Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:55.365750Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:54:55.374863Z INFO Node{id=28}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:55.374976Z INFO Node{id=28}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:55.380352Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:54:55.411995Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:55.424244Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:55.424303Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:54:55.426262Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:55.426290Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:54:55.431413Z INFO Node{id=11}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:55.431510Z INFO Node{id=11}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:55.432725Z INFO Node{id=3}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:55.437822Z INFO Node{id=3}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:55.439471Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:54:55.442366Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:54:55.443354Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727695374) } [INFO] [stdout] 2025-09-01T11:54:55.443379Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=28 [INFO] [stdout] 2025-09-01T11:54:55.471331Z INFO Node{id=28}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=97 [INFO] [stdout] 2025-09-01T11:54:55.471375Z INFO Node{id=28}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:55.494590Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727695431) } [INFO] [stdout] 2025-09-01T11:54:55.502921Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:55.514438Z INFO Node{id=11}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=83 [INFO] [stdout] 2025-09-01T11:54:55.514475Z INFO Node{id=11}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:55.530824Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:54:55.530863Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=20 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:54:55.531264Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727695437) } [INFO] [stdout] 2025-09-01T11:54:55.535263Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:55.535357Z INFO Node{id=28}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:55.541736Z INFO Node{id=28}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:55.541880Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:55.551000Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:55.551117Z INFO Node{id=3}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=114 [INFO] [stdout] 2025-09-01T11:54:55.551149Z INFO Node{id=3}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:55.556474Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:55.585083Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:54:55.585121Z ERROR Node{id=11}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=16 leader=1 local=11 [INFO] [stdout] 2025-09-01T11:54:55.596634Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:55.596706Z INFO Node{id=11}:connect{target=16}:WritePeer{remote=16}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:55.604708Z INFO Node{id=11}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:55.615604Z ERROR ReadClient{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:55.622283Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:55.648602Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:55.676843Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:54:55.758929Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:55.761508Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:55.761542Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:54:55.762088Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeRecovery(RecoverableStateDetails { id: 259493686016308194, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:55.766460Z INFO Node{id=25}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:55.766544Z INFO Node{id=25}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:55.773406Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:55.776236Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:54:55.776316Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:55.783591Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeRecovery(RecoverableStateDetails { id: 1069456004009129247, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:55.784508Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:55.795732Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:55.795781Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:54:55.797803Z INFO Node{id=13}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:55.797886Z INFO Node{id=13}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:55.804189Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:54:55.809841Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727695766) } [INFO] [stdout] 2025-09-01T11:54:55.809869Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=25 [INFO] [stdout] 2025-09-01T11:54:55.822540Z INFO Node{id=25}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=56 [INFO] [stdout] 2025-09-01T11:54:55.822588Z INFO Node{id=25}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:55.825586Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:55.834838Z INFO Node{id=7}: sequenced_broadcast: 74|Subscriber(2): Added, allow_drop: true, next_sequence: 876, min_allowed_seq: 795 [INFO] [stdout] 2025-09-01T11:54:55.840309Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727695797) } [INFO] [stdout] 2025-09-01T11:54:55.842533Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:55.854034Z INFO Node{id=13}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=57 [INFO] [stdout] 2025-09-01T11:54:55.854080Z INFO Node{id=13}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:55.856015Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 876) [INFO] [stdout] 2025-09-01T11:54:55.856132Z INFO Node{id=3}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:55.865808Z INFO Node{id=3}: sequenced_broadcast: 86|SequencedBroadcastWorker Started id=86 [INFO] [stdout] 2025-09-01T11:54:55.865911Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:54:55.865931Z ERROR Node{id=25}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=21 leader=1 local=25 [INFO] [stdout] 2025-09-01T11:54:55.872229Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:55.872319Z INFO Node{id=25}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:55.872454Z INFO Node{id=3}: sequenced_broadcast: 35|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:55.872473Z INFO Node{id=3}: sequenced_broadcast: 35|SequencedBroadcastWorker Stopped id=35 elapsed=9.423058695s iter=2 [INFO] [stdout] 2025-09-01T11:54:55.874259Z INFO Node{id=25}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:55.878863Z ERROR ReadClient{remote=25}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:55.885685Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:55.886322Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:55.903636Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:54:55.923636Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:55.929523Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeRecovery(RecoverableStateDetails { id: 7729601842349720361, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:55.941099Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:55.942694Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:55.950745Z ERROR Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for path to leader [INFO] [stdout] 2025-09-01T11:54:55.958563Z INFO Node{id=30}: sequenced_broadcast: 85|Subscriber rejected, max_seq(1525) < seq(1526) [INFO] [stdout] 2025-09-01T11:54:55.961947Z INFO Node{id=8}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:55.962053Z INFO Node{id=8}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:54:55.962137Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:54:55.966331Z ERROR ReadClient{remote=8}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:55.994371Z INFO Node{id=30}: sequenced_broadcast: 85|Subscriber(1): Added, allow_drop: true, next_sequence: 1526, min_allowed_seq: 1423 [INFO] [stdout] 2025-09-01T11:54:56.019519Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 1526) [INFO] [stdout] 2025-09-01T11:54:56.019664Z INFO Node{id=13}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:56.024279Z INFO Node{id=13}: sequenced_broadcast: 87|SequencedBroadcastWorker Started id=87 [INFO] [stdout] 2025-09-01T11:54:56.031101Z INFO Node{id=13}: sequenced_broadcast: 45|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:56.031134Z INFO Node{id=13}: sequenced_broadcast: 45|SequencedBroadcastWorker Stopped id=45 elapsed=9.572168679s iter=2 [INFO] [stdout] 2025-09-01T11:54:56.089911Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:56.104932Z INFO Node{id=9}: sequenced_broadcast: 73|Subscriber(2): Added, allow_drop: true, next_sequence: 876, min_allowed_seq: 668 [INFO] [stdout] 2025-09-01T11:54:56.109504Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:56.135060Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 876) [INFO] [stdout] 2025-09-01T11:54:56.135180Z INFO Node{id=16}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:56.137940Z INFO Node{id=16}: sequenced_broadcast: 88|SequencedBroadcastWorker Started id=88 [INFO] [stdout] 2025-09-01T11:54:56.145733Z INFO Node{id=16}: sequenced_broadcast: 48|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:56.148799Z INFO Node{id=16}: sequenced_broadcast: 48|SequencedBroadcastWorker Stopped id=48 elapsed=9.689625591s iter=2 [INFO] [stdout] 2025-09-01T11:54:56.238426Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:56.259584Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:54:56.302415Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727694659) } [INFO] [stdout] 2025-09-01T11:54:56.316616Z INFO Node{id=31}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=1657 [INFO] [stdout] 2025-09-01T11:54:56.316660Z INFO Node{id=31}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:56.464681Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:54:56.464716Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 5 peers to 4 peers [INFO] [stdout] 2025-09-01T11:54:56.537694Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:56.537738Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:54:56.539522Z INFO Node{id=28}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:56.539600Z INFO Node{id=28}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:56.541519Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:54:56.598650Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:56.598696Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:54:56.600887Z INFO Node{id=11}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:56.600970Z INFO Node{id=11}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:56.603050Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:54:56.680695Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:54:56.686201Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:54:56.745270Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:54:56.815651Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:54:56.815692Z INFO Node{id=21}: sharedstate::worker::sync_manager: Broadcasted 2 peers to 1 peers [INFO] [stdout] 2025-09-01T11:54:56.875060Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:56.875105Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:54:56.877256Z INFO Node{id=25}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:56.877350Z INFO Node{id=25}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:56.879714Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:54:56.899048Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727696877) } [INFO] [stdout] 2025-09-01T11:54:56.909217Z INFO Node{id=25}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=32 [INFO] [stdout] 2025-09-01T11:54:56.909263Z INFO Node{id=25}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:56.923185Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:56.932143Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:54:56.941749Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:56.944451Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SubscribeRecovery(RecoverableStateDetails { id: 4080161324815589637, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:56.946365Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:56.948439Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:56.950828Z INFO Node{id=16}: sequenced_broadcast: 88|Subscriber(1): Added, allow_drop: true, next_sequence: 876, min_allowed_seq: 876 [INFO] [stdout] 2025-09-01T11:54:56.961895Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 876) [INFO] [stdout] 2025-09-01T11:54:56.962025Z INFO Node{id=25}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:56.964155Z INFO Node{id=25}: sequenced_broadcast: 89|SequencedBroadcastWorker Started id=89 [INFO] [stdout] 2025-09-01T11:54:56.965098Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:56.965125Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=3 peers=4 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:56.966672Z INFO Node{id=25}: sequenced_broadcast: 57|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:56.966691Z INFO Node{id=25}: sequenced_broadcast: 57|SequencedBroadcastWorker Stopped id=57 elapsed=10.506957389s iter=2 [INFO] [stdout] 2025-09-01T11:54:56.967184Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:56.967260Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:56.969568Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:54:57.132818Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:57.139684Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:57.141252Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeRecovery(RecoverableStateDetails { id: 13528030838573203821, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:57.395211Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727693809) } [INFO] [stdout] 2025-09-01T11:54:57.401843Z INFO Node{id=22}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=3592 [INFO] [stdout] 2025-09-01T11:54:57.401881Z INFO Node{id=22}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:57.613277Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727696539) } [INFO] [stdout] 2025-09-01T11:54:57.619546Z INFO Node{id=28}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=1080 [INFO] [stdout] 2025-09-01T11:54:57.619585Z INFO Node{id=28}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:57.696365Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:57.705367Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 5 } [INFO] [stdout] 2025-09-01T11:54:57.814694Z ERROR Node{id=19}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:54:58.521020Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727696600) } [INFO] [stdout] 2025-09-01T11:54:58.526948Z INFO Node{id=11}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=1926 [INFO] [stdout] 2025-09-01T11:54:58.526989Z INFO Node{id=11}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:58.763486Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:54:58.763529Z INFO Node{id=8}: sharedstate::worker::sync_manager: Broadcasted 4 peers to 3 peers [INFO] [stdout] 2025-09-01T11:54:58.831670Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:58.838536Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:58.844559Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeRecovery(RecoverableStateDetails { id: 15966214309367731050, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:58.849562Z ERROR Node{id=20}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:58.851755Z WARN Node{id=20}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:58.851792Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:58.854154Z INFO Node{id=20}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:59.040798Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:54:59.045196Z INFO Node{id=32}: sequenced_broadcast: 81|Subscriber rejected, max_seq(2134) < seq(2136) [INFO] [stdout] 2025-09-01T11:54:59.064619Z INFO Node{id=32}: sequenced_broadcast: 81|Subscriber(2): Added, allow_drop: true, next_sequence: 2136, min_allowed_seq: 1363 [INFO] [stdout] 2025-09-01T11:54:59.076699Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 2136) [INFO] [stdout] 2025-09-01T11:54:59.076844Z INFO Node{id=29}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:54:59.078916Z INFO Node{id=29}: sequenced_broadcast: 90|SequencedBroadcastWorker Started id=90 [INFO] [stdout] 2025-09-01T11:54:59.081091Z INFO Node{id=29}: sequenced_broadcast: 61|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:54:59.081109Z INFO Node{id=29}: sequenced_broadcast: 61|SequencedBroadcastWorker Stopped id=61 elapsed=12.613011244s iter=2 [INFO] [stdout] 2025-09-01T11:54:59.150859Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727694250) } [INFO] [stdout] 2025-09-01T11:54:59.161065Z INFO Node{id=15}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=4911 [INFO] [stdout] 2025-09-01T11:54:59.161111Z INFO Node{id=15}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:54:59.224744Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:59.233105Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeRecovery(RecoverableStateDetails { id: 9391859352936412299, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:59.234107Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:59.237992Z ERROR Node{id=21}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:59.240645Z WARN Node{id=21}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:59.240665Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:59.243161Z INFO Node{id=21}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:59.379599Z WARN Node{id=19}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:59.379631Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:59.381399Z INFO Node{id=19}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:59.383236Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:59.535699Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:59.542611Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:59.550681Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeRecovery(RecoverableStateDetails { id: 15335810314635339748, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:59.555427Z ERROR Node{id=10}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:59.557433Z WARN Node{id=10}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:59.557454Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:59.559495Z INFO Node{id=10}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:59.653469Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:54:59.660099Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:54:59.663878Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeRecovery(RecoverableStateDetails { id: 634985037238024698, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:54:59.668136Z ERROR Node{id=23}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:54:59.672272Z WARN Node{id=23}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:54:59.672306Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:54:59.674239Z INFO Node{id=23}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:54:59.841380Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:54:59.846942Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:54:59.858488Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:54:59.858532Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=4 peers=5 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:54:59.859530Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:54:59.860592Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:54:59.860671Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:54:59.862662Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:55:00.243378Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:00.243419Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=1 peers=2 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:00.245783Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:00.245876Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:00.248324Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:55:00.392471Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:00.392517Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=4 leader=1 [INFO] [stdout] 2025-09-01T11:55:00.394379Z INFO Node{id=19}:connect{target=4}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:00.394464Z INFO Node{id=19}:connect{target=4}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:00.395883Z INFO Node{id=4}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:00.407410Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727700394) } [INFO] [stdout] 2025-09-01T11:55:00.412109Z INFO Node{id=19}:connect{target=4}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:00.412146Z INFO Node{id=19}:connect{target=4}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:00.421479Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:00.434043Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 4 } [INFO] [stdout] 2025-09-01T11:55:00.435331Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:00.440150Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:00.440522Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:00.444348Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 2270, state_accept_seq: 2269, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:00.444933Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:00.444965Z ERROR Node{id=9}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:00.448103Z INFO Node{id=4}: sequenced_broadcast: 79|Subscriber(1): Added, allow_drop: true, next_sequence: 2270, min_allowed_seq: 1166 [INFO] [stdout] 2025-09-01T11:55:00.460327Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 2270) [INFO] [stdout] 2025-09-01T11:55:00.460414Z INFO Node{id=19}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:55:00.560076Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:00.560119Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=7 peers=8 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:00.562326Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:00.562409Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:00.564509Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:55:00.675222Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:00.675263Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=3 peers=4 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:00.677495Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:00.677584Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:00.679914Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:55:00.873145Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:00.878466Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:00.878591Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeRecovery(RecoverableStateDetails { id: 14315515704630917242, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:01.037044Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:01.354397Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:01.969726Z ERROR Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:01.972260Z INFO Node{id=8}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:01.972352Z INFO Node{id=8}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:01.972431Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:01.974653Z ERROR ReadClient{remote=8}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:02.026958Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:02.041936Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:55:02.096634Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:02.113715Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:02.451101Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:02.673442Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:02.738222Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727696967) } [INFO] [stdout] 2025-09-01T11:55:02.905104Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:02.916227Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:55:02.947104Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:02.976726Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:02.976786Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=3 peers=4 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:02.979465Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:02.979580Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:02.981929Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:55:03.044880Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:03.080904Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:03.158755Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:03.172663Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:03.172699Z ERROR Node{id=9}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:03.175241Z INFO Node{id=1}: sequenced_broadcast: 3|Subscriber(20): channel closed, dropping [INFO] [stdout] 2025-09-01T11:55:03.231183Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:03.251612Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:03.354039Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:03.385603Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:03.407753Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:03.850868Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:04.009208Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:04.015417Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 22, 9, 7]) [INFO] [stdout] 2025-09-01T11:55:04.151443Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:04.164363Z ERROR Node{id=15}:connect{target=14}: sharedstate::worker::sync_manager: timeout waiting for path to leader [INFO] [stdout] 2025-09-01T11:55:04.166491Z INFO Node{id=15}:connect{target=14}:WritePeer{remote=14}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:04.166577Z INFO Node{id=15}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:04.166639Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(14) [INFO] [stdout] 2025-09-01T11:55:04.168577Z ERROR ReadClient{remote=15}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:04.862707Z ERROR Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:04.864651Z INFO Node{id=20}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:04.864729Z INFO Node{id=20}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:04.864815Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:04.866979Z ERROR ReadClient{remote=20}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:05.168348Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:05.168393Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:55:05.172405Z INFO Node{id=15}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:05.172506Z INFO Node{id=15}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:05.174440Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:55:05.177974Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:05.251046Z ERROR Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:05.253315Z INFO Node{id=21}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:05.253404Z INFO Node{id=21}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:05.253477Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:05.256529Z ERROR ReadClient{remote=21}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:05.284377Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:05.565312Z ERROR Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:05.567659Z INFO Node{id=10}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:05.567726Z INFO Node{id=10}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:05.567805Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:05.569520Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:05.609332Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727705172) } [INFO] [stdout] 2025-09-01T11:55:05.613310Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727699860) } [INFO] [stdout] 2025-09-01T11:55:05.618689Z INFO Node{id=15}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=446 [INFO] [stdout] 2025-09-01T11:55:05.618728Z INFO Node{id=15}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:05.679398Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:05.679450Z ERROR Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:05.681387Z INFO Node{id=23}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:05.681463Z INFO Node{id=23}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:05.681531Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:05.683721Z ERROR ReadClient{remote=23}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:05.713330Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:05.717340Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:05.717366Z ERROR Node{id=9}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:05.870069Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:05.870113Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=4 peers=5 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:05.872465Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:05.872569Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:05.874422Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:55:05.907173Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:05.931046Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:05.946210Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:55:05.984499Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:05.991823Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 22, 9, 7]) [INFO] [stdout] 2025-09-01T11:55:06.030288Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727700245) } [INFO] [stdout] 2025-09-01T11:55:06.167569Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:06.175919Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:06.176845Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeRecovery(RecoverableStateDetails { id: 1133459967848790104, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:06.255478Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:06.255519Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=1 peers=2 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:06.257389Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:06.257491Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:06.262167Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:06.263634Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:55:06.318771Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:06.323095Z INFO Node{id=7}: sequenced_broadcast: 74|Subscriber(3): Added, allow_drop: true, next_sequence: 876, min_allowed_seq: 795 [INFO] [stdout] 2025-09-01T11:55:06.325898Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:06.330846Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([9]) [INFO] [stdout] 2025-09-01T11:55:06.331597Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 876) [INFO] [stdout] 2025-09-01T11:55:06.331709Z INFO Node{id=15}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:06.337141Z INFO Node{id=15}: sequenced_broadcast: 91|SequencedBroadcastWorker Started id=91 [INFO] [stdout] 2025-09-01T11:55:06.339640Z INFO Node{id=15}: sequenced_broadcast: 47|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:06.339659Z INFO Node{id=15}: sequenced_broadcast: 47|SequencedBroadcastWorker Stopped id=47 elapsed=19.880554355s iter=2 [INFO] [stdout] 2025-09-01T11:55:06.398855Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:06.405072Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:06.410421Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeRecovery(RecoverableStateDetails { id: 1675523409979011756, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:06.419626Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727700562) } [INFO] [stdout] 2025-09-01T11:55:06.486553Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:06.494392Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:06.534504Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 16584620148520749951, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:06.538581Z ERROR Node{id=28}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:06.540685Z WARN Node{id=28}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:06.540704Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:06.542521Z INFO Node{id=28}:connect{target=32}:WritePeer{remote=32}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:06.543402Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727700677) } [INFO] [stdout] 2025-09-01T11:55:06.573552Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:06.573591Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=7 peers=8 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:06.575851Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:06.575925Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:06.577980Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:55:06.683560Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:06.683599Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=3 peers=4 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:06.687980Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:06.688079Z INFO Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:06.688810Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:06.689934Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:55:06.694623Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 18]) [INFO] [stdout] 2025-09-01T11:55:06.935354Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:06.950416Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:06.956457Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 22, 9, 16]) [INFO] [stdout] 2025-09-01T11:55:07.174111Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:07.177876Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 5, 24]) [INFO] [stdout] 2025-09-01T11:55:07.542920Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:07.542965Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:55:07.545139Z INFO Node{id=28}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:07.545239Z INFO Node{id=28}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:07.547351Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:55:07.710002Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:07.719747Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:07.921724Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:07.927909Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30]) [INFO] [stdout] 2025-09-01T11:55:07.981693Z ERROR Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:07.983837Z INFO Node{id=8}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:07.983904Z INFO Node{id=8}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:07.983960Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:07.985832Z ERROR ReadClient{remote=8}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:08.141223Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:55:08.191780Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:08.195392Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:08.195418Z ERROR Node{id=9}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:08.335913Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:08.420131Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727702979) } [INFO] [stdout] 2025-09-01T11:55:08.420266Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:55:08.968347Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:08.973436Z INFO Node{id=5}: sequenced_broadcast: 77|Subscriber rejected, max_seq(4144) < seq(4148) [INFO] [stdout] 2025-09-01T11:55:08.990055Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:08.990100Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=3 peers=4 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:08.992363Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:08.992447Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:08.994791Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:55:08.995134Z INFO Node{id=5}: sequenced_broadcast: 77|Subscriber(2): Added, allow_drop: true, next_sequence: 4148, min_allowed_seq: 2100 [INFO] [stdout] 2025-09-01T11:55:09.010269Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4148) [INFO] [stdout] 2025-09-01T11:55:09.010421Z INFO Node{id=31}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:09.012678Z INFO Node{id=31}: sequenced_broadcast: 92|SequencedBroadcastWorker Started id=92 [INFO] [stdout] 2025-09-01T11:55:09.015036Z INFO Node{id=31}: sequenced_broadcast: 63|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:09.015056Z INFO Node{id=31}: sequenced_broadcast: 63|SequencedBroadcastWorker Stopped id=63 elapsed=22.546847018s iter=2 [INFO] [stdout] 2025-09-01T11:55:09.043925Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:09.047918Z INFO Node{id=18}: sequenced_broadcast: 69|Subscriber(2): Added, allow_drop: true, next_sequence: 4357, min_allowed_seq: 2309 [INFO] [stdout] 2025-09-01T11:55:09.058927Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 4357) [INFO] [stdout] 2025-09-01T11:55:09.059066Z INFO Node{id=11}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:09.061252Z INFO Node{id=11}: sequenced_broadcast: 93|SequencedBroadcastWorker Started id=93 [INFO] [stdout] 2025-09-01T11:55:09.063623Z INFO Node{id=11}: sequenced_broadcast: 43|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:09.063645Z INFO Node{id=11}: sequenced_broadcast: 43|SequencedBroadcastWorker Stopped id=43 elapsed=22.604828625s iter=2 [INFO] [stdout] 2025-09-01T11:55:09.861015Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:09.865146Z INFO Node{id=9}:connect{target=22}:ReadPeer{remote=22}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:09.881830Z ERROR ReadClient{remote=9}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:10.444643Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:10.874530Z ERROR Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:10.874677Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:10.874706Z INFO Node{id=9}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:55:10.876698Z INFO Node{id=20}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:10.876787Z INFO Node{id=20}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:10.876864Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:10.876920Z INFO Node{id=9}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:10.876984Z INFO Node{id=9}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:10.879047Z ERROR ReadClient{remote=20}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:10.879113Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=9 [INFO] [stdout] 2025-09-01T11:55:11.034328Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:11.035659Z INFO Node{id=7}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:11.180041Z ERROR ReadClient{remote=7}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:11.205661Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:11.259795Z ERROR Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:11.261690Z INFO Node{id=21}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:11.261791Z INFO Node{id=21}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:11.261859Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:11.264257Z ERROR ReadClient{remote=21}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:11.417937Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727705872) } [INFO] [stdout] 2025-09-01T11:55:11.512326Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17]) [INFO] [stdout] 2025-09-01T11:55:11.579326Z ERROR Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:11.581508Z INFO Node{id=10}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:11.581597Z INFO Node{id=10}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:11.581673Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:11.583708Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:11.691278Z ERROR Node{id=23}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:11.693880Z INFO Node{id=23}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:11.693959Z INFO Node{id=23}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:11.694027Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:11.698259Z ERROR ReadClient{remote=23}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:11.758633Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727706257) } [INFO] [stdout] 2025-09-01T11:55:11.879744Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:11.879805Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=4 peers=5 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:11.881733Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:11.881829Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:11.884001Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:55:12.044798Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:12.047382Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:12.047418Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:55:12.049407Z INFO Node{id=7}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:12.049473Z INFO Node{id=7}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:12.051228Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:55:12.054391Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727706575) } [INFO] [stdout] 2025-09-01T11:55:12.070952Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727712049) } [INFO] [stdout] 2025-09-01T11:55:12.070987Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=7 [INFO] [stdout] 2025-09-01T11:55:12.073356Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:12.073384Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 5 peers to 4 peers [INFO] [stdout] 2025-09-01T11:55:12.077007Z INFO Node{id=7}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=28 [INFO] [stdout] 2025-09-01T11:55:12.077041Z INFO Node{id=7}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:12.080047Z ERROR Node{id=7}:connect{target=23}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:12.080856Z INFO Node{id=7}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:12.082257Z INFO Node{id=7}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:12.084209Z ERROR ReadClient{remote=7}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:12.088596Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:12.089419Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(23) [INFO] [stdout] 2025-09-01T11:55:12.148382Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727706688) } [INFO] [stdout] 2025-09-01T11:55:12.250306Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:12.256078Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:12.262462Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:12.263567Z INFO Node{id=17}: sequenced_broadcast: 70|Subscriber rejected, seq(876) < min_allowed(2923) [INFO] [stdout] 2025-09-01T11:55:12.265477Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:12.265504Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=1 peers=2 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:12.267174Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:12.267233Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:12.269421Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:55:12.270102Z ERROR Node{id=22}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:12.272592Z WARN Node{id=22}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:12.272611Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:12.274650Z INFO Node{id=22}:connect{target=17}:WritePeer{remote=17}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:12.547638Z ERROR Node{id=28}:connect{target=19}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:12.549494Z INFO Node{id=28}:connect{target=19}:WritePeer{remote=19}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:12.549599Z INFO Node{id=28}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:12.549682Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(19) [INFO] [stdout] 2025-09-01T11:55:12.551824Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:12.583731Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:12.583791Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=7 peers=8 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:12.585472Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:12.585551Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:12.586865Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:55:12.658656Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: Ping(1756727710876) } [INFO] [stdout] 2025-09-01T11:55:12.666694Z INFO Node{id=9}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=1790 [INFO] [stdout] 2025-09-01T11:55:12.666742Z INFO Node{id=9}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:12.696852Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:12.696894Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=4 peers=5 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:55:12.699093Z INFO Node{id=23}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:12.699190Z INFO Node{id=23}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:12.701605Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:55:12.704714Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:12.744051Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19]) [INFO] [stdout] 2025-09-01T11:55:12.777702Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:12.919159Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:13.093125Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:13.093171Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:55:13.095386Z INFO Node{id=7}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:13.095474Z INFO Node{id=7}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:13.097528Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:55:13.116360Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727713095) } [INFO] [stdout] 2025-09-01T11:55:13.122591Z INFO Node{id=7}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=27 [INFO] [stdout] 2025-09-01T11:55:13.122632Z INFO Node{id=7}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:13.134678Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:13.151355Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:13.168180Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:55:13.171156Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:13.178676Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:13.183171Z INFO Node{id=16}: sequenced_broadcast: 88|Subscriber(2): Added, allow_drop: true, next_sequence: 876, min_allowed_seq: 876 [INFO] [stdout] 2025-09-01T11:55:13.192976Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 876) [INFO] [stdout] 2025-09-01T11:55:13.193070Z INFO Node{id=7}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:55:13.275219Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:13.275289Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:55:13.277659Z INFO Node{id=22}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:13.277739Z INFO Node{id=22}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:13.280095Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:55:13.296054Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727713277) } [INFO] [stdout] 2025-09-01T11:55:13.296092Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=22 [INFO] [stdout] 2025-09-01T11:55:13.298613Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:13.298644Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 6 peers to 5 peers [INFO] [stdout] 2025-09-01T11:55:13.302784Z INFO Node{id=22}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=25 [INFO] [stdout] 2025-09-01T11:55:13.302813Z INFO Node{id=22}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:13.304676Z ERROR Node{id=22}:connect{target=20}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:13.304855Z INFO Node{id=22}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:13.306790Z INFO Node{id=22}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:13.306876Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(20) [INFO] [stdout] 2025-09-01T11:55:13.309238Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:13.313539Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:13.551511Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:13.551563Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:55:13.553299Z INFO Node{id=28}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:13.553378Z INFO Node{id=28}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:13.555402Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:55:13.740596Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727713553) } [INFO] [stdout] 2025-09-01T11:55:13.746838Z INFO Node{id=28}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=193 [INFO] [stdout] 2025-09-01T11:55:13.746876Z INFO Node{id=28}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:13.944993Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:13.959129Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:55:13.994676Z ERROR Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:13.997532Z INFO Node{id=8}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:13.997618Z INFO Node{id=8}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:13.997686Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:14.000131Z ERROR ReadClient{remote=8}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:14.004352Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:14.018221Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:14.018263Z ERROR Node{id=16}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:14.018611Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:14.162431Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:14.181484Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:55:14.205252Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:14.210952Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:14.211147Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 16584620148520749951, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:14.309892Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:14.309938Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:55:14.311201Z ERROR Node{id=27}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:55:14.311862Z INFO Node{id=22}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:14.311943Z INFO Node{id=22}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:14.312694Z WARN Node{id=27}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:14.312711Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:14.313963Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:55:14.318702Z ERROR ReadClient{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:14.328659Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727714311) } [INFO] [stdout] 2025-09-01T11:55:14.328693Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=22 [INFO] [stdout] 2025-09-01T11:55:14.334474Z INFO Node{id=22}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=23 [INFO] [stdout] 2025-09-01T11:55:14.334509Z INFO Node{id=22}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:14.339997Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:14.345285Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:14.358775Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:55:14.358812Z ERROR Node{id=22}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=23 leader=1 local=22 [INFO] [stdout] 2025-09-01T11:55:14.361338Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:14.361387Z INFO Node{id=22}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:14.363319Z INFO Node{id=22}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:14.363867Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:14.369019Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:14.419526Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727708992) } [INFO] [stdout] 2025-09-01T11:55:14.512593Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 5]) [INFO] [stdout] 2025-09-01T11:55:14.533027Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:14.537504Z INFO Node{id=16}: sequenced_broadcast: 88|Subscriber(3): Added, allow_drop: true, next_sequence: 876, min_allowed_seq: 876 [INFO] [stdout] 2025-09-01T11:55:14.548082Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 876) [INFO] [stdout] 2025-09-01T11:55:14.548218Z INFO Node{id=28}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:14.550273Z INFO Node{id=28}: sequenced_broadcast: 94|SequencedBroadcastWorker Started id=94 [INFO] [stdout] 2025-09-01T11:55:14.552332Z INFO Node{id=28}: sequenced_broadcast: 60|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:14.552350Z INFO Node{id=28}: sequenced_broadcast: 60|SequencedBroadcastWorker Stopped id=60 elapsed=28.084302276s iter=2 [INFO] [stdout] 2025-09-01T11:55:15.000073Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:15.000115Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=3 peers=4 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:15.004110Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:15.004206Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:15.006900Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:55:15.283996Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:15.285861Z INFO Node{id=16}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:15.323616Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:15.323660Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:55:15.327067Z INFO Node{id=27}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:15.327148Z INFO Node{id=27}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:15.329190Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:55:15.351891Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727715327) } [INFO] [stdout] 2025-09-01T11:55:15.351930Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=27 [INFO] [stdout] 2025-09-01T11:55:15.355640Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:15.355670Z INFO Node{id=21}: sharedstate::worker::sync_manager: Broadcasted 3 peers to 2 peers [INFO] [stdout] 2025-09-01T11:55:15.361136Z INFO Node{id=27}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=34 [INFO] [stdout] 2025-09-01T11:55:15.361600Z INFO Node{id=27}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:15.365265Z ERROR Node{id=27}:connect{target=21}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:15.365353Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:15.365381Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:55:15.365472Z INFO Node{id=27}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:15.368955Z INFO Node{id=27}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:15.369037Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(21) [INFO] [stdout] 2025-09-01T11:55:15.369089Z INFO Node{id=22}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:15.369155Z INFO Node{id=22}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:15.372049Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:55:15.372351Z ERROR ReadClient{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:15.380197Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:15.384258Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:15.406850Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:15.407181Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:15.411786Z INFO Node{id=24}: sequenced_broadcast: 82|Subscriber rejected, seq(876) < min_allowed(3466) [INFO] [stdout] 2025-09-01T11:55:15.496337Z ERROR ReadClient{remote=16}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:15.950707Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:15.994896Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:16.299985Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:16.300032Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=4 leader=1 [INFO] [stdout] 2025-09-01T11:55:16.302663Z INFO Node{id=16}:connect{target=4}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:16.302740Z INFO Node{id=16}:connect{target=4}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:16.307454Z INFO Node{id=4}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:55:16.375202Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:16.375249Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:55:16.377528Z INFO Node{id=27}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:16.377616Z INFO Node{id=27}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:16.379629Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:55:16.539531Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:16.545560Z INFO Node{id=24}: sequenced_broadcast: 82|Subscriber rejected, max_seq(5738) < seq(5739) [INFO] [stdout] 2025-09-01T11:55:16.573638Z INFO Node{id=24}: sequenced_broadcast: 82|Subscriber(2): Added, allow_drop: true, next_sequence: 5739, min_allowed_seq: 3692 [INFO] [stdout] 2025-09-01T11:55:16.592947Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 5739) [INFO] [stdout] 2025-09-01T11:55:16.593084Z INFO Node{id=9}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:16.595040Z INFO Node{id=9}: sequenced_broadcast: 95|SequencedBroadcastWorker Started id=95 [INFO] [stdout] 2025-09-01T11:55:16.597922Z INFO Node{id=9}: sequenced_broadcast: 73|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:16.597964Z INFO Node{id=9}: sequenced_broadcast: 73|SequencedBroadcastWorker Stopped id=73 elapsed=26.710369709s iter=128 [INFO] [stdout] 2025-09-01T11:55:16.696129Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:16.886554Z ERROR Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:16.889286Z INFO Node{id=20}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:16.889367Z INFO Node{id=20}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:16.889430Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:16.892239Z ERROR ReadClient{remote=20}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:16.959363Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:17.076354Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:17.076397Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 6 peers to 5 peers [INFO] [stdout] 2025-09-01T11:55:17.156613Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727715369) } [INFO] [stdout] 2025-09-01T11:55:17.162731Z INFO Node{id=22}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=1793 [INFO] [stdout] 2025-09-01T11:55:17.162793Z INFO Node{id=22}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:17.180623Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:17.213950Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727712699) } [INFO] [stdout] 2025-09-01T11:55:17.224228Z INFO Node{id=23}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=4525 [INFO] [stdout] 2025-09-01T11:55:17.224278Z INFO Node{id=23}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:17.269920Z ERROR Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:17.271645Z INFO Node{id=21}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:17.271712Z INFO Node{id=21}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:17.271793Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:17.274936Z ERROR ReadClient{remote=21}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:17.591463Z ERROR Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:17.593686Z INFO Node{id=10}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:17.593807Z INFO Node{id=10}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:17.593873Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:17.595586Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:17.685086Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727711881) } [INFO] [stdout] 2025-09-01T11:55:17.891909Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:17.891953Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=5 peers=6 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:55:17.894008Z INFO Node{id=20}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:17.894134Z INFO Node{id=20}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:17.896438Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:55:17.914636Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727717894) } [INFO] [stdout] 2025-09-01T11:55:17.921669Z INFO Node{id=20}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=27 [INFO] [stdout] 2025-09-01T11:55:17.921715Z INFO Node{id=20}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:17.931616Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:17.932859Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:17.941653Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:55:17.941691Z ERROR Node{id=20}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=22 leader=1 local=20 [INFO] [stdout] 2025-09-01T11:55:17.944534Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:17.944597Z INFO Node{id=20}:connect{target=22}:WritePeer{remote=22}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:17.945572Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:17.946846Z INFO Node{id=20}:connect{target=22}:ReadPeer{remote=22}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:17.949279Z ERROR ReadClient{remote=20}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:17.949478Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30]) [INFO] [stdout] 2025-09-01T11:55:17.951419Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:18.051496Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:18.051556Z INFO Node{id=1}: sharedstate::worker::sync_manager: Broadcasted 31 peers to 31 peers [INFO] [stdout] 2025-09-01T11:55:18.112739Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727712267) } [INFO] [stdout] 2025-09-01T11:55:18.155350Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:18.275024Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:18.275066Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=2 peers=3 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:55:18.277457Z INFO Node{id=21}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:18.277533Z INFO Node{id=21}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:18.279602Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:55:18.305736Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727718277) } [INFO] [stdout] 2025-09-01T11:55:18.315035Z INFO Node{id=21}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=38 [INFO] [stdout] 2025-09-01T11:55:18.315079Z INFO Node{id=21}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:18.327073Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:18.335409Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:55:18.335443Z ERROR Node{id=21}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=27 leader=1 local=21 [INFO] [stdout] 2025-09-01T11:55:18.337222Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:18.337275Z INFO Node{id=21}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:18.339176Z INFO Node{id=21}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:18.339508Z ERROR ReadClient{remote=21}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:18.343698Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:18.354534Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:18.430946Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:18.453232Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727712585) } [INFO] [stdout] 2025-09-01T11:55:18.596935Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:18.596978Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=7 peers=8 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:18.599356Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:18.599427Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:18.602138Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:55:18.746823Z ERROR Node{id=23}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:55:18.947548Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:18.947592Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=5 peers=6 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:18.949774Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:18.949865Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:18.952173Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:55:19.035837Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:19.043306Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 5, 24]) [INFO] [stdout] 2025-09-01T11:55:19.341313Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:19.341355Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=2 peers=3 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:19.343805Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:19.343894Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:19.346332Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:55:19.632438Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:19.640690Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 22, 9, 16, 7]) [INFO] [stdout] 2025-09-01T11:55:20.007052Z ERROR Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:20.011413Z INFO Node{id=8}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:20.011490Z INFO Node{id=8}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:20.011615Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:20.013997Z ERROR ReadClient{remote=8}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:20.438454Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:20.447346Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:20.447381Z ERROR Node{id=25}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:20.873997Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:20.882473Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:55:20.882507Z ERROR Node{id=22}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=16 leader=1 local=22 [INFO] [stdout] 2025-09-01T11:55:20.885322Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:20.885372Z INFO Node{id=22}:connect{target=16}:WritePeer{remote=16}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:20.890238Z INFO Node{id=22}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:20.892585Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:21.006017Z ERROR Node{id=28}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:55:21.008582Z WARN Node{id=28}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:21.008603Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:21.013720Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:21.013755Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=3 peers=4 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:21.015989Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:21.016062Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:21.018216Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:55:21.021102Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:21.307103Z ERROR Node{id=16}:connect{target=4}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:21.309749Z INFO Node{id=16}:connect{target=4}:WritePeer{remote=4}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:21.313955Z INFO Node{id=16}:connect{target=4}:ReadPeer{remote=4}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:21.314792Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(4) [INFO] [stdout] 2025-09-01T11:55:21.316795Z ERROR ReadClient{remote=16}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:21.361845Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727715004) } [INFO] [stdout] 2025-09-01T11:55:21.381959Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727707545) } [INFO] [stdout] 2025-09-01T11:55:21.382738Z ERROR Node{id=27}:connect{target=19}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:21.385376Z INFO Node{id=27}:connect{target=19}:WritePeer{remote=19}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:21.385463Z INFO Node{id=27}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:21.385531Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(19) [INFO] [stdout] 2025-09-01T11:55:21.387772Z ERROR ReadClient{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:21.516163Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:21.889711Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:21.889936Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=28 leader=1 [INFO] [stdout] 2025-09-01T11:55:21.892392Z INFO Node{id=22}:connect{target=28}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:21.892599Z INFO Node{id=22}:connect{target=28}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:21.894711Z INFO Node{id=28}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:55:21.914102Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727721892) } [INFO] [stdout] 2025-09-01T11:55:21.924110Z INFO Node{id=22}:connect{target=28}: sharedstate::worker::sync_manager: got pong from peer latency=32 [INFO] [stdout] 2025-09-01T11:55:21.924152Z INFO Node{id=22}:connect{target=28}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:21.933797Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:21.946045Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 28 } [INFO] [stdout] 2025-09-01T11:55:21.946079Z ERROR Node{id=22}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=28 leader=1 local=22 [INFO] [stdout] 2025-09-01T11:55:21.948589Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:21.948644Z INFO Node{id=22}:connect{target=28}:WritePeer{remote=28}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:21.950829Z INFO Node{id=22}:connect{target=28}:ReadPeer{remote=28}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:21.953021Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:21.956663Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:22.019600Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:22.019643Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:55:22.021753Z INFO Node{id=28}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:22.025900Z INFO Node{id=28}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:22.028159Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:55:22.054252Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:22.063849Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727722025) } [INFO] [stdout] 2025-09-01T11:55:22.064979Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 22, 9, 16, 7]) [INFO] [stdout] 2025-09-01T11:55:22.070370Z INFO Node{id=28}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=45 [INFO] [stdout] 2025-09-01T11:55:22.070410Z INFO Node{id=28}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:22.082605Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:22.094325Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:55:22.102986Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:22.103267Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:22.109795Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:22.109861Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:22.111654Z INFO Node{id=5}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:22.112578Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:22.112613Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 22, 9, 16, 7]) [INFO] [stdout] 2025-09-01T11:55:22.117057Z INFO Node{id=30}: sequenced_broadcast: 85|Subscriber rejected, seq(876) < min_allowed(4702) [INFO] [stdout] 2025-09-01T11:55:22.119617Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:22.121483Z INFO Node{id=6}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:22.132957Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:22.136375Z INFO Node{id=30}: sequenced_broadcast: 85|Subscriber rejected, max_seq(6752) < seq(6753) [INFO] [stdout] 2025-09-01T11:55:22.153039Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:22.156529Z INFO Node{id=30}: sequenced_broadcast: 85|Subscriber(2): Added, allow_drop: true, next_sequence: 6753, min_allowed_seq: 4705 [INFO] [stdout] 2025-09-01T11:55:22.165655Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:22.170781Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6753) [INFO] [stdout] 2025-09-01T11:55:22.170938Z INFO Node{id=28}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:22.173290Z INFO Node{id=28}: sequenced_broadcast: 96|SequencedBroadcastWorker Started id=96 [INFO] [stdout] 2025-09-01T11:55:22.176672Z INFO Node{id=28}: sequenced_broadcast: 94|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:22.176694Z INFO Node{id=28}: sequenced_broadcast: 94|SequencedBroadcastWorker Stopped id=94 elapsed=7.626401306s iter=2 [INFO] [stdout] 2025-09-01T11:55:22.177032Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:22.177052Z ERROR Node{id=31}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:22.182348Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:22.187032Z INFO Node{id=31}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:22.210855Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:22.210895Z ERROR Node{id=4}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:22.212879Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:22.216686Z INFO Node{id=4}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:22.229978Z ERROR Node{id=23}:connect{target=7}: sharedstate::worker::sync_manager: timeout waiting for path to leader [INFO] [stdout] 2025-09-01T11:55:22.233252Z INFO Node{id=23}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:22.233425Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(7) [INFO] [stdout] 2025-09-01T11:55:22.311430Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:22.316845Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:22.316908Z ERROR Node{id=19}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:22.317074Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:22.317105Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:55:22.319895Z INFO Node{id=16}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:22.319960Z INFO Node{id=16}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:22.320750Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:22.321958Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:55:22.322669Z INFO Node{id=19}:connect{target=4}:ReadPeer{remote=4}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:22.335794Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727722319) } [INFO] [stdout] 2025-09-01T11:55:22.335833Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=16 [INFO] [stdout] 2025-09-01T11:55:22.339016Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:22.339072Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 7 peers to 6 peers [INFO] [stdout] 2025-09-01T11:55:22.343232Z INFO Node{id=16}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=24 [INFO] [stdout] 2025-09-01T11:55:22.343305Z INFO Node{id=16}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:22.345747Z ERROR Node{id=16}:connect{target=23}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:22.345853Z INFO Node{id=16}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:22.346568Z INFO Node{id=16}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:22.347729Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(23) [INFO] [stdout] 2025-09-01T11:55:22.348166Z ERROR ReadClient{remote=16}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:22.355020Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:22.386885Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:22.388085Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:22.388119Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:55:22.390442Z INFO Node{id=27}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:22.390510Z INFO Node{id=27}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:22.393115Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:55:22.395119Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:22.404281Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:22.408978Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727722390) } [INFO] [stdout] 2025-09-01T11:55:22.409013Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=27 [INFO] [stdout] 2025-09-01T11:55:22.411296Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:22.411329Z INFO Node{id=10}: sharedstate::worker::sync_manager: Broadcasted 9 peers to 8 peers [INFO] [stdout] 2025-09-01T11:55:22.414517Z INFO Node{id=27}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=24 [INFO] [stdout] 2025-09-01T11:55:22.414586Z INFO Node{id=27}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:22.416902Z ERROR Node{id=27}:connect{target=10}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:22.417575Z INFO Node{id=27}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:22.417941Z INFO Node{id=27}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:22.418528Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(10) [INFO] [stdout] 2025-09-01T11:55:22.420864Z ERROR ReadClient{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:22.424082Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:22.450598Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:22.533979Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:22.539879Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:22.745494Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:22.950020Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:22.950066Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:55:22.951697Z INFO Node{id=22}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:22.951780Z INFO Node{id=22}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:22.953404Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:55:23.122231Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:23.122277Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:55:23.123988Z INFO Node{id=5}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:23.124061Z INFO Node{id=5}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:23.125547Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:55:23.134440Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:23.134482Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:55:23.134886Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727723124) } [INFO] [stdout] 2025-09-01T11:55:23.134991Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=5 [INFO] [stdout] 2025-09-01T11:55:23.135841Z INFO Node{id=6}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:23.135911Z INFO Node{id=6}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:23.136923Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:23.136947Z INFO Node{id=21}: sharedstate::worker::sync_manager: Broadcasted 4 peers to 3 peers [INFO] [stdout] 2025-09-01T11:55:23.137087Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:55:23.139538Z INFO Node{id=5}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:55:23.139579Z INFO Node{id=5}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:23.141210Z ERROR Node{id=5}:connect{target=21}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:23.141556Z INFO Node{id=5}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:23.142512Z INFO Node{id=5}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:23.142690Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(21) [INFO] [stdout] 2025-09-01T11:55:23.144268Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:23.146424Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:23.148106Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727723135) } [INFO] [stdout] 2025-09-01T11:55:23.152432Z INFO Node{id=6}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:55:23.152468Z INFO Node{id=6}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:23.157952Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:23.167424Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:55:23.167460Z ERROR Node{id=6}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=31 leader=1 local=6 [INFO] [stdout] 2025-09-01T11:55:23.168966Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:23.169351Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:23.169519Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:23.170437Z INFO Node{id=6}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:23.170505Z INFO Node{id=6}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:23.171651Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:23.199294Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:23.199343Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:55:23.200509Z INFO Node{id=31}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:23.200579Z INFO Node{id=31}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:23.201972Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:55:23.211726Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727723200) } [INFO] [stdout] 2025-09-01T11:55:23.215997Z INFO Node{id=31}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:55:23.216030Z INFO Node{id=31}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:23.220883Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:23.220925Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:55:23.222386Z INFO Node{id=4}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:23.222462Z INFO Node{id=4}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:23.222643Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:23.224091Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:55:23.227436Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:55:23.229397Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:55:23.232744Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:23.233437Z ERROR Node{id=13}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:23.234405Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6859, state_accept_seq: 6858, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:23.234840Z WARN Node{id=13}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:23.234858Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:23.234918Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:23.234936Z INFO Node{id=23}: sharedstate::worker::sync_manager: found connect target option_count=6 peers=7 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:55:23.235887Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:23.236323Z INFO Node{id=23}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:23.236479Z INFO Node{id=23}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:23.237326Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:23.242387Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=23 [INFO] [stdout] 2025-09-01T11:55:23.243981Z INFO Node{id=26}: sequenced_broadcast: 78|Subscriber(1): Added, allow_drop: true, next_sequence: 876, min_allowed_seq: 876 [INFO] [stdout] 2025-09-01T11:55:23.251052Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 876) [INFO] [stdout] 2025-09-01T11:55:23.251142Z INFO Node{id=31}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:23.252234Z INFO Node{id=31}: sequenced_broadcast: 97|SequencedBroadcastWorker Started id=97 [INFO] [stdout] 2025-09-01T11:55:23.253653Z INFO Node{id=31}: sequenced_broadcast: 92|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:23.253671Z INFO Node{id=31}: sequenced_broadcast: 92|SequencedBroadcastWorker Stopped id=92 elapsed=14.240971932s iter=1356 [INFO] [stdout] 2025-09-01T11:55:23.330512Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:23.330567Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:55:23.331775Z INFO Node{id=19}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:23.331847Z INFO Node{id=19}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:23.333196Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:23.343115Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727723331) } [INFO] [stdout] 2025-09-01T11:55:23.343152Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=19 [INFO] [stdout] 2025-09-01T11:55:23.345726Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:23.345755Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 7 peers to 6 peers [INFO] [stdout] 2025-09-01T11:55:23.347748Z INFO Node{id=19}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:55:23.347796Z INFO Node{id=19}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:23.348682Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:23.348713Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:55:23.349361Z ERROR Node{id=19}:connect{target=20}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:23.349511Z INFO Node{id=19}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:23.350434Z INFO Node{id=16}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:23.350502Z INFO Node{id=16}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:23.351334Z INFO Node{id=19}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:23.352457Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:55:23.352795Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:23.356157Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:23.362816Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(20) [INFO] [stdout] 2025-09-01T11:55:23.370958Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727723350) } [INFO] [stdout] 2025-09-01T11:55:23.374415Z INFO Node{id=16}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=24 [INFO] [stdout] 2025-09-01T11:55:23.374456Z INFO Node{id=16}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:23.388938Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:23.401125Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:55:23.401163Z ERROR Node{id=16}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 22, 9, 16, 7, 26] peer=26 leader=1 local=16 [INFO] [stdout] 2025-09-01T11:55:23.402877Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:23.404110Z INFO Node{id=16}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:23.404173Z INFO Node{id=16}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:23.405466Z ERROR ReadClient{remote=16}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:23.410865Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:23.419672Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:23.420511Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:23.420552Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:55:23.423149Z INFO Node{id=27}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:23.423230Z INFO Node{id=27}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:23.425010Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:55:23.465894Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:23.470406Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32]) [INFO] [stdout] 2025-09-01T11:55:23.601479Z ERROR Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:23.603091Z INFO Node{id=10}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:23.603168Z INFO Node{id=10}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:23.603245Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:23.604755Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:23.822495Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727718599) } [INFO] [stdout] 2025-09-01T11:55:23.952709Z ERROR Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:23.954441Z INFO Node{id=20}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:23.954529Z INFO Node{id=20}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:23.954621Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:23.956105Z ERROR ReadClient{remote=20}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:24.065657Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:24.066537Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727718949) } [INFO] [stdout] 2025-09-01T11:55:24.070314Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 18]) [INFO] [stdout] 2025-09-01T11:55:24.144753Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:24.144821Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:55:24.146509Z INFO Node{id=5}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:24.146679Z INFO Node{id=5}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:24.148263Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:55:24.175780Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:24.175827Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:55:24.177120Z INFO Node{id=6}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:24.177195Z INFO Node{id=6}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:24.178608Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:55:24.183635Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:24.247195Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:24.247239Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=28 leader=1 [INFO] [stdout] 2025-09-01T11:55:24.248484Z INFO Node{id=13}:connect{target=28}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:24.248569Z INFO Node{id=13}:connect{target=28}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:24.249999Z INFO Node{id=28}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:55:24.262302Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727724248) } [INFO] [stdout] 2025-09-01T11:55:24.266485Z INFO Node{id=13}:connect{target=28}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:24.266521Z INFO Node{id=13}:connect{target=28}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:24.275615Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:24.282636Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 28 } [INFO] [stdout] 2025-09-01T11:55:24.290971Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:24.292708Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:24.295208Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:24.295633Z INFO Node{id=28}: sequenced_broadcast: 96|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6753 [INFO] [stdout] 2025-09-01T11:55:24.304171Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:24.304254Z INFO Node{id=13}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:55:24.330908Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727719343) } [INFO] [stdout] 2025-09-01T11:55:24.335464Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=4992 [INFO] [stdout] 2025-09-01T11:55:24.335506Z INFO Node{id=21}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:24.340825Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:24.347441Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32]) [INFO] [stdout] 2025-09-01T11:55:24.368082Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:24.368127Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:55:24.370034Z INFO Node{id=19}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:24.370184Z INFO Node{id=19}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:24.371861Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:24.386358Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727724370) } [INFO] [stdout] 2025-09-01T11:55:24.393079Z INFO Node{id=19}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=23 [INFO] [stdout] 2025-09-01T11:55:24.393124Z INFO Node{id=19}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:24.402348Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:24.404610Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:24.404643Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:55:24.408138Z INFO Node{id=16}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:24.408229Z INFO Node{id=16}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:24.410054Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:55:24.412519Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:55:24.412552Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13] peer=13 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:55:24.418092Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:24.418499Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:24.420444Z INFO Node{id=19}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:24.420561Z INFO Node{id=19}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:24.422124Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:24.424201Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727724408) } [INFO] [stdout] 2025-09-01T11:55:24.429694Z INFO Node{id=16}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=21 [INFO] [stdout] 2025-09-01T11:55:24.429741Z INFO Node{id=16}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:24.440189Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:24.445196Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:55:24.445228Z ERROR Node{id=16}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 22, 9, 16, 7, 15] peer=15 leader=1 local=16 [INFO] [stdout] 2025-09-01T11:55:24.451048Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:24.452103Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:24.452290Z INFO Node{id=16}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:24.452366Z INFO Node{id=16}:connect{target=15}:WritePeer{remote=15}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:24.453690Z ERROR ReadClient{remote=16}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:24.514614Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:24.595035Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727716302) } [INFO] [stdout] 2025-09-01T11:55:24.604623Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:24.604663Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=8 peers=9 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:55:24.606345Z INFO Node{id=10}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:24.606489Z INFO Node{id=10}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:24.608195Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:55:24.621074Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727724606) } [INFO] [stdout] 2025-09-01T11:55:24.624729Z INFO Node{id=10}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:24.624805Z INFO Node{id=10}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:24.631933Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:24.640309Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:55:24.640347Z ERROR Node{id=10}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=27 leader=1 local=10 [INFO] [stdout] 2025-09-01T11:55:24.642861Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:24.642917Z INFO Node{id=10}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:24.644503Z INFO Node{id=10}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:24.646141Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:24.646836Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:24.697998Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727723423) } [INFO] [stdout] 2025-09-01T11:55:24.703132Z INFO Node{id=27}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=1280 [INFO] [stdout] 2025-09-01T11:55:24.703170Z INFO Node{id=27}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:24.703625Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727716377) } [INFO] [stdout] 2025-09-01T11:55:24.711491Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:24.718929Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:55:24.727979Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:24.729893Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 5316, state_accept_seq: 5315, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:24.732534Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:24.734104Z INFO Node{id=9}: sequenced_broadcast: 95|Subscriber rejected, seq(5316) < min_allowed(5739) [INFO] [stdout] 2025-09-01T11:55:24.737848Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:24.741653Z INFO Node{id=9}: sequenced_broadcast: 95|Subscriber(1): Added, allow_drop: true, next_sequence: 6859, min_allowed_seq: 5739 [INFO] [stdout] 2025-09-01T11:55:24.751092Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6859) [INFO] [stdout] 2025-09-01T11:55:24.751206Z INFO Node{id=27}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:24.752780Z INFO Node{id=27}: sequenced_broadcast: 98|SequencedBroadcastWorker Started id=98 [INFO] [stdout] 2025-09-01T11:55:24.754181Z INFO Node{id=27}: sequenced_broadcast: 83|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:24.754200Z INFO Node{id=27}: sequenced_broadcast: 83|SequencedBroadcastWorker Stopped id=83 elapsed=31.187364526s iter=2086 [INFO] [stdout] 2025-09-01T11:55:24.956290Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:24.956334Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=6 peers=7 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:55:24.958216Z INFO Node{id=20}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:24.958283Z INFO Node{id=20}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:24.959895Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:55:25.254257Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:25.279243Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:25.330773Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727721016) } [INFO] [stdout] 2025-09-01T11:55:25.336328Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=4320 [INFO] [stdout] 2025-09-01T11:55:25.336369Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:25.423196Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:25.423242Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:55:25.424691Z INFO Node{id=19}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:25.424786Z INFO Node{id=19}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:25.426303Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:25.441099Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727725424) } [INFO] [stdout] 2025-09-01T11:55:25.445447Z INFO Node{id=19}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=21 [INFO] [stdout] 2025-09-01T11:55:25.445485Z INFO Node{id=19}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:25.454731Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:25.454807Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:55:25.456125Z INFO Node{id=16}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:25.456203Z INFO Node{id=16}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:25.457684Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:25.457930Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:55:25.470475Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727725456) } [INFO] [stdout] 2025-09-01T11:55:25.470521Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=16 [INFO] [stdout] 2025-09-01T11:55:25.472001Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:25.474733Z INFO Node{id=16}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:25.474785Z INFO Node{id=16}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:25.476844Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:55:25.478088Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:25.484189Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:25.486459Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:25.488563Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:25.489748Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:55:25.489788Z ERROR Node{id=16}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=20 leader=1 local=16 [INFO] [stdout] 2025-09-01T11:55:25.491023Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:25.492334Z INFO Node{id=16}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:25.492442Z INFO Node{id=26}: sequenced_broadcast: 78|Subscriber(2): Added, allow_drop: true, next_sequence: 876, min_allowed_seq: 876 [INFO] [stdout] 2025-09-01T11:55:25.492581Z INFO Node{id=16}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:25.493637Z ERROR ReadClient{remote=16}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:25.495652Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:25.502439Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 876) [INFO] [stdout] 2025-09-01T11:55:25.502557Z INFO Node{id=19}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:25.503683Z INFO Node{id=19}: sequenced_broadcast: 99|SequencedBroadcastWorker Started id=99 [INFO] [stdout] 2025-09-01T11:55:25.505384Z INFO Node{id=19}: sequenced_broadcast: 71|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:25.505401Z INFO Node{id=19}: sequenced_broadcast: 71|SequencedBroadcastWorker Stopped id=71 elapsed=38.377678736s iter=3439 [INFO] [stdout] 2025-09-01T11:55:25.509131Z ERROR Node{id=32}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:25.509974Z WARN Node{id=32}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:25.509993Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:25.645531Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:25.645587Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=8 peers=9 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:25.647050Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:25.647125Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:25.649889Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:55:25.712928Z ERROR Node{id=8}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:55:26.110935Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 31) [INFO] [stdout] 2025-09-01T11:55:26.448126Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:26.452648Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:26.452682Z ERROR Node{id=25}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:26.494058Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:26.494104Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:55:26.495574Z INFO Node{id=16}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:26.495648Z INFO Node{id=16}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:26.497254Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:55:26.517429Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:26.517476Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:55:26.519664Z INFO Node{id=32}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:26.519730Z INFO Node{id=32}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:26.520988Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:26.715155Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727726495) } [INFO] [stdout] 2025-09-01T11:55:26.719818Z INFO Node{id=16}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=224 [INFO] [stdout] 2025-09-01T11:55:26.719857Z INFO Node{id=16}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:26.876176Z ERROR Node{id=31}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:55:26.877694Z WARN Node{id=31}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:26.877714Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:26.883105Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:26.932091Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:26.943662Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:55:26.956080Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727723222) } [INFO] [stdout] 2025-09-01T11:55:26.957751Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:26.958951Z INFO Node{id=25}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:26.959860Z INFO Node{id=4}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=3737 [INFO] [stdout] 2025-09-01T11:55:26.959888Z INFO Node{id=4}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:26.959965Z ERROR ReadClient{remote=25}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:27.101887Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: Ping(1756727723236) } [INFO] [stdout] 2025-09-01T11:55:27.106565Z INFO Node{id=23}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=3870 [INFO] [stdout] 2025-09-01T11:55:27.106603Z INFO Node{id=23}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:27.150615Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:27.169224Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:27.174226Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:27.180705Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:27.186892Z INFO Node{id=30}: sequenced_broadcast: 85|Subscriber rejected, seq(876) < min_allowed(4770) [INFO] [stdout] 2025-09-01T11:55:27.471720Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:27.474111Z INFO Node{id=30}: sequenced_broadcast: 85|Subscriber(3): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 4770 [INFO] [stdout] 2025-09-01T11:55:27.479028Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:27.479182Z INFO Node{id=16}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:27.480721Z INFO Node{id=16}: sequenced_broadcast: 100|SequencedBroadcastWorker Started id=100 [INFO] [stdout] 2025-09-01T11:55:27.482421Z INFO Node{id=16}: sequenced_broadcast: 88|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:27.482440Z INFO Node{id=16}: sequenced_broadcast: 88|SequencedBroadcastWorker Stopped id=88 elapsed=31.3444765s iter=5 [INFO] [stdout] 2025-09-01T11:55:27.482850Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:55:27.485237Z ERROR ReadClient{remote=7}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:55:27.486779Z ERROR Node{id=7}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:27.490020Z WARN Node{id=7}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:27.490040Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:27.510702Z ERROR Node{id=3}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:27.512141Z WARN Node{id=3}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:27.512161Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:27.653100Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:27.886497Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:27.886544Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:55:27.888091Z INFO Node{id=31}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:27.888170Z INFO Node{id=31}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:27.890006Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:55:27.901268Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727727888) } [INFO] [stdout] 2025-09-01T11:55:27.901310Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=31 [INFO] [stdout] 2025-09-01T11:55:27.903676Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:27.903706Z INFO Node{id=10}: sharedstate::worker::sync_manager: Broadcasted 10 peers to 9 peers [INFO] [stdout] 2025-09-01T11:55:27.906236Z INFO Node{id=31}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:27.906264Z INFO Node{id=31}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:27.908435Z ERROR Node{id=31}:connect{target=10}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:27.909173Z INFO Node{id=31}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:27.909869Z INFO Node{id=31}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:27.910227Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(10) [INFO] [stdout] 2025-09-01T11:55:27.911380Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:27.914364Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:27.952833Z ERROR Node{id=22}:connect{target=14}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:27.954370Z INFO Node{id=22}:connect{target=14}:WritePeer{remote=14}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:27.954448Z INFO Node{id=22}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:27.954518Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(14) [INFO] [stdout] 2025-09-01T11:55:27.956358Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:27.963365Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:27.963408Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=11 leader=1 [INFO] [stdout] 2025-09-01T11:55:27.964859Z INFO Node{id=25}:connect{target=11}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:27.964935Z INFO Node{id=25}:connect{target=11}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:27.966382Z INFO Node{id=11}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:55:27.976731Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727727964) } [INFO] [stdout] 2025-09-01T11:55:27.980753Z INFO Node{id=25}:connect{target=11}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:55:27.981013Z INFO Node{id=25}:connect{target=11}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:27.987742Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:27.992337Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 11 } [INFO] [stdout] 2025-09-01T11:55:27.999347Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:28.001370Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:28.002392Z INFO Node{id=11}: sequenced_broadcast: 93|Subscriber rejected, seq(876) < min_allowed(6864) [INFO] [stdout] 2025-09-01T11:55:28.003236Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:28.011656Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:28.013303Z INFO Node{id=11}: sequenced_broadcast: 93|Subscriber rejected, max_seq(8913) < seq(8916) [INFO] [stdout] 2025-09-01T11:55:28.029778Z INFO Node{id=11}: sequenced_broadcast: 93|Subscriber(1): Added, allow_drop: true, next_sequence: 8916, min_allowed_seq: 6868 [INFO] [stdout] 2025-09-01T11:55:28.036824Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 8916) [INFO] [stdout] 2025-09-01T11:55:28.036936Z INFO Node{id=25}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:28.038047Z INFO Node{id=25}: sequenced_broadcast: 101|SequencedBroadcastWorker Started id=101 [INFO] [stdout] 2025-09-01T11:55:28.040391Z INFO Node{id=25}: sequenced_broadcast: 89|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:28.040412Z INFO Node{id=25}: sequenced_broadcast: 89|SequencedBroadcastWorker Stopped id=89 elapsed=31.076235343s iter=2 [INFO] [stdout] 2025-09-01T11:55:28.081489Z ERROR ReadClient{remote=23}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:55:28.231115Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:28.236610Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:55:28.310243Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:28.313386Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 5, 24]) [INFO] [stdout] 2025-09-01T11:55:28.347042Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:28.347086Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 8 peers to 7 peers [INFO] [stdout] 2025-09-01T11:55:28.496808Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:28.496854Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:55:28.498140Z INFO Node{id=7}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:28.498220Z INFO Node{id=7}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:28.499678Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:55:28.517237Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727728498) } [INFO] [stdout] 2025-09-01T11:55:28.518354Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:28.518388Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:55:28.519964Z INFO Node{id=3}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:28.520029Z INFO Node{id=3}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:28.521568Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:55:28.522373Z INFO Node{id=7}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=24 [INFO] [stdout] 2025-09-01T11:55:28.522400Z INFO Node{id=7}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:28.531104Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:28.542815Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:28.547478Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:55:28.551691Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:28.553674Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:28.556040Z INFO Node{id=25}: sequenced_broadcast: 101|Subscriber rejected, seq(876) < min_allowed(8916) [INFO] [stdout] 2025-09-01T11:55:28.558946Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:28.559911Z INFO Node{id=25}: sequenced_broadcast: 101|Subscriber rejected, max_seq(9102) < seq(9105) [INFO] [stdout] 2025-09-01T11:55:28.576371Z INFO Node{id=25}: sequenced_broadcast: 101|Subscriber(1): Added, allow_drop: true, next_sequence: 9105, min_allowed_seq: 8916 [INFO] [stdout] 2025-09-01T11:55:28.584371Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 9105) [INFO] [stdout] 2025-09-01T11:55:28.584488Z INFO Node{id=7}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:28.585827Z INFO Node{id=7}: sequenced_broadcast: 102|SequencedBroadcastWorker Started id=102 [INFO] [stdout] 2025-09-01T11:55:28.587709Z INFO Node{id=7}: sequenced_broadcast: 74|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:28.587729Z INFO Node{id=7}: sequenced_broadcast: 74|SequencedBroadcastWorker Stopped id=74 elapsed=37.495448883s iter=49 [INFO] [stdout] 2025-09-01T11:55:28.590491Z INFO ReadClient{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:28.592245Z ERROR Node{id=26}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:28.592329Z ERROR Node{id=15}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:28.593714Z WARN Node{id=15}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:28.593732Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:28.594054Z WARN Node{id=26}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:28.594073Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:28.733688Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727724958) } [INFO] [stdout] 2025-09-01T11:55:28.738060Z INFO Node{id=20}:connect{target=19}: sharedstate::worker::sync_manager: got pong from peer latency=3780 [INFO] [stdout] 2025-09-01T11:55:28.738097Z INFO Node{id=20}:connect{target=19}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:28.765357Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:28.765399Z INFO Node{id=8}: sharedstate::worker::sync_manager: Broadcasted 4 peers to 3 peers [INFO] [stdout] 2025-09-01T11:55:28.912071Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:28.912116Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:55:28.913599Z INFO Node{id=31}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:28.913782Z INFO Node{id=31}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:28.914944Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:55:28.923593Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727728913) } [INFO] [stdout] 2025-09-01T11:55:28.923631Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=31 [INFO] [stdout] 2025-09-01T11:55:28.925515Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:28.925543Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 8 peers to 7 peers [INFO] [stdout] 2025-09-01T11:55:28.927792Z INFO Node{id=31}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=14 [INFO] [stdout] 2025-09-01T11:55:28.927820Z INFO Node{id=31}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:28.929109Z ERROR Node{id=31}:connect{target=23}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:28.929229Z INFO Node{id=31}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:28.930625Z INFO Node{id=31}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:28.930701Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(23) [INFO] [stdout] 2025-09-01T11:55:28.932557Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:28.934752Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:28.955862Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:28.955906Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:55:28.957043Z INFO Node{id=22}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:28.957105Z INFO Node{id=22}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:28.958018Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:55:28.967372Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727728957) } [INFO] [stdout] 2025-09-01T11:55:28.973248Z INFO Node{id=22}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:55:28.973285Z INFO Node{id=22}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:28.978479Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:28.983606Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:55:28.989849Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:28.991773Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:28.994207Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:28.994469Z INFO Node{id=12}: sequenced_broadcast: 84|Subscriber rejected, seq(876) < min_allowed(4811) [INFO] [stdout] 2025-09-01T11:55:28.999212Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:29.002846Z INFO Node{id=12}: sequenced_broadcast: 84|Subscriber(1): Added, allow_drop: true, next_sequence: 6859, min_allowed_seq: 4811 [INFO] [stdout] 2025-09-01T11:55:29.008838Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6859) [INFO] [stdout] 2025-09-01T11:55:29.008938Z INFO Node{id=22}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:29.010191Z INFO Node{id=22}: sequenced_broadcast: 103|SequencedBroadcastWorker Started id=103 [INFO] [stdout] 2025-09-01T11:55:29.012171Z INFO Node{id=22}: sequenced_broadcast: 72|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:29.012191Z INFO Node{id=22}: sequenced_broadcast: 72|SequencedBroadcastWorker Stopped id=72 elapsed=41.792101603s iter=475 [INFO] [stdout] 2025-09-01T11:55:29.044804Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:29.148275Z ERROR Node{id=5}:connect{target=17}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:29.149589Z INFO Node{id=5}:connect{target=17}:WritePeer{remote=17}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:29.149669Z INFO Node{id=5}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:29.151210Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:29.157005Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(17) [INFO] [stdout] 2025-09-01T11:55:29.178953Z ERROR Node{id=6}:connect{target=18}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:29.180574Z INFO Node{id=6}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:29.180655Z INFO Node{id=6}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:29.180721Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(18) [INFO] [stdout] 2025-09-01T11:55:29.184625Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:29.219703Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727728520) } [INFO] [stdout] 2025-09-01T11:55:29.223635Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:29.226154Z INFO Node{id=3}:connect{target=19}: sharedstate::worker::sync_manager: got pong from peer latency=706 [INFO] [stdout] 2025-09-01T11:55:29.226186Z INFO Node{id=3}:connect{target=19}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:29.231737Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 19 } [INFO] [stdout] 2025-09-01T11:55:29.235881Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:29.237377Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:29.238706Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeRecovery(RecoverableStateDetails { id: 15966214309367731050, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:29.240341Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:29.241549Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 19 } [INFO] [stdout] 2025-09-01T11:55:29.241594Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:29.243782Z INFO Node{id=19}: sequenced_broadcast: 99|Subscriber(1): Added, allow_drop: true, next_sequence: 876, min_allowed_seq: 876 [INFO] [stdout] 2025-09-01T11:55:29.249080Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:29.251135Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:29.252781Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 876) [INFO] [stdout] 2025-09-01T11:55:29.252877Z INFO Node{id=20}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:29.253463Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:29.254388Z INFO Node{id=20}: sequenced_broadcast: 104|SequencedBroadcastWorker Started id=104 [INFO] [stdout] 2025-09-01T11:55:29.254898Z INFO Node{id=19}: sequenced_broadcast: 99|Subscriber(2): Added, allow_drop: true, next_sequence: 876, min_allowed_seq: 876 [INFO] [stdout] 2025-09-01T11:55:29.256421Z INFO Node{id=20}: sequenced_broadcast: 52|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:29.256441Z INFO Node{id=20}: sequenced_broadcast: 52|SequencedBroadcastWorker Stopped id=52 elapsed=42.797011477s iter=2 [INFO] [stdout] 2025-09-01T11:55:29.260810Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 876) [INFO] [stdout] 2025-09-01T11:55:29.260874Z INFO Node{id=3}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:55:29.419508Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727725647) } [INFO] [stdout] 2025-09-01T11:55:29.423810Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=3776 [INFO] [stdout] 2025-09-01T11:55:29.423847Z INFO Node{id=10}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:29.479794Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:29.482440Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 18]) [INFO] [stdout] 2025-09-01T11:55:29.600386Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:29.600431Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:55:29.601685Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:29.601715Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=11 leader=1 [INFO] [stdout] 2025-09-01T11:55:29.602386Z INFO Node{id=26}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:29.602453Z INFO Node{id=26}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:29.605437Z INFO Node{id=15}:connect{target=11}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:29.605501Z INFO Node{id=15}:connect{target=11}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:29.606009Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:55:29.606923Z INFO Node{id=11}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:55:29.635828Z ERROR Node{id=4}:connect{target=24}:WritePeer{remote=24}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:55:29.932605Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:29.932651Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:55:29.933968Z INFO Node{id=31}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:29.934031Z INFO Node{id=31}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:29.935516Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:55:30.158114Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:30.158162Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:55:30.160011Z INFO Node{id=5}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:30.160087Z INFO Node{id=5}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:30.161940Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:55:30.171254Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727730160) } [INFO] [stdout] 2025-09-01T11:55:30.175641Z INFO Node{id=5}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:55:30.175674Z INFO Node{id=5}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:30.182218Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:30.182260Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:55:30.183331Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:30.183473Z INFO Node{id=6}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:30.183546Z INFO Node{id=6}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:30.185315Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:55:30.191345Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:55:30.195030Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:30.196952Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6859, state_accept_seq: 6858, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:30.199240Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:30.199450Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:30.202903Z INFO Node{id=29}: sequenced_broadcast: 90|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 4770 [INFO] [stdout] 2025-09-01T11:55:30.211533Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:30.211657Z INFO Node{id=5}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:30.212867Z INFO Node{id=5}: sequenced_broadcast: 105|SequencedBroadcastWorker Started id=105 [INFO] [stdout] 2025-09-01T11:55:30.214239Z INFO Node{id=5}: sequenced_broadcast: 77|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:30.214257Z INFO Node{id=5}: sequenced_broadcast: 77|SequencedBroadcastWorker Stopped id=77 elapsed=37.948148346s iter=2975 [INFO] [stdout] 2025-09-01T11:55:30.219125Z ERROR Node{id=24}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:30.220522Z WARN Node{id=24}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:30.220585Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:30.250312Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:30.298527Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727726519) } [INFO] [stdout] 2025-09-01T11:55:30.304974Z INFO Node{id=32}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=3785 [INFO] [stdout] 2025-09-01T11:55:30.305016Z INFO Node{id=32}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:30.338283Z ERROR Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: timeout waiting for path to leader [INFO] [stdout] 2025-09-01T11:55:30.339886Z INFO Node{id=8}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:30.339961Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(1) [INFO] [stdout] 2025-09-01T11:55:30.676476Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727729605) } [INFO] [stdout] 2025-09-01T11:55:30.681123Z INFO Node{id=15}:connect{target=11}: sharedstate::worker::sync_manager: got pong from peer latency=1076 [INFO] [stdout] 2025-09-01T11:55:30.681156Z INFO Node{id=15}:connect{target=11}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:30.820231Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:30.825861Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:55:30.825902Z ERROR Node{id=4}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=24 leader=1 local=4 [INFO] [stdout] 2025-09-01T11:55:30.827437Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:30.828632Z INFO Node{id=4}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:30.984410Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:30.990530Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:55:31.229630Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:31.229677Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:55:31.231116Z INFO Node{id=24}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:31.231184Z INFO Node{id=24}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:31.232390Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:55:31.261503Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:31.264083Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeRecovery(RecoverableStateDetails { id: 634985037238024698, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:31.265875Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:31.266194Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:31.269469Z INFO Node{id=16}: sequenced_broadcast: 100|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:31.275850Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:31.275951Z INFO Node{id=23}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:31.277267Z INFO Node{id=23}: sequenced_broadcast: 106|SequencedBroadcastWorker Started id=106 [INFO] [stdout] 2025-09-01T11:55:31.281253Z INFO Node{id=23}: sequenced_broadcast: 55|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:31.281275Z INFO Node{id=23}: sequenced_broadcast: 55|SequencedBroadcastWorker Stopped id=55 elapsed=44.821669881s iter=2 [INFO] [stdout] 2025-09-01T11:55:31.342298Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:31.342340Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=3 peers=4 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:31.345563Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:31.345637Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:31.347099Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:55:31.729393Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:55:31.757945Z ERROR Node{id=22}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:31.759386Z WARN Node{id=22}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:31.759406Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:31.828935Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:31.828980Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:55:31.830439Z INFO Node{id=4}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:31.830508Z INFO Node{id=4}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:31.831866Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:55:31.843353Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727731830) } [INFO] [stdout] 2025-09-01T11:55:31.843386Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=4 [INFO] [stdout] 2025-09-01T11:55:31.847529Z INFO Node{id=4}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:55:31.847571Z INFO Node{id=4}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:31.854619Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:31.860187Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:55:31.860216Z ERROR Node{id=4}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23] peer=23 leader=1 local=4 [INFO] [stdout] 2025-09-01T11:55:31.862134Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:31.862193Z INFO Node{id=4}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:31.864102Z INFO Node{id=4}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:31.865482Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:31.866291Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:32.098835Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:32.201104Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:32.207231Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:32.212931Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeRecovery(RecoverableStateDetails { id: 9391859352936412299, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:32.216460Z ERROR Node{id=21}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:32.217754Z WARN Node{id=21}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:32.217791Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:32.218628Z INFO Node{id=21}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:32.552566Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:32.559171Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 11 } [INFO] [stdout] 2025-09-01T11:55:32.574118Z INFO Node{id=14}: sequenced_broadcast: 2|Subscriber(75): channel closed, dropping [INFO] [stdout] 2025-09-01T11:55:32.583085Z INFO Node{id=14}: sequenced_broadcast: 1|Subscriber(75): channel closed, dropping [INFO] [stdout] 2025-09-01T11:55:32.766968Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:32.767013Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:55:32.768494Z INFO Node{id=22}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:32.768577Z INFO Node{id=22}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:32.770180Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:55:32.780987Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727732768) } [INFO] [stdout] 2025-09-01T11:55:32.787659Z INFO Node{id=22}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=19 [INFO] [stdout] 2025-09-01T11:55:32.787705Z INFO Node{id=22}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:32.795681Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:32.801788Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:55:32.801820Z ERROR Node{id=22}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=15 leader=1 local=22 [INFO] [stdout] 2025-09-01T11:55:32.803319Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:32.803373Z INFO Node{id=22}:connect{target=15}:WritePeer{remote=15}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:32.807111Z INFO Node{id=22}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:32.808780Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:32.810191Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:32.866074Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:32.866119Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:55:32.867949Z INFO Node{id=4}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:32.868019Z INFO Node{id=4}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:32.869403Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:55:33.034297Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727722951) } [INFO] [stdout] 2025-09-01T11:55:33.220082Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:33.220127Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=3 peers=4 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:55:33.221941Z INFO Node{id=21}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:33.222016Z INFO Node{id=21}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:33.223390Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:55:33.289289Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:33.293090Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:33.293118Z ERROR Node{id=12}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:33.300270Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:33.470989Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:33.552982Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:33.560128Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:55:33.756667Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727724177) } [INFO] [stdout] 2025-09-01T11:55:33.804924Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:33.804969Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=11 leader=1 [INFO] [stdout] 2025-09-01T11:55:33.806392Z INFO Node{id=22}:connect{target=11}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:33.806467Z INFO Node{id=22}:connect{target=11}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:33.808122Z INFO Node{id=11}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:55:33.928585Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:33.928630Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 9 peers to 8 peers [INFO] [stdout] 2025-09-01T11:55:33.974867Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727724146) } [INFO] [stdout] 2025-09-01T11:55:34.075046Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:34.281776Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727730183) } [INFO] [stdout] 2025-09-01T11:55:34.284164Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:34.289633Z INFO Node{id=6}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=4106 [INFO] [stdout] 2025-09-01T11:55:34.289677Z INFO Node{id=6}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:34.295325Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:34.298869Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28]) [INFO] [stdout] 2025-09-01T11:55:34.300558Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727731231) } [INFO] [stdout] 2025-09-01T11:55:34.306947Z INFO Node{id=24}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=3075 [INFO] [stdout] 2025-09-01T11:55:34.306986Z INFO Node{id=24}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:34.349661Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:34.357143Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727733222) } [INFO] [stdout] 2025-09-01T11:55:34.361679Z INFO Node{id=21}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=1139 [INFO] [stdout] 2025-09-01T11:55:34.361713Z INFO Node{id=21}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:34.370300Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:34.377932Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 5 } [INFO] [stdout] 2025-09-01T11:55:34.385933Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:34.387417Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeRecovery(RecoverableStateDetails { id: 9391859352936412299, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:34.388567Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:34.389781Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:34.391704Z INFO Node{id=5}: sequenced_broadcast: 105|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:34.402169Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:34.402281Z INFO Node{id=21}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:34.403876Z INFO Node{id=21}: sequenced_broadcast: 107|SequencedBroadcastWorker Started id=107 [INFO] [stdout] 2025-09-01T11:55:34.405556Z INFO Node{id=21}: sequenced_broadcast: 53|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:34.405575Z INFO Node{id=21}: sequenced_broadcast: 53|SequencedBroadcastWorker Stopped id=53 elapsed=47.946087116s iter=2 [INFO] [stdout] 2025-09-01T11:55:34.448368Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:34.459153Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:55:34.459190Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=24 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:55:34.461943Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:34.463349Z INFO Node{id=32}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:34.463424Z INFO Node{id=32}:connect{target=24}:WritePeer{remote=24}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:34.465024Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:34.604303Z ERROR Node{id=26}:connect{target=18}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:34.607917Z INFO Node{id=26}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:34.607992Z INFO Node{id=26}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:34.609124Z ERROR ReadClient{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:34.625833Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(18) [INFO] [stdout] 2025-09-01T11:55:34.721187Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:34.727100Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:34.730604Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 5, 24, 9]) [INFO] [stdout] 2025-09-01T11:55:34.935961Z ERROR Node{id=31}:connect{target=18}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:34.937300Z INFO Node{id=31}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:34.937395Z INFO Node{id=31}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:34.937472Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(18) [INFO] [stdout] 2025-09-01T11:55:34.939162Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:35.044865Z ERROR ReadClient{remote=8}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:55:35.052980Z INFO WriteClient{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:35.295115Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:35.449943Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727731345) } [INFO] [stdout] 2025-09-01T11:55:35.453937Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=4108 [INFO] [stdout] 2025-09-01T11:55:35.453972Z INFO Node{id=8}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:35.458311Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:35.459877Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:35.460043Z INFO Node{id=11}: sequenced_broadcast: 93|Subscriber rejected, seq(876) < min_allowed(9091) [INFO] [stdout] 2025-09-01T11:55:35.464068Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:35.464419Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:35.464447Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:55:35.465843Z INFO Node{id=32}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:35.465906Z INFO Node{id=32}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:35.467412Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:35.479349Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:35.609875Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:35.616968Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30]) [INFO] [stdout] 2025-09-01T11:55:35.627713Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:35.627774Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:55:35.629556Z INFO Node{id=26}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:35.629627Z INFO Node{id=26}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:35.631219Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:55:35.643729Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727735629) } [INFO] [stdout] 2025-09-01T11:55:35.650334Z INFO Node{id=26}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=21 [INFO] [stdout] 2025-09-01T11:55:35.650376Z INFO Node{id=26}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:35.659954Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:35.673817Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:35.676062Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:55:35.676094Z ERROR Node{id=26}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=6 leader=1 local=26 [INFO] [stdout] 2025-09-01T11:55:35.677859Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:35.679235Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:35.682311Z INFO Node{id=26}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:35.682386Z INFO Node{id=26}:connect{target=6}:WritePeer{remote=6}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:35.684128Z ERROR ReadClient{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:35.759106Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:35.767493Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:35.767607Z ERROR Node{id=19}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:35.939691Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:35.939737Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=28 leader=1 [INFO] [stdout] 2025-09-01T11:55:35.946383Z INFO Node{id=31}:connect{target=28}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:35.946463Z INFO Node{id=31}:connect{target=28}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:35.949300Z INFO Node{id=28}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:55:35.976049Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727735946) } [INFO] [stdout] 2025-09-01T11:55:35.982546Z INFO Node{id=31}:connect{target=28}: sharedstate::worker::sync_manager: got pong from peer latency=36 [INFO] [stdout] 2025-09-01T11:55:35.982592Z INFO Node{id=31}:connect{target=28}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:35.995334Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:36.009107Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 28 } [INFO] [stdout] 2025-09-01T11:55:36.020479Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:36.026030Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:36.030651Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:36.033228Z INFO Node{id=28}: sequenced_broadcast: 96|Subscriber rejected, seq(876) < min_allowed(6753) [INFO] [stdout] 2025-09-01T11:55:36.038224Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:36.044075Z INFO Node{id=28}: sequenced_broadcast: 96|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6753 [INFO] [stdout] 2025-09-01T11:55:36.053826Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:36.053948Z INFO Node{id=31}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:36.055469Z INFO Node{id=31}: sequenced_broadcast: 108|SequencedBroadcastWorker Started id=108 [INFO] [stdout] 2025-09-01T11:55:36.057582Z INFO Node{id=31}: sequenced_broadcast: 97|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:36.057610Z INFO Node{id=31}: sequenced_broadcast: 97|SequencedBroadcastWorker Stopped id=97 elapsed=12.805356199s iter=2 [INFO] [stdout] 2025-09-01T11:55:36.681184Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:36.681229Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:55:36.682750Z INFO Node{id=26}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:36.682840Z INFO Node{id=26}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:36.684487Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:55:36.702639Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727736682) } [INFO] [stdout] 2025-09-01T11:55:36.702676Z INFO Node{id=8}: sharedstate::worker::sync_manager: Discovered Peer client_id=26 [INFO] [stdout] 2025-09-01T11:55:36.705967Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:36.705997Z INFO Node{id=8}: sharedstate::worker::sync_manager: Broadcasted 5 peers to 4 peers [INFO] [stdout] 2025-09-01T11:55:36.709152Z INFO Node{id=26}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=27 [INFO] [stdout] 2025-09-01T11:55:36.709186Z INFO Node{id=26}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:36.712019Z ERROR Node{id=26}:connect{target=8}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:36.712366Z INFO Node{id=26}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:36.713244Z INFO Node{id=26}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:36.715070Z ERROR ReadClient{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:36.720462Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:36.730176Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(8) [INFO] [stdout] 2025-09-01T11:55:36.946289Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:37.154483Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727732868) } [INFO] [stdout] 2025-09-01T11:55:37.164597Z INFO Node{id=4}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=4296 [INFO] [stdout] 2025-09-01T11:55:37.164640Z INFO Node{id=4}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:37.697888Z ERROR Node{id=7}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:55:37.732003Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:37.732048Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:55:37.733779Z INFO Node{id=26}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:37.733847Z INFO Node{id=26}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:37.735563Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:55:37.748899Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727737733) } [INFO] [stdout] 2025-09-01T11:55:37.753410Z INFO Node{id=26}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:55:37.753444Z INFO Node{id=26}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:37.759380Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:37.767479Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:55:37.773485Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:37.774078Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:37.776754Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:37.779047Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:37.779623Z INFO Node{id=9}: sequenced_broadcast: 95|Subscriber rejected, seq(876) < min_allowed(5739) [INFO] [stdout] 2025-09-01T11:55:37.785945Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:37.788983Z INFO Node{id=9}: sequenced_broadcast: 95|Subscriber(2): Added, allow_drop: true, next_sequence: 6859, min_allowed_seq: 5739 [INFO] [stdout] 2025-09-01T11:55:37.796407Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6859) [INFO] [stdout] 2025-09-01T11:55:37.796504Z INFO Node{id=26}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:37.797897Z INFO Node{id=26}: sequenced_broadcast: 109|SequencedBroadcastWorker Started id=109 [INFO] [stdout] 2025-09-01T11:55:37.799431Z INFO Node{id=26}: sequenced_broadcast: 78|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:37.799449Z INFO Node{id=26}: sequenced_broadcast: 78|SequencedBroadcastWorker Stopped id=78 elapsed=45.073534241s iter=4 [INFO] [stdout] 2025-09-01T11:55:37.803584Z ERROR Node{id=19}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:37.805751Z WARN Node{id=19}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:37.805791Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:37.845630Z WARN Node{id=7}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:37.845662Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:37.847286Z ERROR ReadClient{remote=7}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:37.859497Z INFO Node{id=25}: sequenced_broadcast: 1|Subscriber(101): channel closed, dropping [INFO] [stdout] 2025-09-01T11:55:37.999446Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:38.001642Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:38.002835Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:38.004624Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeRecovery(RecoverableStateDetails { id: 15335810314635339748, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:38.007439Z ERROR Node{id=10}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:38.008596Z WARN Node{id=10}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:38.008614Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:38.009933Z INFO Node{id=10}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:38.012192Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727733806) } [INFO] [stdout] 2025-09-01T11:55:38.020572Z INFO Node{id=22}:connect{target=11}: sharedstate::worker::sync_manager: got pong from peer latency=4214 [INFO] [stdout] 2025-09-01T11:55:38.020613Z INFO Node{id=22}:connect{target=11}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:38.157964Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:38.160346Z ERROR Node{id=18}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:55:38.273615Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:38.314565Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:38.431577Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:38.432676Z ERROR Node{id=17}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:55:38.514964Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:38.632368Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:38.649561Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:38.653863Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:55:38.653889Z ERROR Node{id=6}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=32 leader=1 local=6 [INFO] [stdout] 2025-09-01T11:55:38.654627Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:38.654751Z INFO Node{id=6}:connect{target=32}:WritePeer{remote=32}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:38.655958Z INFO Node{id=6}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:38.656825Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:38.668972Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:38.672311Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:55:38.777061Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727729602) } [INFO] [stdout] 2025-09-01T11:55:38.814980Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:38.815104Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:55:38.816218Z INFO Node{id=19}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:38.816284Z INFO Node{id=19}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:38.817096Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:38.817245Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:38.820105Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:38.820128Z ERROR Node{id=12}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:38.823922Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727738816) } [INFO] [stdout] 2025-09-01T11:55:38.823947Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=19 [INFO] [stdout] 2025-09-01T11:55:38.825216Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:38.825242Z INFO Node{id=21}: sharedstate::worker::sync_manager: Broadcasted 5 peers to 4 peers [INFO] [stdout] 2025-09-01T11:55:38.826430Z INFO Node{id=19}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=10 [INFO] [stdout] 2025-09-01T11:55:38.826457Z INFO Node{id=19}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:38.827787Z ERROR Node{id=19}:connect{target=21}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:38.828585Z INFO Node{id=19}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:38.828728Z INFO Node{id=19}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:38.829531Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:38.832188Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:38.832546Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(21) [INFO] [stdout] 2025-09-01T11:55:38.852939Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:38.852983Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:55:38.853935Z INFO Node{id=7}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:38.854088Z INFO Node{id=7}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:38.854860Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:55:38.903002Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727729934) } [INFO] [stdout] 2025-09-01T11:55:38.969850Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:55:38.992669Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:38.995082Z INFO Node{id=11}: sequenced_broadcast: 93|Subscriber(2): Added, allow_drop: true, next_sequence: 11852, min_allowed_seq: 9804 [INFO] [stdout] 2025-09-01T11:55:38.998374Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 11852) [INFO] [stdout] 2025-09-01T11:55:38.998458Z INFO Node{id=15}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:38.999169Z INFO Node{id=15}: sequenced_broadcast: 110|SequencedBroadcastWorker Started id=110 [INFO] [stdout] 2025-09-01T11:55:38.999680Z INFO Node{id=15}: sequenced_broadcast: 91|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:38.999697Z INFO Node{id=15}: sequenced_broadcast: 91|SequencedBroadcastWorker Stopped id=91 elapsed=32.662528509s iter=2 [INFO] [stdout] 2025-09-01T11:55:39.009498Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:39.009541Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=9 peers=10 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:55:39.010604Z INFO Node{id=10}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:39.010689Z INFO Node{id=10}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:39.013089Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:55:39.021433Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727739010) } [INFO] [stdout] 2025-09-01T11:55:39.024159Z INFO Node{id=10}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=14 [INFO] [stdout] 2025-09-01T11:55:39.024187Z INFO Node{id=10}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:39.030844Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:39.034862Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:55:39.039441Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:39.041400Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeRecovery(RecoverableStateDetails { id: 15335810314635339748, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:39.042386Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:39.042536Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:39.045416Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:39.046044Z INFO Node{id=31}: sequenced_broadcast: 108|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:39.046863Z INFO Node{id=12}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:39.047202Z ERROR ReadClient{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:39.051100Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:39.051194Z INFO Node{id=10}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:39.052431Z INFO Node{id=10}: sequenced_broadcast: 111|SequencedBroadcastWorker Started id=111 [INFO] [stdout] 2025-09-01T11:55:39.053264Z INFO Node{id=10}: sequenced_broadcast: 42|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:39.053281Z INFO Node{id=10}: sequenced_broadcast: 42|SequencedBroadcastWorker Stopped id=42 elapsed=52.597578842s iter=2 [INFO] [stdout] 2025-09-01T11:55:39.170816Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:39.170861Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:55:39.171833Z INFO Node{id=18}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:39.171898Z INFO Node{id=18}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:39.173188Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:55:39.208213Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727739171) } [INFO] [stdout] 2025-09-01T11:55:39.214014Z INFO Node{id=18}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=43 [INFO] [stdout] 2025-09-01T11:55:39.214055Z INFO Node{id=18}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:39.234068Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:39.242966Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:39.247233Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:39.252256Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:55:39.279263Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:39.283691Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:39.287044Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 11852, state_accept_seq: 11851, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:39.290269Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:39.294153Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:39.294179Z ERROR Node{id=14}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:39.296017Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:39.297730Z INFO Node{id=14}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:39.320954Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:39.325377Z INFO Node{id=29}: sequenced_broadcast: 90|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 4770 [INFO] [stdout] 2025-09-01T11:55:39.332591Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:39.332700Z INFO Node{id=18}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:39.335878Z INFO Node{id=18}: sequenced_broadcast: 112|SequencedBroadcastWorker Started id=112 [INFO] [stdout] 2025-09-01T11:55:39.337819Z INFO Node{id=18}: sequenced_broadcast: 69|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:39.337839Z INFO Node{id=18}: sequenced_broadcast: 69|SequencedBroadcastWorker Stopped id=69 elapsed=52.459241215s iter=6024 [INFO] [stdout] 2025-09-01T11:55:39.339879Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:55:39.342609Z ERROR Node{id=2}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:39.342691Z ERROR Node{id=11}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:39.342885Z WARN Node{id=11}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:39.342903Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:39.344223Z WARN Node{id=2}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:39.344242Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:39.356916Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:39.439016Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:39.439062Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:55:39.440131Z INFO Node{id=17}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:39.440210Z INFO Node{id=17}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:39.441473Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:55:39.453916Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727739440) } [INFO] [stdout] 2025-09-01T11:55:39.456737Z INFO Node{id=17}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:55:39.456792Z INFO Node{id=17}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:39.464688Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:39.476346Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:39.481164Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:55:39.482545Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:39.490943Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 11944, state_accept_seq: 11943, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:39.493161Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:39.498712Z INFO Node{id=15}: sequenced_broadcast: 110|Subscriber(1): Added, allow_drop: true, next_sequence: 11852, min_allowed_seq: 11852 [INFO] [stdout] 2025-09-01T11:55:39.504865Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 11852) [INFO] [stdout] 2025-09-01T11:55:39.504967Z INFO Node{id=17}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:39.508168Z INFO Node{id=17}: sequenced_broadcast: 113|SequencedBroadcastWorker Started id=113 [INFO] [stdout] 2025-09-01T11:55:39.509559Z INFO Node{id=17}: sequenced_broadcast: 70|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:39.509578Z INFO Node{id=17}: sequenced_broadcast: 70|SequencedBroadcastWorker Stopped id=70 elapsed=52.606884429s iter=6096 [INFO] [stdout] 2025-09-01T11:55:39.623537Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:39.628676Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:55:39.649019Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727738854) } [INFO] [stdout] 2025-09-01T11:55:39.652561Z INFO Node{id=7}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=798 [INFO] [stdout] 2025-09-01T11:55:39.652687Z INFO Node{id=7}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:39.655904Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:39.655937Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:55:39.657671Z INFO Node{id=6}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:39.657745Z INFO Node{id=6}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:39.659253Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:55:39.672895Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727739657) } [INFO] [stdout] 2025-09-01T11:55:39.672933Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=6 [INFO] [stdout] 2025-09-01T11:55:39.675042Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:39.675081Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 10 peers to 9 peers [INFO] [stdout] 2025-09-01T11:55:39.677025Z INFO Node{id=6}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:55:39.677055Z INFO Node{id=6}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:39.678834Z ERROR Node{id=6}:connect{target=23}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:39.679590Z INFO Node{id=6}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:39.680519Z INFO Node{id=6}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:39.680730Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(23) [INFO] [stdout] 2025-09-01T11:55:39.683025Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:39.686438Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:39.708029Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:39.713279Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:55:39.713323Z ERROR Node{id=7}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=24 leader=1 local=7 [INFO] [stdout] 2025-09-01T11:55:39.714515Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:39.714572Z INFO Node{id=7}:connect{target=24}:WritePeer{remote=24}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:39.716180Z INFO Node{id=7}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:39.716644Z ERROR ReadClient{remote=7}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:39.719124Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:39.833684Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:39.833727Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:55:39.834947Z INFO Node{id=19}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:39.835013Z INFO Node{id=19}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:39.835798Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:39.842938Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727739835) } [INFO] [stdout] 2025-09-01T11:55:39.842966Z INFO Node{id=8}: sharedstate::worker::sync_manager: Discovered Peer client_id=19 [INFO] [stdout] 2025-09-01T11:55:39.845983Z INFO Node{id=19}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=10 [INFO] [stdout] 2025-09-01T11:55:39.846012Z INFO Node{id=19}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:39.850248Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:39.853904Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:55:39.853929Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=8 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:55:39.855491Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:39.856526Z INFO Node{id=19}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:39.856613Z INFO Node{id=19}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:39.857056Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:39.859240Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:40.052103Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:40.052150Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:55:40.052934Z ERROR Node{id=12}:connect{target=13}: sharedstate::worker::sync_manager: io error connecting to peer error=Custom { kind: InvalidInput, error: "connection blocked" } [INFO] [stdout] 2025-09-01T11:55:40.053159Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(13) [INFO] [stdout] 2025-09-01T11:55:40.193361Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:40.306161Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:40.306210Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:55:40.308068Z INFO Node{id=14}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:40.308143Z INFO Node{id=14}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:40.309577Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:55:40.324359Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:40.329093Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:40.329119Z ERROR Node{id=25}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:40.330909Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:40.332601Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727740308) } [INFO] [stdout] 2025-09-01T11:55:40.335926Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 11 } [INFO] [stdout] 2025-09-01T11:55:40.335951Z ERROR Node{id=22}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=11 leader=1 local=22 [INFO] [stdout] 2025-09-01T11:55:40.335991Z INFO Node{id=14}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=27 [INFO] [stdout] 2025-09-01T11:55:40.336012Z INFO Node{id=14}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:40.339111Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:40.339168Z INFO Node{id=22}:connect{target=11}:WritePeer{remote=11}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:40.340888Z INFO Node{id=22}:connect{target=11}:ReadPeer{remote=11}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:40.341468Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:40.352587Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:40.352633Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:55:40.355366Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:40.355397Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=4 leader=1 [INFO] [stdout] 2025-09-01T11:55:40.356076Z INFO Node{id=11}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:40.356142Z INFO Node{id=11}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:40.356895Z INFO Node{id=2}:connect{target=4}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:40.356951Z INFO Node{id=2}:connect{target=4}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:40.357666Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:55:40.358383Z INFO Node{id=4}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:55:40.361211Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:40.366362Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:55:40.366388Z ERROR Node{id=14}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 31] peer=31 leader=1 local=14 [INFO] [stdout] 2025-09-01T11:55:40.372696Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:40.372861Z INFO Node{id=14}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:40.373124Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727740356) } [INFO] [stdout] 2025-09-01T11:55:40.374098Z INFO Node{id=14}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:40.374249Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727740356) } [INFO] [stdout] 2025-09-01T11:55:40.374906Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:40.377970Z INFO Node{id=11}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=21 [INFO] [stdout] 2025-09-01T11:55:40.378004Z INFO Node{id=11}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:40.378702Z INFO Node{id=2}:connect{target=4}: sharedstate::worker::sync_manager: got pong from peer latency=22 [INFO] [stdout] 2025-09-01T11:55:40.378729Z INFO Node{id=2}:connect{target=4}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:40.384988Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:40.385047Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:40.389201Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:40.390145Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 4 } [INFO] [stdout] 2025-09-01T11:55:40.390167Z ERROR Node{id=2}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=4 leader=1 local=2 [INFO] [stdout] 2025-09-01T11:55:40.390885Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:55:40.392371Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:40.392519Z INFO Node{id=2}:connect{target=4}:WritePeer{remote=4}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:40.393570Z INFO Node{id=2}:connect{target=4}:ReadPeer{remote=4}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:40.394728Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:40.395724Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:40.395787Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:40.397557Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 11852, state_accept_seq: 11851, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:40.398787Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:40.399626Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:40.401077Z INFO Node{id=26}: sequenced_broadcast: 109|Subscriber(1): Added, allow_drop: true, next_sequence: 6859, min_allowed_seq: 6859 [INFO] [stdout] 2025-09-01T11:55:40.406957Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6859) [INFO] [stdout] 2025-09-01T11:55:40.407060Z INFO Node{id=11}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:40.408020Z INFO Node{id=11}: sequenced_broadcast: 114|SequencedBroadcastWorker Started id=114 [INFO] [stdout] 2025-09-01T11:55:40.409202Z INFO Node{id=11}: sequenced_broadcast: 93|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:40.409222Z INFO Node{id=11}: sequenced_broadcast: 93|SequencedBroadcastWorker Stopped id=93 elapsed=31.347948399s iter=3787 [INFO] [stdout] 2025-09-01T11:55:40.410730Z INFO ReadClient{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:40.413398Z ERROR Node{id=25}:connect{target=11}:ReadPeer{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:40.413482Z ERROR Node{id=15}:connect{target=11}:ReadPeer{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:40.414458Z WARN Node{id=25}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:40.414475Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:40.415427Z WARN Node{id=15}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:40.415445Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:40.470023Z ERROR Node{id=32}:connect{target=17}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:40.471052Z INFO Node{id=32}:connect{target=17}:WritePeer{remote=17}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:40.471208Z INFO Node{id=32}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:40.472459Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:40.474420Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(17) [INFO] [stdout] 2025-09-01T11:55:40.632240Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:40.634750Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:40.634792Z ERROR Node{id=20}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:40.681352Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:40.681397Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:55:40.682478Z INFO Node{id=6}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:40.682545Z INFO Node{id=6}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:40.683135Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:55:40.690694Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727740682) } [INFO] [stdout] 2025-09-01T11:55:40.693217Z INFO Node{id=6}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:55:40.693247Z INFO Node{id=6}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:40.697458Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:40.700219Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:55:40.704343Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:40.706134Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:40.707100Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:40.708129Z INFO Node{id=13}: sequenced_broadcast: 87|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 4770 [INFO] [stdout] 2025-09-01T11:55:40.712819Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:40.712888Z INFO Node{id=6}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:55:40.714823Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:40.715889Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:40.715921Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:55:40.717408Z INFO Node{id=7}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:40.717474Z INFO Node{id=7}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:40.718649Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:55:40.719616Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:55:40.719639Z ERROR Node{id=4}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30] peer=30 leader=1 local=4 [INFO] [stdout] 2025-09-01T11:55:40.720729Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:40.720799Z INFO Node{id=4}:connect{target=30}:WritePeer{remote=30}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:40.722146Z INFO Node{id=4}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:40.722963Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:40.728371Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727740717) } [INFO] [stdout] 2025-09-01T11:55:40.728409Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=7 [INFO] [stdout] 2025-09-01T11:55:40.729332Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:40.729360Z INFO Node{id=10}: sharedstate::worker::sync_manager: Broadcasted 11 peers to 10 peers [INFO] [stdout] 2025-09-01T11:55:40.731913Z INFO Node{id=7}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=14 [INFO] [stdout] 2025-09-01T11:55:40.731940Z INFO Node{id=7}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:40.733327Z ERROR Node{id=7}:connect{target=10}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:40.733569Z INFO Node{id=7}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:40.734536Z INFO Node{id=7}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:40.735053Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(10) [INFO] [stdout] 2025-09-01T11:55:40.736366Z ERROR ReadClient{remote=7}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:40.738882Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:40.797985Z ERROR ReadClient{remote=9}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:55:40.802187Z ERROR Node{id=9}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:40.802446Z WARN Node{id=9}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:40.802464Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:40.857740Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:40.857951Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:55:40.858776Z INFO Node{id=19}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:40.858839Z INFO Node{id=19}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:40.859587Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:40.867245Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727740858) } [INFO] [stdout] 2025-09-01T11:55:40.871965Z INFO Node{id=19}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:55:40.871996Z INFO Node{id=19}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:40.880819Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:40.894151Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:40.921479Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727735465) } [INFO] [stdout] 2025-09-01T11:55:40.937684Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:55:40.937721Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=15 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:55:40.938922Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:40.941869Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:40.942653Z INFO Node{id=19}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:40.942812Z INFO Node{id=19}:connect{target=15}:WritePeer{remote=15}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:40.943747Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:40.991594Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:41.055377Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:41.055423Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:55:41.056649Z INFO Node{id=12}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:41.056802Z INFO Node{id=12}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:41.057852Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:55:41.109027Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:41.111594Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16]) [INFO] [stdout] 2025-09-01T11:55:41.177557Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:41.179882Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 29]) [INFO] [stdout] 2025-09-01T11:55:41.341270Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:41.341319Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:55:41.342511Z INFO Node{id=22}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:41.342598Z INFO Node{id=22}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:41.344032Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:55:41.359697Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727741342) } [INFO] [stdout] 2025-09-01T11:55:41.362388Z INFO Node{id=22}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:55:41.362421Z INFO Node{id=22}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:41.374201Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:41.374251Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:55:41.375842Z INFO Node{id=14}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:41.375919Z INFO Node{id=14}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:41.377464Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:55:41.379620Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:41.385862Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:55:41.394143Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:41.394187Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:55:41.395559Z INFO Node{id=2}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:41.395637Z INFO Node{id=2}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:41.397295Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:55:41.398918Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:41.400209Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6859, state_accept_seq: 6858, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:41.402433Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:41.405751Z INFO Node{id=26}: sequenced_broadcast: 109|Subscriber(2): Added, allow_drop: true, next_sequence: 6859, min_allowed_seq: 6859 [INFO] [stdout] 2025-09-01T11:55:41.412925Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6859) [INFO] [stdout] 2025-09-01T11:55:41.413000Z INFO Node{id=22}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:55:41.419610Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:41.419650Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:55:41.420151Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:41.420179Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:55:41.420427Z INFO Node{id=25}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:41.420490Z INFO Node{id=25}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:41.421130Z INFO Node{id=15}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:41.421179Z INFO Node{id=15}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:41.421241Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:55:41.422329Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:55:41.437394Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727741421) } [INFO] [stdout] 2025-09-01T11:55:41.437431Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=15 [INFO] [stdout] 2025-09-01T11:55:41.441644Z INFO Node{id=15}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:55:41.441673Z INFO Node{id=15}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:41.447716Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:41.458346Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:41.463177Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:55:41.464839Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:41.470851Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 11852, state_accept_seq: 11851, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:41.472030Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:41.474943Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:41.474975Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:55:41.475696Z INFO Node{id=23}: sequenced_broadcast: 106|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:41.476729Z INFO Node{id=32}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:41.476827Z INFO Node{id=32}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:41.477902Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:41.485286Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:41.485396Z INFO Node{id=15}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:41.487270Z INFO Node{id=15}: sequenced_broadcast: 115|SequencedBroadcastWorker Started id=115 [INFO] [stdout] 2025-09-01T11:55:41.489491Z INFO Node{id=15}: sequenced_broadcast: 110|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:41.489511Z INFO Node{id=15}: sequenced_broadcast: 110|SequencedBroadcastWorker Stopped id=110 elapsed=2.490323785s iter=3 [INFO] [stdout] 2025-09-01T11:55:41.491627Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727741476) } [INFO] [stdout] 2025-09-01T11:55:41.491652Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=32 [INFO] [stdout] 2025-09-01T11:55:41.492419Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:55:41.493855Z ERROR Node{id=17}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:41.496163Z INFO Node{id=32}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:55:41.496192Z INFO Node{id=32}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:41.498974Z WARN Node{id=17}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:41.498995Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:41.503722Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:41.509401Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:55:41.509427Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 31, 10] peer=10 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:55:41.510966Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:41.512162Z INFO Node{id=32}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:41.512237Z INFO Node{id=32}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:41.512953Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:41.513936Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:41.708024Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:41.708065Z INFO Node{id=8}: sharedstate::worker::sync_manager: Broadcasted 6 peers to 5 peers [INFO] [stdout] 2025-09-01T11:55:41.722233Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:41.722285Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:55:41.723793Z INFO Node{id=4}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:41.723861Z INFO Node{id=4}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:41.725223Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:55:41.736813Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:41.736851Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:55:41.737292Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727741723) } [INFO] [stdout] 2025-09-01T11:55:41.737314Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=4 [INFO] [stdout] 2025-09-01T11:55:41.738435Z INFO Node{id=7}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:41.738512Z INFO Node{id=7}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:41.739814Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:55:41.741092Z INFO Node{id=1}: sequenced_broadcast: 1|Subscriber(20): channel closed, dropping [INFO] [stdout] 2025-09-01T11:55:41.741380Z INFO Node{id=4}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:41.741407Z INFO Node{id=4}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:41.748034Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:41.753484Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:55:41.753515Z ERROR Node{id=4}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 29, 5, 21] peer=21 leader=1 local=4 [INFO] [stdout] 2025-09-01T11:55:41.755885Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:41.755934Z INFO Node{id=4}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:41.757388Z INFO Node{id=4}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:41.758672Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:41.759893Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:41.775585Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727741738) } [INFO] [stdout] 2025-09-01T11:55:41.778016Z INFO Node{id=7}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=40 [INFO] [stdout] 2025-09-01T11:55:41.778049Z INFO Node{id=7}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:41.805115Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:41.808424Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:41.808458Z INFO Node{id=9}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:55:41.809903Z INFO Node{id=9}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:41.809961Z INFO Node{id=9}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:41.810039Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:55:41.811902Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=9 [INFO] [stdout] 2025-09-01T11:55:41.824622Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: Ping(1756727741809) } [INFO] [stdout] 2025-09-01T11:55:41.824656Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=9 [INFO] [stdout] 2025-09-01T11:55:41.831835Z INFO Node{id=9}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=22 [INFO] [stdout] 2025-09-01T11:55:41.831876Z INFO Node{id=9}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:41.834838Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:41.837059Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:41.839004Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 11686, state_accept_seq: 11685, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:41.845140Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:41.850848Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:55:41.855729Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:41.857301Z INFO Node{id=31}: sequenced_broadcast: 108|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:41.865899Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:41.866003Z INFO Node{id=7}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:41.867264Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:41.867470Z INFO Node{id=7}: sequenced_broadcast: 116|SequencedBroadcastWorker Started id=116 [INFO] [stdout] 2025-09-01T11:55:41.869358Z INFO Node{id=1}: sequenced_broadcast: 2|Subscriber(20): channel closed, dropping [INFO] [stdout] 2025-09-01T11:55:41.869415Z INFO Node{id=7}: sequenced_broadcast: 102|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:41.869428Z INFO Node{id=7}: sequenced_broadcast: 102|SequencedBroadcastWorker Stopped id=102 elapsed=13.283581006s iter=1369 [INFO] [stdout] 2025-09-01T11:55:41.869831Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:41.869872Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6859, state_accept_seq: 6858, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:41.879689Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:41.882771Z INFO Node{id=23}: sequenced_broadcast: 106|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:41.892324Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:41.892451Z INFO Node{id=9}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:41.893372Z INFO Node{id=9}: sequenced_broadcast: 117|SequencedBroadcastWorker Started id=117 [INFO] [stdout] 2025-09-01T11:55:41.894182Z INFO Node{id=9}: sequenced_broadcast: 95|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:41.894200Z INFO Node{id=9}: sequenced_broadcast: 95|SequencedBroadcastWorker Stopped id=95 elapsed=25.299138271s iter=597 [INFO] [stdout] 2025-09-01T11:55:41.900616Z ERROR Node{id=27}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:41.900694Z ERROR Node{id=26}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:41.901665Z WARN Node{id=26}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:41.901683Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:41.902031Z WARN Node{id=27}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:41.902047Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:41.945477Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:41.945518Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:55:41.946619Z INFO Node{id=19}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:41.946690Z INFO Node{id=19}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:41.947498Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:41.955699Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727741946) } [INFO] [stdout] 2025-09-01T11:55:41.959141Z INFO Node{id=19}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:55:41.959171Z INFO Node{id=19}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:41.964389Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:41.975465Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:41.986923Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:55:41.986959Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 31, 7] peer=7 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:55:41.990026Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:41.991551Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:41.992446Z INFO Node{id=19}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:41.992520Z INFO Node{id=19}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:41.993687Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:42.105868Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:42.109467Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:42.109639Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6859, state_accept_seq: 6858, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:42.121347Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:42.127398Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:42.186516Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727741395) } [INFO] [stdout] 2025-09-01T11:55:42.506452Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:42.506502Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=4 leader=1 [INFO] [stdout] 2025-09-01T11:55:42.508033Z INFO Node{id=17}:connect{target=4}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:42.508114Z INFO Node{id=17}:connect{target=4}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:42.509947Z INFO Node{id=4}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:55:42.515954Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:42.515997Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:55:42.517395Z INFO Node{id=32}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:42.517471Z INFO Node{id=32}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:42.518832Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:42.523283Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727742508) } [INFO] [stdout] 2025-09-01T11:55:42.526859Z INFO Node{id=17}:connect{target=4}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:42.526890Z INFO Node{id=17}:connect{target=4}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:42.530053Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727742517) } [INFO] [stdout] 2025-09-01T11:55:42.537195Z INFO Node{id=32}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:55:42.537237Z INFO Node{id=32}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:42.538240Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:42.544629Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:42.549608Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:42.555532Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 4 } [INFO] [stdout] 2025-09-01T11:55:42.555574Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=4 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:55:42.556579Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:42.557872Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:42.558406Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:42.559256Z INFO Node{id=17}:connect{target=4}:ReadPeer{remote=4}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:42.559323Z INFO Node{id=17}:connect{target=4}:WritePeer{remote=4}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:42.560708Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:42.568407Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:55:42.568449Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=27 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:55:42.569344Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:42.571167Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:42.572083Z INFO Node{id=32}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:42.572165Z INFO Node{id=32}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:42.573205Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:42.636914Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:42.643182Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:42.648221Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:42.648253Z ERROR Node{id=20}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:42.757802Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:42.757850Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:55:42.759459Z INFO Node{id=4}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:42.759537Z INFO Node{id=4}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:42.760424Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:55:42.770049Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727742759) } [INFO] [stdout] 2025-09-01T11:55:42.772618Z INFO Node{id=4}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:55:42.772647Z INFO Node{id=4}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:42.781609Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:42.786263Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:55:42.786293Z ERROR Node{id=4}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 31, 7] peer=7 leader=1 local=4 [INFO] [stdout] 2025-09-01T11:55:42.788340Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:42.788381Z INFO Node{id=4}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:42.789591Z INFO Node{id=4}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:42.790853Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:42.790916Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:42.908515Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:42.908572Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:55:42.910684Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:42.910713Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:55:42.911148Z INFO Node{id=27}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:42.911229Z INFO Node{id=27}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:42.912307Z INFO Node{id=26}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:42.912362Z INFO Node{id=26}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:42.912456Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:55:42.913702Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:55:42.926943Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727742912) } [INFO] [stdout] 2025-09-01T11:55:42.931065Z INFO Node{id=26}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=19 [INFO] [stdout] 2025-09-01T11:55:42.931098Z INFO Node{id=26}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:42.939415Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:42.950526Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:42.950711Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:55:42.950732Z ERROR Node{id=26}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=24 leader=1 local=26 [INFO] [stdout] 2025-09-01T11:55:42.954532Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:42.957063Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:42.961060Z INFO Node{id=26}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:42.961244Z INFO Node{id=26}:connect{target=24}:WritePeer{remote=24}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:42.968424Z ERROR ReadClient{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:42.973942Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727741056) } [INFO] [stdout] 2025-09-01T11:55:42.978098Z INFO Node{id=12}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=1922 [INFO] [stdout] 2025-09-01T11:55:42.978130Z INFO Node{id=12}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:42.993655Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:42.993702Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:55:42.993783Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:42.995145Z INFO Node{id=19}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:42.995208Z INFO Node{id=19}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:42.996701Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:43.002001Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:55:43.002027Z ERROR Node{id=12}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=17 leader=1 local=12 [INFO] [stdout] 2025-09-01T11:55:43.007099Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:43.007158Z INFO Node{id=12}:connect{target=17}:WritePeer{remote=17}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:43.008544Z INFO Node{id=12}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:43.013250Z ERROR ReadClient{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:43.014213Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:43.020267Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727742995) } [INFO] [stdout] 2025-09-01T11:55:43.024501Z INFO Node{id=19}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=29 [INFO] [stdout] 2025-09-01T11:55:43.024537Z INFO Node{id=19}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:43.034654Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:43.043198Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:55:43.043234Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=14 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:55:43.047997Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:43.048739Z INFO Node{id=19}:connect{target=14}:WritePeer{remote=14}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:43.049732Z INFO Node{id=19}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:43.050459Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:43.054873Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:43.206749Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:43.212543Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:43.212779Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeRecovery(RecoverableStateDetails { id: 15928016797218250794, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:43.560296Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:43.560340Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:55:43.561050Z INFO Node{id=17}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:43.561112Z INFO Node{id=17}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:43.561742Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:55:43.572519Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:43.572566Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:55:43.573307Z INFO Node{id=32}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:43.573378Z INFO Node{id=32}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:43.574208Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:43.591712Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727743573) } [INFO] [stdout] 2025-09-01T11:55:43.591751Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=32 [INFO] [stdout] 2025-09-01T11:55:43.597423Z INFO Node{id=32}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=24 [INFO] [stdout] 2025-09-01T11:55:43.597465Z INFO Node{id=32}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:43.603963Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:43.615360Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:43.630214Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727741375) } [INFO] [stdout] 2025-09-01T11:55:43.632623Z INFO Node{id=14}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=2257 [INFO] [stdout] 2025-09-01T11:55:43.632655Z INFO Node{id=14}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:43.636388Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:55:43.636415Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 29, 5, 21] peer=21 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:55:43.639126Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:43.640917Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:43.641698Z INFO Node{id=32}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:43.641795Z INFO Node{id=32}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:43.642596Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:43.789909Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:43.789956Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:55:43.791116Z INFO Node{id=4}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:43.791182Z INFO Node{id=4}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:43.791909Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:55:43.800311Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727743791) } [INFO] [stdout] 2025-09-01T11:55:43.802619Z INFO Node{id=4}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:55:43.802649Z INFO Node{id=4}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:43.807650Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:43.811596Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:55:43.811623Z ERROR Node{id=4}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9] peer=9 leader=1 local=4 [INFO] [stdout] 2025-09-01T11:55:43.813352Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:43.813399Z INFO Node{id=4}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:43.814188Z INFO Node{id=4}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:43.814916Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:43.815848Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:43.824325Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:43.825935Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:43.825957Z ERROR Node{id=29}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:43.826419Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:43.826446Z INFO Node{id=21}: sharedstate::worker::sync_manager: Broadcasted 7 peers to 6 peers [INFO] [stdout] 2025-09-01T11:55:43.841182Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:43.843308Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30]) [INFO] [stdout] 2025-09-01T11:55:43.958673Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:43.958720Z INFO Node{id=26}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:55:43.959606Z INFO Node{id=26}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:43.959668Z INFO Node{id=26}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:43.960633Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=26 [INFO] [stdout] 2025-09-01T11:55:43.965900Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: Ping(1756727743959) } [INFO] [stdout] 2025-09-01T11:55:43.968838Z INFO Node{id=26}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:55:43.968866Z INFO Node{id=26}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:43.972539Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:43.976567Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:55:43.978288Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:43.980281Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:43.980825Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6859, state_accept_seq: 6858, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:43.981711Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:43.983232Z INFO Node{id=15}: sequenced_broadcast: 115|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:43.986577Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:43.986672Z INFO Node{id=26}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:43.987316Z INFO Node{id=26}: sequenced_broadcast: 118|SequencedBroadcastWorker Started id=118 [INFO] [stdout] 2025-09-01T11:55:43.988348Z INFO Node{id=26}: sequenced_broadcast: 109|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:43.988365Z INFO Node{id=26}: sequenced_broadcast: 109|SequencedBroadcastWorker Stopped id=109 elapsed=6.190447344s iter=4 [INFO] [stdout] 2025-09-01T11:55:43.990240Z INFO ReadClient{remote=11}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:43.990891Z ERROR Node{id=11}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:43.990959Z ERROR Node{id=22}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:43.991703Z WARN Node{id=11}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:43.991720Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:43.991782Z WARN Node{id=22}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:43.991796Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:44.008737Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:44.008802Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=28 leader=1 [INFO] [stdout] 2025-09-01T11:55:44.009582Z INFO Node{id=12}:connect{target=28}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:44.009674Z INFO Node{id=12}:connect{target=28}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:44.010456Z INFO Node{id=28}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:55:44.049746Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:44.049811Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:55:44.050560Z INFO Node{id=19}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:44.050687Z INFO Node{id=19}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:44.051851Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:44.079887Z ERROR WriteClient{remote=2}: sharedstate::net::message_channels: failed to send message, closing write error=Elapsed(()) [INFO] [stdout] 2025-09-01T11:55:44.233209Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:44.301468Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:44.349462Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:44.350663Z INFO Node{id=29}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:44.379352Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:44.439459Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:44.445817Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:44.448976Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 29, 5]) [INFO] [stdout] 2025-09-01T11:55:44.645674Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:44.645720Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:55:44.650912Z INFO Node{id=32}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:44.650990Z INFO Node{id=32}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:44.651179Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:44.651621Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:44.654955Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:44.704144Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:44.704197Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 12 peers to 11 peers [INFO] [stdout] 2025-09-01T11:55:44.704430Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:44.704450Z ERROR Node{id=20}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:44.705453Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727744650) } [INFO] [stdout] 2025-09-01T11:55:44.709229Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 12) [INFO] [stdout] 2025-09-01T11:55:44.710167Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 12) [INFO] [stdout] 2025-09-01T11:55:44.711421Z INFO Node{id=32}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=61 [INFO] [stdout] 2025-09-01T11:55:44.711449Z INFO Node{id=32}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:44.714862Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:44.719515Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:55:44.719551Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=22 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:55:44.720616Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:44.721657Z INFO Node{id=32}:connect{target=22}:ReadPeer{remote=22}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:44.721727Z INFO Node{id=32}:connect{target=22}:WritePeer{remote=22}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:44.722327Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:44.722415Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:44.814581Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:44.814627Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:55:44.815437Z INFO Node{id=4}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:44.815507Z INFO Node{id=4}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:44.817218Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:55:44.822851Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727744815) } [INFO] [stdout] 2025-09-01T11:55:44.824870Z INFO Node{id=4}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:55:44.824898Z INFO Node{id=4}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:44.830627Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:44.833074Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:55:44.833097Z ERROR Node{id=4}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=27 leader=1 local=4 [INFO] [stdout] 2025-09-01T11:55:44.834038Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:44.834086Z INFO Node{id=4}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:44.834723Z INFO Node{id=4}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:44.835330Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:44.837071Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:44.907835Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727741420) } [INFO] [stdout] 2025-09-01T11:55:44.909722Z INFO Node{id=25}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=3489 [INFO] [stdout] 2025-09-01T11:55:44.909751Z INFO Node{id=25}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:44.995587Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:44.995638Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:55:44.998936Z INFO Node{id=11}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:44.999018Z INFO Node{id=11}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:44.999099Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:44.999125Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:55:44.999914Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:55:45.000007Z INFO Node{id=22}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:45.000137Z INFO Node{id=22}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:45.000962Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:55:45.013233Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727744999) } [INFO] [stdout] 2025-09-01T11:55:45.013280Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=11 [INFO] [stdout] 2025-09-01T11:55:45.014496Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727745000) } [INFO] [stdout] 2025-09-01T11:55:45.017842Z INFO Node{id=11}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:45.017876Z INFO Node{id=11}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:45.026009Z INFO Node{id=22}:connect{target=19}: sharedstate::worker::sync_manager: got pong from peer latency=26 [INFO] [stdout] 2025-09-01T11:55:45.026050Z INFO Node{id=22}:connect{target=19}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:45.038991Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:45.040575Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:45.044435Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:55:45.045938Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 19 } [INFO] [stdout] 2025-09-01T11:55:45.045961Z ERROR Node{id=22}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=19 leader=1 local=22 [INFO] [stdout] 2025-09-01T11:55:45.047095Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:45.047145Z INFO Node{id=22}:connect{target=19}:WritePeer{remote=19}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:45.048308Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:45.048353Z INFO Node{id=22}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:45.048772Z ERROR ReadClient{remote=22}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:45.051428Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6859, state_accept_seq: 6858, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:45.051699Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:45.052929Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:45.053703Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:45.059697Z INFO Node{id=21}: sequenced_broadcast: 107|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:45.063970Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:45.064082Z INFO Node{id=11}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:45.066728Z INFO Node{id=11}: sequenced_broadcast: 119|SequencedBroadcastWorker Started id=119 [INFO] [stdout] 2025-09-01T11:55:45.068085Z INFO Node{id=11}: sequenced_broadcast: 114|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:45.068104Z INFO Node{id=11}: sequenced_broadcast: 114|SequencedBroadcastWorker Stopped id=114 elapsed=4.660062904s iter=2 [INFO] [stdout] 2025-09-01T11:55:45.354689Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:45.354738Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:55:45.356075Z INFO Node{id=29}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:45.356153Z INFO Node{id=29}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:45.357217Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:55:45.377666Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727745356) } [INFO] [stdout] 2025-09-01T11:55:45.381994Z INFO Node{id=29}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=25 [INFO] [stdout] 2025-09-01T11:55:45.382031Z INFO Node{id=29}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:45.392166Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:45.400737Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:55:45.408289Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:45.410742Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:45.415225Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:45.417004Z INFO Node{id=13}: sequenced_broadcast: 87|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 4770 [INFO] [stdout] 2025-09-01T11:55:45.426340Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:45.426423Z INFO Node{id=29}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:55:45.509579Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:45.514091Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(23268) < seq(23316) [INFO] [stdout] 2025-09-01T11:55:45.534171Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(23275) < seq(23316) [INFO] [stdout] 2025-09-01T11:55:45.536098Z ERROR Node{id=1}: sharedstate::utils: Error(src/worker/sync_updater.rs:215), failed to add fresh client error=SequenceTooFarAhead { seq: 23316, max: 23275 } [INFO] [stdout] 2025-09-01T11:55:45.536127Z ERROR Node{id=1}: sharedstate::worker::sync_manager: Failed to add fresh client [INFO] [stdout] 2025-09-01T11:55:45.539017Z ERROR Node{id=8}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:45.540232Z WARN Node{id=8}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:45.540253Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:45.541233Z INFO Node{id=8}:connect{target=1}:WritePeer{remote=1}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:45.545877Z ERROR ReadClient{remote=3}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:55:45.549072Z ERROR Node{id=3}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:45.549685Z WARN Node{id=3}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:45.549702Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:45.554510Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727744050) } [INFO] [stdout] 2025-09-01T11:55:45.557108Z INFO Node{id=19}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=1507 [INFO] [stdout] 2025-09-01T11:55:45.557308Z INFO Node{id=19}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:45.619005Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:45.723729Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:45.723797Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:55:45.725210Z INFO Node{id=32}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:45.725290Z INFO Node{id=32}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:45.726208Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:45.731090Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:45.731123Z INFO Node{id=10}: sharedstate::worker::sync_manager: Broadcasted 12 peers to 11 peers [INFO] [stdout] 2025-09-01T11:55:45.734962Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727745725) } [INFO] [stdout] 2025-09-01T11:55:45.738738Z INFO Node{id=32}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:55:45.738793Z INFO Node{id=32}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:45.746863Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:45.753415Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:55:45.753452Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=14 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:55:45.755451Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:45.755813Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:45.756616Z INFO Node{id=32}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:45.756694Z INFO Node{id=32}:connect{target=14}:WritePeer{remote=14}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:45.758539Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:45.809037Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:45.811171Z INFO Node{id=30}: sequenced_broadcast: 85|Subscriber(4): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 4770 [INFO] [stdout] 2025-09-01T11:55:45.816138Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:45.816245Z INFO Node{id=24}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:45.816808Z INFO Node{id=24}: sequenced_broadcast: 120|SequencedBroadcastWorker Started id=120 [INFO] [stdout] 2025-09-01T11:55:45.817924Z INFO Node{id=24}: sequenced_broadcast: 82|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:45.817942Z INFO Node{id=24}: sequenced_broadcast: 82|SequencedBroadcastWorker Stopped id=82 elapsed=52.316832208s iter=2929 [INFO] [stdout] 2025-09-01T11:55:45.835941Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:45.835988Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:55:45.836631Z INFO Node{id=4}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:45.836695Z INFO Node{id=4}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:45.837670Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:55:46.013152Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:46.050073Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:46.050119Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:55:46.051322Z INFO Node{id=22}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:46.051389Z INFO Node{id=22}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:46.051965Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:55:46.156583Z ERROR ReadClient{remote=12}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:55:46.233268Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727745836) } [INFO] [stdout] 2025-09-01T11:55:46.241534Z INFO Node{id=4}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=405 [INFO] [stdout] 2025-09-01T11:55:46.241576Z INFO Node{id=4}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:46.359263Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:55:46.362466Z ERROR Node{id=14}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:46.365911Z ERROR Node{id=14}:connect{target=29}: sharedstate::worker::sync_manager: channel closed waiting for leader path [INFO] [stdout] 2025-09-01T11:55:46.367444Z INFO Node{id=14}:connect{target=29}:WritePeer{remote=29}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:46.367529Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(29) [INFO] [stdout] 2025-09-01T11:55:46.397167Z ERROR Node{id=2}:connect{target=9}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:46.398709Z INFO Node{id=2}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:46.398811Z INFO Node{id=2}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:46.398911Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(9) [INFO] [stdout] 2025-09-01T11:55:46.399602Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:46.541691Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:46.541739Z INFO Node{id=8}: sharedstate::worker::sync_manager: found connect target option_count=5 peers=6 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:55:46.542879Z INFO Node{id=8}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:46.542951Z INFO Node{id=8}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:46.544333Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=8 [INFO] [stdout] 2025-09-01T11:55:46.554841Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:46.554884Z INFO Node{id=3}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:55:46.555646Z INFO Node{id=3}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:46.555710Z INFO Node{id=3}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:46.556713Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=3 [INFO] [stdout] 2025-09-01T11:55:46.564557Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: Ping(1756727746555) } [INFO] [stdout] 2025-09-01T11:55:46.567550Z INFO Node{id=3}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=12 [INFO] [stdout] 2025-09-01T11:55:46.567586Z INFO Node{id=3}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:46.571905Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: Ping(1756727746542) } [INFO] [stdout] 2025-09-01T11:55:46.574652Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:46.575436Z INFO Node{id=8}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=33 [INFO] [stdout] 2025-09-01T11:55:46.575465Z INFO Node{id=8}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:46.578976Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:55:46.581586Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:46.583522Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:46.584377Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:46.585036Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:55:46.587362Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:46.588053Z INFO Node{id=9}: sequenced_broadcast: 117|Subscriber rejected, seq(876) < min_allowed(6818) [INFO] [stdout] 2025-09-01T11:55:46.588872Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:46.589833Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeRecovery(RecoverableStateDetails { id: 15928016797218250794, generation: 1, recover_accept_seq: 1, state_accept_seq: 0, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:46.589893Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:46.591322Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:46.591884Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:46.593369Z INFO Node{id=9}: sequenced_broadcast: 117|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:46.594792Z INFO Node{id=26}: sequenced_broadcast: 118|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:46.599181Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:46.599290Z INFO Node{id=3}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:46.600337Z INFO Node{id=3}: sequenced_broadcast: 121|SequencedBroadcastWorker Started id=121 [INFO] [stdout] 2025-09-01T11:55:46.600397Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:46.600505Z INFO Node{id=8}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:46.601783Z INFO Node{id=8}: sequenced_broadcast: 122|SequencedBroadcastWorker Started id=122 [INFO] [stdout] 2025-09-01T11:55:46.602362Z INFO Node{id=3}: sequenced_broadcast: 86|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:46.602380Z INFO Node{id=3}: sequenced_broadcast: 86|SequencedBroadcastWorker Stopped id=86 elapsed=50.736536209s iter=2 [INFO] [stdout] 2025-09-01T11:55:46.602740Z INFO Node{id=8}: sequenced_broadcast: 40|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:46.602774Z INFO Node{id=8}: sequenced_broadcast: 40|SequencedBroadcastWorker Stopped id=40 elapsed=60.147274706s iter=2 [INFO] [stdout] 2025-09-01T11:55:46.619840Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727742911) } [INFO] [stdout] 2025-09-01T11:55:46.624528Z INFO Node{id=27}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=3713 [INFO] [stdout] 2025-09-01T11:55:46.624566Z INFO Node{id=27}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:46.706125Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:46.713247Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:46.717364Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:46.717391Z ERROR Node{id=20}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:46.758181Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:46.758227Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:55:46.759421Z INFO Node{id=32}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:46.759486Z INFO Node{id=32}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:46.760955Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:46.824397Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727746759) } [INFO] [stdout] 2025-09-01T11:55:46.830431Z INFO Node{id=32}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=71 [INFO] [stdout] 2025-09-01T11:55:46.830470Z INFO Node{id=32}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:46.881735Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:46.893479Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:55:46.893517Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16] peer=16 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:55:46.897265Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:46.898126Z INFO Node{id=32}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:46.898203Z INFO Node{id=32}:connect{target=16}:WritePeer{remote=16}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:46.899439Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:46.941567Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:47.079171Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727743561) } [INFO] [stdout] 2025-09-01T11:55:47.083038Z INFO Node{id=17}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=3522 [INFO] [stdout] 2025-09-01T11:55:47.083069Z INFO Node{id=17}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:47.128834Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:47.133523Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:55:47.145463Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:47.368951Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:47.369001Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:55:47.370650Z INFO Node{id=14}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:47.370734Z INFO Node{id=14}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:47.372060Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:55:47.380524Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:47.383510Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727747370) } [INFO] [stdout] 2025-09-01T11:55:47.385189Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:55:47.385233Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 31] peer=31 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:55:47.387725Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:47.387804Z INFO Node{id=19}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:47.387875Z INFO Node{id=14}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:55:47.387898Z INFO Node{id=14}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:47.389430Z INFO Node{id=19}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:47.391079Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:47.399096Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:47.401130Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:47.401162Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:55:47.402656Z INFO Node{id=2}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:47.402728Z INFO Node{id=2}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:47.404312Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:55:47.405167Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:55:47.405189Z ERROR Node{id=14}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=25 leader=1 local=14 [INFO] [stdout] 2025-09-01T11:55:47.407533Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:47.407592Z INFO Node{id=14}:connect{target=25}:WritePeer{remote=25}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:47.409211Z INFO Node{id=14}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:47.410722Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:47.412136Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:47.416068Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727747402) } [INFO] [stdout] 2025-09-01T11:55:47.416098Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=2 [INFO] [stdout] 2025-09-01T11:55:47.421145Z INFO Node{id=2}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=19 [INFO] [stdout] 2025-09-01T11:55:47.421182Z INFO Node{id=2}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:47.427846Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:47.432950Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:55:47.437457Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:47.444347Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 11852, state_accept_seq: 11851, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:47.444644Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:47.445883Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:47.448394Z INFO Node{id=21}: sequenced_broadcast: 107|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:47.452608Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:47.452706Z INFO Node{id=2}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:47.454156Z INFO Node{id=2}: sequenced_broadcast: 123|SequencedBroadcastWorker Started id=123 [INFO] [stdout] 2025-09-01T11:55:47.456211Z INFO Node{id=2}: sequenced_broadcast: 80|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:47.456230Z INFO Node{id=2}: sequenced_broadcast: 80|SequencedBroadcastWorker Stopped id=80 elapsed=54.103006597s iter=5247 [INFO] [stdout] 2025-09-01T11:55:47.595437Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727744009) } [INFO] [stdout] 2025-09-01T11:55:47.898592Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:47.898639Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:55:47.899998Z INFO Node{id=32}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:47.900059Z INFO Node{id=32}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:47.901216Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:47.912266Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727747900) } [INFO] [stdout] 2025-09-01T11:55:47.915875Z INFO Node{id=32}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:55:47.915918Z INFO Node{id=32}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:47.924226Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:47.931685Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:55:47.931719Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 29, 18] peer=18 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:55:47.932985Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:47.934117Z INFO Node{id=32}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:47.934195Z INFO Node{id=32}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:47.934322Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:47.937524Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:48.053346Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:48.053397Z INFO Node{id=1}: sharedstate::worker::sync_manager: Broadcasted 31 peers to 31 peers [INFO] [stdout] 2025-09-01T11:55:48.389034Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:48.389082Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:55:48.390397Z INFO Node{id=19}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:48.390473Z INFO Node{id=19}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:48.391401Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:48.408889Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:48.408936Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:55:48.409094Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727748390) } [INFO] [stdout] 2025-09-01T11:55:48.410224Z INFO Node{id=14}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:48.410310Z INFO Node{id=14}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:48.411823Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:55:48.412953Z INFO Node{id=19}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=22 [INFO] [stdout] 2025-09-01T11:55:48.412979Z INFO Node{id=19}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:48.421201Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727748410) } [INFO] [stdout] 2025-09-01T11:55:48.426123Z INFO Node{id=14}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:55:48.426161Z INFO Node{id=14}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:48.426501Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:48.433356Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 5 } [INFO] [stdout] 2025-09-01T11:55:48.433388Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 29, 5] peer=5 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:55:48.433428Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:48.434878Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:48.438318Z INFO Node{id=19}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:48.438402Z INFO Node{id=19}:connect{target=5}:WritePeer{remote=5}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:48.439836Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:48.441212Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:55:48.441236Z ERROR Node{id=14}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9] peer=9 leader=1 local=14 [INFO] [stdout] 2025-09-01T11:55:48.443392Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:48.443439Z INFO Node{id=14}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:48.444646Z INFO Node{id=14}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:48.444869Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:48.445845Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:48.447215Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:48.673829Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:48.718331Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:48.719028Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:48.723221Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:55:48.723251Z ERROR Node{id=25}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=32 leader=1 local=25 [INFO] [stdout] 2025-09-01T11:55:48.723938Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:48.724474Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:48.724524Z INFO Node{id=25}:connect{target=32}:WritePeer{remote=32}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:48.725847Z INFO Node{id=25}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:48.726631Z ERROR ReadClient{remote=25}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:48.727309Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:48.727330Z ERROR Node{id=20}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:48.815251Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:48.819019Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:48.821196Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:48.823750Z INFO Node{id=13}: sequenced_broadcast: 87|Subscriber(3): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 4770 [INFO] [stdout] 2025-09-01T11:55:48.830368Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:48.830413Z INFO Node{id=21}: sharedstate::worker::sync_manager: Broadcasted 9 peers to 8 peers [INFO] [stdout] 2025-09-01T11:55:48.831171Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:48.831233Z INFO Node{id=4}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:55:48.834850Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 9) [INFO] [stdout] 2025-09-01T11:55:48.834891Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 9) [INFO] [stdout] 2025-09-01T11:55:48.936726Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:48.937798Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:55:48.939027Z INFO Node{id=32}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:48.939094Z INFO Node{id=32}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:48.940036Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:48.955918Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727748939) } [INFO] [stdout] 2025-09-01T11:55:48.955954Z INFO Node{id=8}: sharedstate::worker::sync_manager: Discovered Peer client_id=32 [INFO] [stdout] 2025-09-01T11:55:48.959370Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:48.959399Z INFO Node{id=8}: sharedstate::worker::sync_manager: Broadcasted 7 peers to 6 peers [INFO] [stdout] 2025-09-01T11:55:48.961633Z INFO Node{id=32}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=22 [INFO] [stdout] 2025-09-01T11:55:48.961660Z INFO Node{id=32}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:48.963271Z ERROR Node{id=32}:connect{target=8}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:55:48.963620Z INFO Node{id=32}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:48.964208Z INFO Node{id=32}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:48.964647Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(8) [INFO] [stdout] 2025-09-01T11:55:48.965885Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:48.970692Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:49.011416Z ERROR Node{id=12}:connect{target=28}: sharedstate::worker::sync_manager: timeout waiting for pong from peer [INFO] [stdout] 2025-09-01T11:55:49.012975Z INFO Node{id=12}:connect{target=28}:WritePeer{remote=28}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:49.013052Z INFO Node{id=12}:connect{target=28}:ReadPeer{remote=28}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:49.013129Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(28) [INFO] [stdout] 2025-09-01T11:55:49.038848Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:49.233172Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:49.234883Z INFO Node{id=20}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:49.242548Z ERROR ReadClient{remote=20}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:49.257112Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:49.264870Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:49.273710Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 28]) [INFO] [stdout] 2025-09-01T11:55:49.437969Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:49.438017Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=28 leader=1 [INFO] [stdout] 2025-09-01T11:55:49.440274Z INFO Node{id=19}:connect{target=28}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:49.440377Z INFO Node{id=19}:connect{target=28}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:49.441809Z INFO Node{id=28}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:49.446206Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:49.446239Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:55:49.447612Z INFO Node{id=14}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:49.447676Z INFO Node{id=14}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:49.448861Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:55:49.457895Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727749447) } [INFO] [stdout] 2025-09-01T11:55:49.464021Z INFO Node{id=14}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:55:49.464060Z INFO Node{id=14}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:49.470959Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:49.480345Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:55:49.480382Z ERROR Node{id=14}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 15, 26] peer=26 leader=1 local=14 [INFO] [stdout] 2025-09-01T11:55:49.481706Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:49.481777Z INFO Node{id=14}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:49.482803Z INFO Node{id=14}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:49.483882Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:49.488162Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:49.725798Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:49.725841Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:55:49.726594Z INFO Node{id=25}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:49.726653Z INFO Node{id=25}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:49.727699Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:55:49.884619Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:49.930964Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727746051) } [INFO] [stdout] 2025-09-01T11:55:49.933730Z INFO Node{id=22}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=3882 [INFO] [stdout] 2025-09-01T11:55:49.934791Z INFO Node{id=22}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:49.965409Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:49.965453Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:55:49.967204Z INFO Node{id=32}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:49.967278Z INFO Node{id=32}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:49.968534Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:50.001014Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727749967) } [INFO] [stdout] 2025-09-01T11:55:50.004374Z INFO Node{id=32}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=37 [INFO] [stdout] 2025-09-01T11:55:50.004408Z INFO Node{id=32}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:50.014829Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:50.014882Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=11 leader=1 [INFO] [stdout] 2025-09-01T11:55:50.016409Z INFO Node{id=12}:connect{target=11}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:50.016484Z INFO Node{id=12}:connect{target=11}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:50.017637Z INFO Node{id=11}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:55:50.030938Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727750016) } [INFO] [stdout] 2025-09-01T11:55:50.034802Z INFO Node{id=12}:connect{target=11}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:50.034836Z INFO Node{id=12}:connect{target=11}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:50.043899Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:50.045183Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:50.049258Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 11 } [INFO] [stdout] 2025-09-01T11:55:50.054066Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:50.056408Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6859, state_accept_seq: 6858, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:50.057707Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:50.057956Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:55:50.057979Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 15, 26] peer=26 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:55:50.061175Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:50.061742Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:50.062506Z INFO Node{id=11}: sequenced_broadcast: 119|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:50.063223Z INFO Node{id=32}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:50.063302Z INFO Node{id=32}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:50.064485Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:50.071174Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:50.071306Z INFO Node{id=12}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:50.072919Z INFO Node{id=12}: sequenced_broadcast: 124|SequencedBroadcastWorker Started id=124 [INFO] [stdout] 2025-09-01T11:55:50.074408Z INFO Node{id=12}: sequenced_broadcast: 84|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:50.074427Z INFO Node{id=12}: sequenced_broadcast: 84|SequencedBroadcastWorker Stopped id=84 elapsed=56.09118758s iter=2873 [INFO] [stdout] 2025-09-01T11:55:50.089721Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:50.251628Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:50.251679Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=7 peers=8 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:50.253087Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:50.253165Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:50.254495Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:55:50.273228Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727750253) } [INFO] [stdout] 2025-09-01T11:55:50.279847Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=26 [INFO] [stdout] 2025-09-01T11:55:50.279888Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:50.291123Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:50.297714Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:55:50.308441Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:50.310924Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:50.313070Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:50.314030Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, seq(876) < min_allowed(22372) [INFO] [stdout] 2025-09-01T11:55:50.316258Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:50.319649Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(24421) < seq(24422) [INFO] [stdout] 2025-09-01T11:55:50.335008Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber(5): Added, allow_drop: true, next_sequence: 24422, min_allowed_seq: 22375 [INFO] [stdout] 2025-09-01T11:55:50.340739Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 24422) [INFO] [stdout] 2025-09-01T11:55:50.340859Z INFO Node{id=20}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:50.342104Z INFO Node{id=20}: sequenced_broadcast: 125|SequencedBroadcastWorker Started id=125 [INFO] [stdout] 2025-09-01T11:55:50.343866Z INFO Node{id=20}: sequenced_broadcast: 104|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:50.343886Z INFO Node{id=20}: sequenced_broadcast: 104|SequencedBroadcastWorker Stopped id=104 elapsed=21.089478171s iter=2 [INFO] [stdout] 2025-09-01T11:55:50.483096Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:50.483144Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=11 leader=1 [INFO] [stdout] 2025-09-01T11:55:50.485896Z INFO Node{id=14}:connect{target=11}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:50.485961Z INFO Node{id=14}:connect{target=11}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:50.487747Z INFO Node{id=11}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:55:50.513561Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727750485) } [INFO] [stdout] 2025-09-01T11:55:50.522138Z INFO Node{id=14}:connect{target=11}: sharedstate::worker::sync_manager: got pong from peer latency=37 [INFO] [stdout] 2025-09-01T11:55:50.522185Z INFO Node{id=14}:connect{target=11}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:50.537043Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:50.543843Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 11 } [INFO] [stdout] 2025-09-01T11:55:50.543878Z ERROR Node{id=14}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 29, 5, 21, 11] peer=11 leader=1 local=14 [INFO] [stdout] 2025-09-01T11:55:50.546413Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:50.546471Z INFO Node{id=14}:connect{target=11}:WritePeer{remote=11}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:50.547841Z INFO Node{id=14}:connect{target=11}:ReadPeer{remote=11}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:50.548317Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:50.560904Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:50.687391Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:50.694003Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:55:50.701965Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:51.063177Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:51.063225Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:55:51.065026Z INFO Node{id=32}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:51.065094Z INFO Node{id=32}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:51.066814Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:51.112486Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:51.134038Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:51.149196Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:55:51.185196Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:51.338948Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:51.351660Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16]) [INFO] [stdout] 2025-09-01T11:55:51.464595Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:51.551189Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:51.551260Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:55:51.553518Z INFO Node{id=14}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:51.553597Z INFO Node{id=14}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:51.554889Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:55:51.566677Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727751553) } [INFO] [stdout] 2025-09-01T11:55:51.571014Z INFO Node{id=14}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:51.571046Z INFO Node{id=14}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:51.580261Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:51.585693Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:55:51.585722Z ERROR Node{id=14}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3] peer=3 leader=1 local=14 [INFO] [stdout] 2025-09-01T11:55:51.587573Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:51.587634Z INFO Node{id=14}:connect{target=3}:WritePeer{remote=3}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:51.589128Z INFO Node{id=14}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:51.590604Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:51.591406Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:51.812722Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:51.852659Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:52.094632Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:52.098831Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 28, 31]) [INFO] [stdout] 2025-09-01T11:55:52.589123Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:52.589171Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:55:52.591370Z INFO Node{id=14}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:52.591440Z INFO Node{id=14}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:52.592750Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:55:52.610602Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727752591) } [INFO] [stdout] 2025-09-01T11:55:52.614530Z INFO Node{id=14}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=23 [INFO] [stdout] 2025-09-01T11:55:52.614574Z INFO Node{id=14}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:52.623559Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:52.630888Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:55:52.630926Z ERROR Node{id=14}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=22 leader=1 local=14 [INFO] [stdout] 2025-09-01T11:55:52.632289Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:52.632342Z INFO Node{id=14}:connect{target=22}:WritePeer{remote=22}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:52.633914Z INFO Node{id=14}:connect{target=22}:ReadPeer{remote=22}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:52.634468Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:52.638666Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:53.128784Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:53.251042Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:53.254241Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 28]) [INFO] [stdout] 2025-09-01T11:55:53.472739Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:53.478056Z INFO Node{id=30}:connect{target=32}:ReadPeer{remote=32}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:53.503541Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:53.511665Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 28, 31]) [INFO] [stdout] 2025-09-01T11:55:53.541257Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:53.552010Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:53.555181Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29]) [INFO] [stdout] 2025-09-01T11:55:53.556577Z ERROR ReadClient{remote=20}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:55:53.561936Z ERROR Node{id=20}:connect{target=1}:ReadPeer{remote=1}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:53.562204Z WARN Node{id=20}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:53.562220Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:53.567198Z INFO Node{id=1}: sequenced_broadcast: 5|Subscriber(20): channel closed, dropping [INFO] [stdout] 2025-09-01T11:55:53.634680Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:53.634726Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=4 leader=1 [INFO] [stdout] 2025-09-01T11:55:53.635628Z INFO Node{id=14}:connect{target=4}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:53.635692Z INFO Node{id=14}:connect{target=4}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:53.636827Z INFO Node{id=4}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:55:53.644580Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727753635) } [INFO] [stdout] 2025-09-01T11:55:53.649906Z INFO Node{id=14}:connect{target=4}: sharedstate::worker::sync_manager: got pong from peer latency=14 [INFO] [stdout] 2025-09-01T11:55:53.649948Z INFO Node{id=14}:connect{target=4}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:53.654659Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:53.661886Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 4 } [INFO] [stdout] 2025-09-01T11:55:53.667960Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:53.669014Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 11944, state_accept_seq: 11943, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:53.670465Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:53.670829Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:53.672877Z INFO Node{id=4}: sequenced_broadcast: 79|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 4770 [INFO] [stdout] 2025-09-01T11:55:53.672899Z INFO Node{id=4}: sequenced_broadcast: 1|Subscriber(79): channel closed, dropping [INFO] [stdout] 2025-09-01T11:55:53.679183Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:53.679293Z INFO Node{id=14}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:53.680512Z INFO Node{id=14}: sequenced_broadcast: 126|SequencedBroadcastWorker Started id=126 [INFO] [stdout] 2025-09-01T11:55:53.682142Z INFO Node{id=14}: sequenced_broadcast: 75|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:53.682160Z INFO Node{id=14}: sequenced_broadcast: 75|SequencedBroadcastWorker Stopped id=75 elapsed=62.269447218s iter=5621 [INFO] [stdout] 2025-09-01T11:55:53.736341Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727749440) } [INFO] [stdout] 2025-09-01T11:55:53.739873Z INFO Node{id=19}:connect{target=28}: sharedstate::worker::sync_manager: got pong from peer latency=4299 [INFO] [stdout] 2025-09-01T11:55:53.739908Z INFO Node{id=19}:connect{target=28}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:53.845286Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:53.978509Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:54.056677Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:54.059224Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 15]) [INFO] [stdout] 2025-09-01T11:55:54.086953Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727749726) } [INFO] [stdout] 2025-09-01T11:55:54.090872Z INFO Node{id=25}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=4364 [INFO] [stdout] 2025-09-01T11:55:54.090910Z INFO Node{id=25}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:54.170002Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:54.174833Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:55:54.450212Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:54.488160Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:54.488203Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:55:54.489003Z INFO Node{id=30}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:54.489066Z INFO Node{id=30}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:54.489743Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:55:54.497918Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727754489) } [INFO] [stdout] 2025-09-01T11:55:54.500437Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:54.500821Z INFO Node{id=30}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:55:54.500849Z INFO Node{id=30}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:54.503627Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 29, 5]) [INFO] [stdout] 2025-09-01T11:55:54.504690Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:54.510245Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:55:54.510272Z ERROR Node{id=30}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=17 leader=1 local=30 [INFO] [stdout] 2025-09-01T11:55:54.511079Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:54.511397Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:54.512453Z ERROR Node{id=30}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:55:54.512538Z INFO Node{id=30}:connect{target=17}:WritePeer{remote=17}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:54.513808Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:54.568455Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:54.568497Z INFO Node{id=20}: sharedstate::worker::sync_manager: found connect target option_count=7 peers=8 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:55:54.569823Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:54.569893Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:54.570821Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=20 [INFO] [stdout] 2025-09-01T11:55:54.582671Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: Ping(1756727754569) } [INFO] [stdout] 2025-09-01T11:55:54.585180Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:55:54.585209Z INFO Node{id=20}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:54.590508Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:54.598182Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:55:54.602110Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:54.603784Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 25404, state_accept_seq: 25403, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:54.604441Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:54.605445Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber(6): Added, allow_drop: true, next_sequence: 25404, min_allowed_seq: 23503 [INFO] [stdout] 2025-09-01T11:55:54.610633Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 25404) [INFO] [stdout] 2025-09-01T11:55:54.610706Z INFO Node{id=20}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:55:54.801335Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:54.807228Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:54.808947Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6859, state_accept_seq: 6858, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:54.811465Z ERROR Node{id=27}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:54.814833Z WARN Node{id=27}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:54.814856Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:54.816036Z INFO Node{id=27}:connect{target=30}:WritePeer{remote=30}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:55.046027Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:55.089489Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727751065) } [INFO] [stdout] 2025-09-01T11:55:55.093268Z INFO Node{id=32}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=4028 [INFO] [stdout] 2025-09-01T11:55:55.093305Z INFO Node{id=32}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:55.106263Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:55.112511Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:55.113469Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 11852, state_accept_seq: 11851, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:55.116253Z ERROR Node{id=17}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:55.117831Z WARN Node{id=17}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:55.117850Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:55.119174Z INFO Node{id=17}:connect{target=29}:WritePeer{remote=29}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:55.403142Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:55.479698Z ERROR Node{id=15}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:55:55.482614Z WARN Node{id=15}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:55:55.482634Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:55.515429Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:55.515473Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:55:55.517034Z INFO Node{id=30}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:55.517121Z INFO Node{id=30}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:55.518477Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:55:55.530170Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727755517) } [INFO] [stdout] 2025-09-01T11:55:55.535091Z INFO Node{id=30}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:55.535128Z INFO Node{id=30}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:55.544948Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:55.559074Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:55:55.559120Z ERROR Node{id=30}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3] peer=3 leader=1 local=30 [INFO] [stdout] 2025-09-01T11:55:55.559282Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:55.560577Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:55.562473Z INFO Node{id=30}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:55.562555Z INFO Node{id=30}:connect{target=3}:WritePeer{remote=3}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:55.566113Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:55.579851Z ERROR ReadClient{remote=15}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "connection shutdown" }) [INFO] [stdout] 2025-09-01T11:55:55.817538Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:55.817588Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:55:55.819002Z INFO Node{id=27}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:55.819072Z INFO Node{id=27}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:55.820639Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:55:55.832875Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727755819) } [INFO] [stdout] 2025-09-01T11:55:55.836163Z INFO Node{id=27}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=17 [INFO] [stdout] 2025-09-01T11:55:55.836194Z INFO Node{id=27}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:55.842708Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:55.851440Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:55:55.857076Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:55.858384Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6859, state_accept_seq: 6858, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:55.860440Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:55.861137Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:55.868325Z INFO Node{id=24}: sequenced_broadcast: 120|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:55.875302Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:55.875426Z INFO Node{id=27}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:55.876545Z INFO Node{id=27}: sequenced_broadcast: 127|SequencedBroadcastWorker Started id=127 [INFO] [stdout] 2025-09-01T11:55:55.878495Z INFO Node{id=27}: sequenced_broadcast: 98|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:55.878515Z INFO Node{id=27}: sequenced_broadcast: 98|SequencedBroadcastWorker Stopped id=98 elapsed=31.125715161s iter=2 [INFO] [stdout] 2025-09-01T11:55:56.120524Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:56.120580Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=28 leader=1 [INFO] [stdout] 2025-09-01T11:55:56.122154Z INFO Node{id=17}:connect{target=28}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:56.122227Z INFO Node{id=17}:connect{target=28}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:56.123630Z INFO Node{id=28}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:55:56.488353Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:56.488403Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:55:56.490015Z INFO Node{id=15}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:56.490090Z INFO Node{id=15}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:56.491555Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:55:56.504521Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727756490) } [INFO] [stdout] 2025-09-01T11:55:56.508696Z INFO Node{id=15}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:56.508746Z INFO Node{id=15}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:56.516384Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:56.525628Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:55:56.529699Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:56.536115Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:56.536246Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:56.538642Z INFO Node{id=6}: sequenced_broadcast: 76|Subscriber(3): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 4770 [INFO] [stdout] 2025-09-01T11:55:56.548197Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:56.548280Z INFO Node{id=15}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:55:56.564097Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:56.564146Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:55:56.565643Z INFO Node{id=30}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:56.565716Z INFO Node{id=30}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:56.568459Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:55:56.583123Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:56.585464Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727756565) } [INFO] [stdout] 2025-09-01T11:55:56.587235Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:56.592531Z INFO Node{id=30}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=27 [INFO] [stdout] 2025-09-01T11:55:56.592590Z INFO Node{id=30}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:56.594616Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:56.601341Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9]) [INFO] [stdout] 2025-09-01T11:55:56.603077Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:56.612133Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:55:56.612181Z ERROR Node{id=30}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 31, 7] peer=7 leader=1 local=30 [INFO] [stdout] 2025-09-01T11:55:56.615794Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:56.617322Z INFO Node{id=30}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:56.617404Z INFO Node{id=30}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:56.619435Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:56.621019Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:56.645051Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:56.651596Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 15, 26]) [INFO] [stdout] 2025-09-01T11:55:57.139913Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:57.246551Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:57.255579Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 29, 5, 21]) [INFO] [stdout] 2025-09-01T11:55:57.435791Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:57.586174Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:57.595522Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 28 } [INFO] [stdout] 2025-09-01T11:55:57.595565Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28] peer=28 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:55:57.596940Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:57.597014Z INFO Node{id=19}:connect{target=28}:WritePeer{remote=28}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:57.598133Z INFO Node{id=19}:connect{target=28}:ReadPeer{remote=28}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:57.598233Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:57.619319Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:57.619364Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:55:57.620462Z INFO Node{id=30}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:57.620525Z INFO Node{id=30}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:57.621790Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:55:57.631462Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727757620) } [INFO] [stdout] 2025-09-01T11:55:57.638188Z INFO Node{id=30}:connect{target=19}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:57.638233Z INFO Node{id=30}:connect{target=19}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:57.647091Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:57.659865Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:57.663436Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 19 } [INFO] [stdout] 2025-09-01T11:55:57.663462Z ERROR Node{id=30}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=19 leader=1 local=30 [INFO] [stdout] 2025-09-01T11:55:57.665262Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:57.666666Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:57.670288Z INFO Node{id=30}:connect{target=19}:ReadPeer{remote=19}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:57.670375Z INFO Node{id=30}:connect{target=19}:WritePeer{remote=19}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:57.671575Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:57.919868Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:57.922646Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 28, 13]) [INFO] [stdout] 2025-09-01T11:55:57.922683Z WARN Node{id=6}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:55:57.924241Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:57.925451Z INFO Node{id=6}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:57.928797Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:58.002930Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:58.007915Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:55:58.007948Z ERROR Node{id=25}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=30 leader=1 local=25 [INFO] [stdout] 2025-09-01T11:55:58.009122Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:58.009175Z INFO Node{id=25}:connect{target=30}:WritePeer{remote=30}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:58.014156Z INFO Node{id=25}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:58.014611Z ERROR ReadClient{remote=25}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:58.025492Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:58.028075Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6859, state_accept_seq: 6858, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:58.029216Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:58.348615Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:55:58.348656Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 8 peers to 7 peers [INFO] [stdout] 2025-09-01T11:55:58.599260Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:58.599306Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:55:58.600476Z INFO Node{id=19}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:58.600552Z INFO Node{id=19}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:58.601560Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:58.613541Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727758600) } [INFO] [stdout] 2025-09-01T11:55:58.618258Z INFO Node{id=19}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:55:58.618298Z INFO Node{id=19}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:58.626656Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:58.630839Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:55:58.630870Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18] peer=18 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:55:58.632675Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:58.632739Z INFO Node{id=19}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:58.633854Z INFO Node{id=19}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:58.634370Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:58.635689Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:58.667851Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:58.667894Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:55:58.669030Z INFO Node{id=30}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:58.669095Z INFO Node{id=30}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:58.669943Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:55:58.678613Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727758669) } [INFO] [stdout] 2025-09-01T11:55:58.682151Z INFO Node{id=30}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:55:58.682182Z INFO Node{id=30}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:58.688308Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:58.695198Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:55:58.695230Z ERROR Node{id=30}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9] peer=9 leader=1 local=30 [INFO] [stdout] 2025-09-01T11:55:58.696492Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:58.697109Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:58.698106Z ERROR Node{id=30}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:55:58.698180Z INFO Node{id=30}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:58.698916Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:58.907291Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:58.915391Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:58.917889Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29]) [INFO] [stdout] 2025-09-01T11:55:58.923173Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:55:58.923205Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 13] peer=13 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:55:58.924442Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:58.925630Z INFO Node{id=32}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:58.925817Z INFO Node{id=32}:connect{target=13}:WritePeer{remote=13}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:58.926592Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:58.929958Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:58.929993Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:55:58.931448Z INFO Node{id=6}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:58.931526Z INFO Node{id=6}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:58.932867Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:55:58.941337Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727758931) } [INFO] [stdout] 2025-09-01T11:55:58.945128Z INFO Node{id=6}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=14 [INFO] [stdout] 2025-09-01T11:55:58.945161Z INFO Node{id=6}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:58.951428Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:58.963509Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:58.968873Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:55:58.968905Z ERROR Node{id=6}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=25 leader=1 local=6 [INFO] [stdout] 2025-09-01T11:55:58.970876Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:58.973284Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:58.974251Z INFO Node{id=6}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:58.974325Z INFO Node{id=6}:connect{target=25}:WritePeer{remote=25}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:58.975882Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:58.999146Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:59.004215Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:55:59.004244Z ERROR Node{id=24}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:55:59.005619Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:59.005948Z INFO Node{id=24}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:59.010742Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:59.010793Z INFO Node{id=25}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:55:59.014429Z INFO Node{id=25}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:59.014513Z INFO Node{id=25}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:59.016002Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=25 [INFO] [stdout] 2025-09-01T11:55:59.035905Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: Ping(1756727759014) } [INFO] [stdout] 2025-09-01T11:55:59.041505Z INFO Node{id=25}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=27 [INFO] [stdout] 2025-09-01T11:55:59.041541Z INFO Node{id=25}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:59.051915Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:59.058470Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:55:59.066150Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:59.067950Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 11852, state_accept_seq: 11851, history: [] }) } [INFO] [stdout] 2025-09-01T11:55:59.069005Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:55:59.070265Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:55:59.074629Z INFO Node{id=18}: sequenced_broadcast: 112|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:55:59.084698Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:55:59.084916Z INFO Node{id=25}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:55:59.086251Z INFO Node{id=25}: sequenced_broadcast: 128|SequencedBroadcastWorker Started id=128 [INFO] [stdout] 2025-09-01T11:55:59.090693Z INFO Node{id=25}: sequenced_broadcast: 101|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:55:59.090716Z INFO Node{id=25}: sequenced_broadcast: 101|SequencedBroadcastWorker Stopped id=101 elapsed=31.052646808s iter=1529 [INFO] [stdout] 2025-09-01T11:55:59.275655Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:55:59.284354Z ERROR ReadClient{remote=24}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:59.622275Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:59.626131Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 29, 5, 21]) [INFO] [stdout] 2025-09-01T11:55:59.635939Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:59.635982Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:55:59.637507Z INFO Node{id=19}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:59.637581Z INFO Node{id=19}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:59.638868Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:55:59.651557Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727759637) } [INFO] [stdout] 2025-09-01T11:55:59.657517Z INFO Node{id=19}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:55:59.657572Z INFO Node{id=19}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:59.665259Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:59.673479Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:55:59.673524Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25] peer=25 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:55:59.677457Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:59.677527Z INFO Node{id=19}:connect{target=25}:WritePeer{remote=25}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:59.681062Z INFO Node{id=19}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:59.682483Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:59.682594Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:59.698285Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:59.698332Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:55:59.702159Z INFO Node{id=30}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:59.702322Z INFO Node{id=30}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:59.703689Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:55:59.719838Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:59.722056Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23]) [INFO] [stdout] 2025-09-01T11:55:59.926698Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:59.926745Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:55:59.928330Z INFO Node{id=32}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:59.928400Z INFO Node{id=32}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:59.929888Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:55:59.940028Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727759928) } [INFO] [stdout] 2025-09-01T11:55:59.944071Z INFO Node{id=32}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:55:59.944111Z INFO Node{id=32}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:55:59.951142Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:55:59.956626Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:55:59.956662Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 29, 5, 21, 2] peer=2 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:55:59.960772Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:55:59.960834Z INFO Node{id=32}:connect{target=2}:WritePeer{remote=2}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:55:59.962475Z INFO Node{id=32}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:55:59.964096Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:55:59.965415Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:55:59.977493Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:55:59.977537Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:55:59.978531Z INFO Node{id=6}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:55:59.978618Z INFO Node{id=6}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:55:59.979917Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:56:00.013080Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:00.013123Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=13 leader=1 [INFO] [stdout] 2025-09-01T11:56:00.013525Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727756122) } [INFO] [stdout] 2025-09-01T11:56:00.014529Z INFO Node{id=24}:connect{target=13}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:00.014613Z INFO Node{id=24}:connect{target=13}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:00.015931Z INFO Node{id=13}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:56:00.018290Z INFO Node{id=17}:connect{target=28}: sharedstate::worker::sync_manager: got pong from peer latency=3896 [INFO] [stdout] 2025-09-01T11:56:00.018321Z INFO Node{id=17}:connect{target=28}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:00.055866Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:00.678914Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:00.678960Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=11 leader=1 [INFO] [stdout] 2025-09-01T11:56:00.680498Z INFO Node{id=19}:connect{target=11}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:00.680584Z INFO Node{id=19}:connect{target=11}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:00.681240Z INFO Node{id=11}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:00.689847Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727760680) } [INFO] [stdout] 2025-09-01T11:56:00.692739Z INFO Node{id=19}:connect{target=11}: sharedstate::worker::sync_manager: got pong from peer latency=12 [INFO] [stdout] 2025-09-01T11:56:00.692796Z INFO Node{id=19}:connect{target=11}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:00.698186Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:00.703269Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 11 } [INFO] [stdout] 2025-09-01T11:56:00.703302Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 29, 5, 21, 11] peer=11 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:56:00.705336Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:00.705390Z INFO Node{id=19}:connect{target=11}:WritePeer{remote=11}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:00.706903Z INFO Node{id=19}:connect{target=11}:ReadPeer{remote=11}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:00.707452Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:00.709971Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:00.964242Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:00.964290Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:56:00.966206Z INFO Node{id=32}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:00.966280Z INFO Node{id=32}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:00.968354Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:56:00.982644Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727760966) } [INFO] [stdout] 2025-09-01T11:56:00.986885Z INFO Node{id=32}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:56:00.986920Z INFO Node{id=32}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:00.993553Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:00.999343Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:56:00.999382Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 31, 7] peer=7 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:56:01.001435Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:01.001490Z INFO Node{id=32}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:01.002923Z INFO Node{id=32}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:01.004129Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:01.005140Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:01.113442Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:01.191126Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:01.352785Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:01.491205Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:01.493142Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16]) [INFO] [stdout] 2025-09-01T11:56:01.500856Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:01.519234Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:01.522243Z INFO Node{id=29}: sequenced_broadcast: 90|Subscriber(3): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 4770 [INFO] [stdout] 2025-09-01T11:56:01.527461Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:01.527569Z INFO Node{id=22}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:01.528791Z INFO Node{id=22}: sequenced_broadcast: 129|SequencedBroadcastWorker Started id=129 [INFO] [stdout] 2025-09-01T11:56:01.529883Z INFO Node{id=22}: sequenced_broadcast: 103|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:01.529901Z INFO Node{id=22}: sequenced_broadcast: 103|SequencedBroadcastWorker Stopped id=103 elapsed=32.519690753s iter=2 [INFO] [stdout] 2025-09-01T11:56:01.706636Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:01.706680Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:56:01.707740Z INFO Node{id=19}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:01.707830Z INFO Node{id=19}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:01.709054Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:01.717143Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727761707) } [INFO] [stdout] 2025-09-01T11:56:01.720227Z INFO Node{id=19}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:56:01.720257Z INFO Node{id=19}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:01.725674Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:01.729551Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:56:01.729579Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=17 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:56:01.731317Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:01.731369Z INFO Node{id=19}:connect{target=17}:WritePeer{remote=17}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:01.732283Z INFO Node{id=19}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:01.732849Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:01.734958Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:02.002929Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:02.002987Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:56:02.004172Z INFO Node{id=32}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:02.004236Z INFO Node{id=32}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:02.005048Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:56:02.014433Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727762004) } [INFO] [stdout] 2025-09-01T11:56:02.018676Z INFO Node{id=32}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=14 [INFO] [stdout] 2025-09-01T11:56:02.018715Z INFO Node{id=32}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:02.027172Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:02.033520Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:56:02.033565Z ERROR Node{id=32}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9] peer=9 leader=1 local=32 [INFO] [stdout] 2025-09-01T11:56:02.035838Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:02.035884Z INFO Node{id=32}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:02.037113Z INFO Node{id=32}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:02.038727Z ERROR ReadClient{remote=32}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:02.039329Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:02.061650Z INFO sharedstate::testing::fuzzy_test: unblock all connections and let system recover [INFO] [stdout] 2025-09-01T11:56:02.100195Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:02.110709Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:02.115203Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 28, 31]) [INFO] [stdout] 2025-09-01T11:56:02.424440Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:02.732931Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:02.732978Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:56:02.733908Z INFO Node{id=19}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:02.733974Z INFO Node{id=19}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:02.734720Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:02.742500Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727762733) } [INFO] [stdout] 2025-09-01T11:56:02.749395Z INFO Node{id=19}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:56:02.749433Z INFO Node{id=19}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:02.756124Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:02.763880Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:56:02.763922Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 24, 27] peer=27 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:56:02.767220Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:02.767274Z INFO Node{id=19}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:02.767641Z INFO Node{id=19}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:02.768334Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:02.774398Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:02.930469Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727759702) } [INFO] [stdout] 2025-09-01T11:56:02.936896Z INFO Node{id=30}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=3234 [INFO] [stdout] 2025-09-01T11:56:02.936936Z INFO Node{id=30}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:03.037280Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:03.037327Z INFO Node{id=32}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:56:03.038013Z INFO Node{id=32}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:03.038081Z INFO Node{id=32}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:03.039018Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=32 [INFO] [stdout] 2025-09-01T11:56:03.052044Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: Ping(1756727763038) } [INFO] [stdout] 2025-09-01T11:56:03.052086Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=32 [INFO] [stdout] 2025-09-01T11:56:03.054841Z INFO Node{id=32}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=16 [INFO] [stdout] 2025-09-01T11:56:03.054871Z INFO Node{id=32}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:03.063630Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:03.066857Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:56:03.072315Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:03.073267Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:03.074440Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:03.074781Z INFO Node{id=20}: sequenced_broadcast: 125|Subscriber rejected, seq(6818) < min_allowed(25698) [INFO] [stdout] 2025-09-01T11:56:03.076324Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:03.080084Z INFO Node{id=20}: sequenced_broadcast: 125|Subscriber(1): Added, allow_drop: true, next_sequence: 27746, min_allowed_seq: 25698 [INFO] [stdout] 2025-09-01T11:56:03.087556Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 27746) [INFO] [stdout] 2025-09-01T11:56:03.087672Z INFO Node{id=32}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:03.088499Z INFO Node{id=32}: sequenced_broadcast: 130|SequencedBroadcastWorker Started id=130 [INFO] [stdout] 2025-09-01T11:56:03.089708Z INFO Node{id=32}: sequenced_broadcast: 81|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:03.089727Z INFO Node{id=32}: sequenced_broadcast: 81|SequencedBroadcastWorker Stopped id=81 elapsed=69.623099348s iter=2822 [INFO] [stdout] 2025-09-01T11:56:03.162015Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727759978) } [INFO] [stdout] 2025-09-01T11:56:03.169152Z INFO Node{id=6}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=3191 [INFO] [stdout] 2025-09-01T11:56:03.169191Z INFO Node{id=6}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:03.216200Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727760014) } [INFO] [stdout] 2025-09-01T11:56:03.218648Z INFO Node{id=24}:connect{target=13}: sharedstate::worker::sync_manager: got pong from peer latency=3204 [INFO] [stdout] 2025-09-01T11:56:03.218681Z INFO Node{id=24}:connect{target=13}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:03.255971Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:03.351242Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:03.351290Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 9 peers to 8 peers [INFO] [stdout] 2025-09-01T11:56:03.356910Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 9) [INFO] [stdout] 2025-09-01T11:56:03.364594Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:03.367926Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 28 } [INFO] [stdout] 2025-09-01T11:56:03.367954Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28] peer=28 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:03.369150Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:03.369218Z INFO Node{id=17}:connect{target=28}:WritePeer{remote=28}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:03.369713Z INFO Node{id=17}:connect{target=28}:ReadPeer{remote=28}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:03.372093Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:03.513152Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:03.516458Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:03.518968Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 28, 31]) [INFO] [stdout] 2025-09-01T11:56:03.558636Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:03.663777Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:03.668136Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:03.672836Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 4]) [INFO] [stdout] 2025-09-01T11:56:03.768993Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:03.769040Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:56:03.770039Z INFO Node{id=19}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:03.770129Z INFO Node{id=19}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:03.773065Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:03.780610Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727763770) } [INFO] [stdout] 2025-09-01T11:56:03.785228Z INFO Node{id=19}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:56:03.785273Z INFO Node{id=19}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:03.788659Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:03.794262Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:56:03.794299Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 29, 5, 21, 2] peer=2 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:56:03.798891Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:03.799000Z INFO Node{id=19}:connect{target=2}:WritePeer{remote=2}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:03.799576Z INFO Node{id=19}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:03.800083Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:03.801564Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:04.061853Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:04.066153Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:04.068907Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 6, 15]) [INFO] [stdout] 2025-09-01T11:56:04.177058Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:04.371203Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:04.371340Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:56:04.372110Z INFO Node{id=17}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:04.372175Z INFO Node{id=17}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:04.372967Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:04.379857Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727764372) } [INFO] [stdout] 2025-09-01T11:56:04.381969Z INFO Node{id=17}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:56:04.381996Z INFO Node{id=17}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:04.385799Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:04.389892Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:56:04.389921Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 29, 5, 21, 2] peer=2 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:04.390901Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:04.390986Z INFO Node{id=17}:connect{target=2}:WritePeer{remote=2}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:04.391440Z INFO Node{id=17}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:04.391696Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:04.397147Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:04.506063Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:04.525258Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:04.527692Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5]) [INFO] [stdout] 2025-09-01T11:56:04.600210Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:04.608209Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:04.610733Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:56:04.801869Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:04.801916Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:56:04.802851Z INFO Node{id=19}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:04.802925Z INFO Node{id=19}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:04.803847Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:05.303818Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:05.310041Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 13 } [INFO] [stdout] 2025-09-01T11:56:05.392450Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:05.392496Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:56:05.393327Z INFO Node{id=17}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:05.393396Z INFO Node{id=17}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:05.394254Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:05.400984Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727765393) } [INFO] [stdout] 2025-09-01T11:56:05.405536Z INFO Node{id=17}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=12 [INFO] [stdout] 2025-09-01T11:56:05.405577Z INFO Node{id=17}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:05.409121Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:05.414479Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:56:05.414511Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9] peer=9 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:05.415598Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:05.415653Z INFO Node{id=17}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:05.416194Z INFO Node{id=17}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:05.416708Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:05.417928Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:05.427320Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:05.433671Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:56:05.433707Z ERROR Node{id=6}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=30 leader=1 local=6 [INFO] [stdout] 2025-09-01T11:56:05.434863Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:05.435706Z INFO Node{id=6}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:05.435797Z INFO Node{id=6}:connect{target=30}:WritePeer{remote=30}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:05.436632Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:05.618910Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:05.620435Z INFO Node{id=28}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:05.631667Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:05.666104Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:05.852723Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:05.857038Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:05.859175Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:05.859198Z ERROR Node{id=27}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:06.109815Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:06.113679Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:56:06.113704Z ERROR Node{id=30}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29] peer=29 leader=1 local=30 [INFO] [stdout] 2025-09-01T11:56:06.114489Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:06.115126Z INFO Node{id=30}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:06.115198Z INFO Node{id=30}:connect{target=29}:WritePeer{remote=29}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:06.115964Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:06.416292Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:06.416342Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:56:06.416999Z INFO Node{id=17}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:06.417070Z INFO Node{id=17}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:06.417752Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:06.423216Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727766417) } [INFO] [stdout] 2025-09-01T11:56:06.425792Z INFO Node{id=17}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=8 [INFO] [stdout] 2025-09-01T11:56:06.425820Z INFO Node{id=17}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:06.429882Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:06.433484Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:56:06.433510Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 31, 7] peer=7 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:06.435096Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:06.435153Z INFO Node{id=17}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:06.436141Z INFO Node{id=17}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:06.436322Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:06.436351Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:56:06.436930Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:06.437148Z INFO Node{id=6}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:06.437215Z INFO Node{id=6}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:06.437281Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:06.437998Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:56:06.526067Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:06.602528Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:06.608081Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:06.609668Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9]) [INFO] [stdout] 2025-09-01T11:56:06.623715Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:06.623778Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:56:06.624300Z INFO Node{id=28}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:06.624375Z INFO Node{id=28}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:06.625203Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:06.630473Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727766624) } [INFO] [stdout] 2025-09-01T11:56:06.632587Z INFO Node{id=28}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=8 [INFO] [stdout] 2025-09-01T11:56:06.632617Z INFO Node{id=28}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:06.637245Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:06.640499Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:56:06.640523Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25] peer=25 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:56:06.642153Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:06.642718Z INFO Node{id=28}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:06.642814Z INFO Node{id=28}:connect{target=25}:WritePeer{remote=25}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:06.643704Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:06.643904Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:06.652228Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:06.656840Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:06.658991Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 6, 15, 26]) [INFO] [stdout] 2025-09-01T11:56:06.725395Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:06.726633Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:06.726656Z ERROR Node{id=15}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:06.915619Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727764802) } [INFO] [stdout] 2025-09-01T11:56:06.916692Z INFO Node{id=19}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=2114 [INFO] [stdout] 2025-09-01T11:56:06.916724Z INFO Node{id=19}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:07.077845Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:07.078957Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:07.079286Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:07.079922Z INFO Node{id=13}: sequenced_broadcast: 87|Subscriber(4): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 4770 [INFO] [stdout] 2025-09-01T11:56:07.079943Z INFO Node{id=13}: sequenced_broadcast: 1|Subscriber(87): channel closed, dropping [INFO] [stdout] 2025-09-01T11:56:07.081806Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:07.081864Z INFO Node{id=24}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:07.115749Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:07.115813Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=22 leader=1 [INFO] [stdout] 2025-09-01T11:56:07.116386Z INFO Node{id=30}:connect{target=22}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:07.116453Z INFO Node{id=30}:connect{target=22}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:07.117058Z INFO Node{id=22}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:56:07.121060Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727767116) } [INFO] [stdout] 2025-09-01T11:56:07.122514Z INFO Node{id=30}:connect{target=22}: sharedstate::worker::sync_manager: got pong from peer latency=6 [INFO] [stdout] 2025-09-01T11:56:07.122549Z INFO Node{id=30}:connect{target=22}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:07.130057Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:07.134723Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:07.154080Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 22 } [INFO] [stdout] 2025-09-01T11:56:07.154118Z ERROR Node{id=30}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 22] peer=22 leader=1 local=30 [INFO] [stdout] 2025-09-01T11:56:07.155008Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:07.155327Z INFO Node{id=30}:connect{target=22}:ReadPeer{remote=22}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:07.155395Z INFO Node{id=30}:connect{target=22}:WritePeer{remote=22}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:07.155518Z ERROR ReadClient{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:07.176436Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:07.256463Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:07.258661Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:07.259634Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5, 21]) [INFO] [stdout] 2025-09-01T11:56:07.314151Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727766437) } [INFO] [stdout] 2025-09-01T11:56:07.314193Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=6 [INFO] [stdout] 2025-09-01T11:56:07.315306Z INFO Node{id=6}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=878 [INFO] [stdout] 2025-09-01T11:56:07.315337Z INFO Node{id=6}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:07.438097Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:07.438147Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:56:07.438893Z INFO Node{id=17}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:07.438962Z INFO Node{id=17}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:07.439728Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:07.443481Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727767438) } [INFO] [stdout] 2025-09-01T11:56:07.443515Z INFO Node{id=8}: sharedstate::worker::sync_manager: Discovered Peer client_id=17 [INFO] [stdout] 2025-09-01T11:56:07.443716Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:07.443742Z INFO Node{id=8}: sharedstate::worker::sync_manager: Broadcasted 8 peers to 7 peers [INFO] [stdout] 2025-09-01T11:56:07.444406Z INFO Node{id=17}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=6 [INFO] [stdout] 2025-09-01T11:56:07.444440Z INFO Node{id=17}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:07.445829Z ERROR Node{id=17}:connect{target=8}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:07.446052Z INFO Node{id=17}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:07.446119Z INFO Node{id=17}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:07.446180Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(8) [INFO] [stdout] 2025-09-01T11:56:07.446287Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:07.449897Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:07.643361Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:07.643407Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:56:07.643755Z INFO Node{id=28}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:07.643839Z INFO Node{id=28}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:07.643975Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:07.648021Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727767643) } [INFO] [stdout] 2025-09-01T11:56:07.649264Z INFO Node{id=28}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=6 [INFO] [stdout] 2025-09-01T11:56:07.649290Z INFO Node{id=28}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:07.652249Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:07.655312Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:56:07.655338Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 6, 15, 26] peer=26 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:56:07.655876Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:07.656362Z INFO Node{id=28}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:07.656435Z INFO Node{id=28}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:07.656601Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:07.656734Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:07.860042Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:07.862025Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:07.863214Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 28, 13, 24]) [INFO] [stdout] 2025-09-01T11:56:07.867836Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:08.158137Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:08.158186Z INFO Node{id=30}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:56:08.158634Z INFO Node{id=30}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:08.158713Z INFO Node{id=30}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:08.158894Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=30 [INFO] [stdout] 2025-09-01T11:56:08.366966Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:08.367018Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 10 peers to 9 peers [INFO] [stdout] 2025-09-01T11:56:08.367419Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 10) [INFO] [stdout] 2025-09-01T11:56:08.367458Z ERROR Node{id=6}:connect{target=20}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:08.367538Z INFO Node{id=6}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:08.367625Z INFO Node{id=6}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:08.367788Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(20) [INFO] [stdout] 2025-09-01T11:56:08.368278Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:08.447140Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:08.447188Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=25 leader=1 [INFO] [stdout] 2025-09-01T11:56:08.447804Z INFO Node{id=17}:connect{target=25}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:08.447874Z INFO Node{id=17}:connect{target=25}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:08.448029Z INFO Node{id=25}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:08.451010Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727768447) } [INFO] [stdout] 2025-09-01T11:56:08.451831Z INFO Node{id=17}:connect{target=25}: sharedstate::worker::sync_manager: got pong from peer latency=4 [INFO] [stdout] 2025-09-01T11:56:08.451857Z INFO Node{id=17}:connect{target=25}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:08.455249Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:08.456245Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 25 } [INFO] [stdout] 2025-09-01T11:56:08.457845Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:08.458264Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 11852, state_accept_seq: 11851, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:08.458754Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:08.459361Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:08.459879Z INFO Node{id=25}: sequenced_broadcast: 128|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:08.461118Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:08.461208Z INFO Node{id=17}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:08.461557Z INFO Node{id=17}: sequenced_broadcast: 131|SequencedBroadcastWorker Started id=131 [INFO] [stdout] 2025-09-01T11:56:08.462933Z INFO Node{id=17}: sequenced_broadcast: 113|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:08.462961Z INFO Node{id=17}: sequenced_broadcast: 113|SequencedBroadcastWorker Stopped id=113 elapsed=28.954769411s iter=2 [INFO] [stdout] 2025-09-01T11:56:08.657356Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:08.657402Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:56:08.658085Z INFO Node{id=28}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:08.658156Z INFO Node{id=28}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:08.659073Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:08.667644Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727768658) } [INFO] [stdout] 2025-09-01T11:56:08.669870Z INFO Node{id=28}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:56:08.669991Z INFO Node{id=28}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:08.674223Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:08.676820Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:56:08.676921Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 13, 24, 27] peer=27 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:56:08.677749Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:08.678355Z INFO Node{id=28}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:08.678434Z INFO Node{id=28}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:08.679738Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:08.683096Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:08.713378Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:08.716649Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:56:08.716676Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29] peer=29 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:56:08.717965Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:08.718011Z INFO Node{id=19}:connect{target=29}:WritePeer{remote=29}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:08.719094Z INFO Node{id=19}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:08.720255Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:08.727889Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:08.731921Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:08.733456Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:08.733478Z ERROR Node{id=15}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:08.919255Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:09.059661Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:09.092377Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:09.097251Z ERROR ReadClient{remote=12}: sharedstate::net::message_channels: failed to read from network error=NextMessageTimeout(12s) [INFO] [stdout] 2025-09-01T11:56:09.098726Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:09.099381Z ERROR Node{id=12}:connect{target=11}:ReadPeer{remote=11}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:09.100256Z WARN Node{id=12}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:56:09.100275Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:09.102517Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18]) [INFO] [stdout] 2025-09-01T11:56:09.276778Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:09.278629Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:09.278650Z ERROR Node{id=31}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:09.369832Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:09.369880Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:56:09.370836Z INFO Node{id=6}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:09.370927Z INFO Node{id=6}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:09.371582Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:56:09.379186Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727769370) } [INFO] [stdout] 2025-09-01T11:56:09.381048Z INFO Node{id=6}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:56:09.381083Z INFO Node{id=6}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:09.385205Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:09.390219Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:56:09.390254Z ERROR Node{id=6}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 13, 24] peer=24 leader=1 local=6 [INFO] [stdout] 2025-09-01T11:56:09.392032Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:09.392086Z INFO Node{id=6}:connect{target=24}:WritePeer{remote=24}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:09.392404Z INFO Node{id=6}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:09.393161Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:09.394433Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:09.627287Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:09.632243Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:09.634903Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5, 21]) [INFO] [stdout] 2025-09-01T11:56:09.679555Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:09.679602Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:56:09.680055Z INFO Node{id=28}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:09.680139Z INFO Node{id=28}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:09.681033Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:09.719288Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:09.719344Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:56:09.720248Z INFO Node{id=19}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:09.720336Z INFO Node{id=19}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:09.721168Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:09.723798Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:09.726617Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:09.727158Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727769720) } [INFO] [stdout] 2025-09-01T11:56:09.727180Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=19 [INFO] [stdout] 2025-09-01T11:56:09.728153Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:09.728584Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:09.728615Z INFO Node{id=10}: sharedstate::worker::sync_manager: Broadcasted 13 peers to 12 peers [INFO] [stdout] 2025-09-01T11:56:09.728909Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29]) [INFO] [stdout] 2025-09-01T11:56:09.729968Z INFO Node{id=19}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:56:09.729995Z INFO Node{id=19}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:09.731034Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23]) [INFO] [stdout] 2025-09-01T11:56:09.731894Z ERROR Node{id=19}:connect{target=10}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:09.732530Z INFO Node{id=19}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:09.732731Z INFO Node{id=19}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:09.732860Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(10) [INFO] [stdout] 2025-09-01T11:56:09.733502Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:09.734692Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:10.107659Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:10.107710Z INFO Node{id=12}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:56:10.108289Z INFO Node{id=12}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:10.108380Z INFO Node{id=12}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:10.108580Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=12 [INFO] [stdout] 2025-09-01T11:56:10.112493Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: Ping(1756727770108) } [INFO] [stdout] 2025-09-01T11:56:10.114054Z INFO Node{id=12}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=6 [INFO] [stdout] 2025-09-01T11:56:10.114078Z INFO Node{id=12}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:10.116284Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:10.118534Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:56:10.123396Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:10.124578Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:10.125501Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:10.126185Z INFO Node{id=16}: sequenced_broadcast: 100|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:10.129014Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:10.129077Z INFO Node{id=12}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:10.280828Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:10.282107Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29]) [INFO] [stdout] 2025-09-01T11:56:10.393431Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:10.393477Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:56:10.394230Z INFO Node{id=6}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:10.394324Z INFO Node{id=6}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:10.394654Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:56:10.546435Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:10.686861Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: Ping(1756727768158) } [INFO] [stdout] 2025-09-01T11:56:10.686899Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=30 [INFO] [stdout] 2025-09-01T11:56:10.687831Z INFO Node{id=30}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=2529 [INFO] [stdout] 2025-09-01T11:56:10.687859Z INFO Node{id=30}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:10.690558Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:10.693617Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:10.695734Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 20 } [INFO] [stdout] 2025-09-01T11:56:10.696391Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:10.696573Z INFO Node{id=20}: sequenced_broadcast: 125|Subscriber rejected, seq(6818) < min_allowed(29795) [INFO] [stdout] 2025-09-01T11:56:10.696909Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:10.697084Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:10.697665Z INFO Node{id=20}: sequenced_broadcast: 125|Subscriber(2): Added, allow_drop: true, next_sequence: 31843, min_allowed_seq: 29795 [INFO] [stdout] 2025-09-01T11:56:10.698106Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 31843) [INFO] [stdout] 2025-09-01T11:56:10.698367Z INFO Node{id=30}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:10.698397Z INFO Node{id=30}: sequenced_broadcast: 132|SequencedBroadcastWorker Started id=132 [INFO] [stdout] 2025-09-01T11:56:10.698488Z INFO Node{id=30}: sequenced_broadcast: 85|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:10.698502Z INFO Node{id=30}: sequenced_broadcast: 85|SequencedBroadcastWorker Stopped id=85 elapsed=76.378059539s iter=2805 [INFO] [stdout] 2025-09-01T11:56:10.699512Z ERROR Node{id=16}:connect{target=30}:ReadPeer{remote=30}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:10.699584Z WARN Node{id=16}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:56:10.699596Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:10.735355Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:10.735416Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:10.735441Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:56:10.736307Z INFO Node{id=19}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:10.736381Z INFO Node{id=19}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:10.737321Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:10.740738Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:10.743285Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:10.743318Z ERROR Node{id=15}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:10.744745Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727770736) } [INFO] [stdout] 2025-09-01T11:56:10.746920Z INFO Node{id=19}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=10 [INFO] [stdout] 2025-09-01T11:56:10.746948Z INFO Node{id=19}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:10.751044Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:10.754253Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:56:10.754276Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=16 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:56:10.755538Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:10.755585Z INFO Node{id=19}:connect{target=16}:WritePeer{remote=16}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:10.756526Z INFO Node{id=19}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:10.757389Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:10.757493Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:11.279176Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:11.396576Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727769680) } [INFO] [stdout] 2025-09-01T11:56:11.397904Z INFO Node{id=28}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=1717 [INFO] [stdout] 2025-09-01T11:56:11.397933Z INFO Node{id=28}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:11.494253Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:11.499125Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:11.501222Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:11.501246Z ERROR Node{id=23}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:11.703320Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:11.703363Z INFO Node{id=16}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:56:11.703791Z INFO Node{id=16}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:11.703856Z INFO Node{id=16}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:11.704032Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=16 [INFO] [stdout] 2025-09-01T11:56:11.706774Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: Ping(1756727771703) } [INFO] [stdout] 2025-09-01T11:56:11.707618Z INFO Node{id=16}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=4 [INFO] [stdout] 2025-09-01T11:56:11.707645Z INFO Node{id=16}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:11.710268Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:11.711307Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:56:11.712592Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:11.714534Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:11.715407Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:11.715531Z INFO Node{id=17}: sequenced_broadcast: 131|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:11.718915Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:11.718982Z INFO Node{id=16}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:11.756184Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:11.756229Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:56:11.756695Z INFO Node{id=19}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:11.756786Z INFO Node{id=19}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:11.757075Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:11.762044Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727771756) } [INFO] [stdout] 2025-09-01T11:56:11.762684Z INFO Node{id=19}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=6 [INFO] [stdout] 2025-09-01T11:56:11.762710Z INFO Node{id=19}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:11.763362Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:11.764373Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:56:11.764395Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=6 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:56:11.764562Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:11.764618Z INFO Node{id=19}:connect{target=6}:WritePeer{remote=6}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:11.765091Z INFO Node{id=19}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:11.765983Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:11.766135Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:12.067288Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067330Z INFO Node{id=1}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067479Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067498Z INFO Node{id=30}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067594Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067611Z INFO Node{id=29}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067633Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067644Z INFO Node{id=13}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067663Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067672Z INFO Node{id=20}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067695Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067705Z INFO Node{id=2}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067725Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067734Z INFO Node{id=3}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067755Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067786Z INFO Node{id=4}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067812Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067822Z INFO Node{id=5}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067841Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067859Z INFO Node{id=6}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067885Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067896Z INFO Node{id=7}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067922Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067934Z INFO Node{id=8}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067956Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067966Z INFO Node{id=9}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.067984Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.067993Z INFO Node{id=10}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068014Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068024Z INFO Node{id=11}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068041Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068050Z INFO Node{id=12}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068068Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068078Z INFO Node{id=14}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068100Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068110Z INFO Node{id=15}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068167Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068268Z INFO Node{id=16}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068308Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068321Z INFO Node{id=17}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068344Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068355Z INFO Node{id=18}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068376Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068387Z INFO Node{id=19}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068411Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068422Z INFO Node{id=21}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068442Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068452Z INFO Node{id=22}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068472Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068483Z INFO Node{id=23}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068507Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068518Z INFO Node{id=24}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068541Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068553Z INFO Node{id=25}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068577Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068594Z INFO Node{id=26}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068622Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068634Z INFO Node{id=27}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068657Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068667Z INFO Node{id=31}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.068688Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.068697Z INFO Node{id=32}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.069709Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: SetLeader(1) [INFO] [stdout] 2025-09-01T11:56:12.069729Z INFO Node{id=28}: sharedstate::worker::sync_manager: ignore set leader, no change leader=1 [INFO] [stdout] 2025-09-01T11:56:12.116192Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:12.117232Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:12.117529Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([31]) [INFO] [stdout] 2025-09-01T11:56:12.126672Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727770394) } [INFO] [stdout] 2025-09-01T11:56:12.127025Z INFO Node{id=6}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=1733 [INFO] [stdout] 2025-09-01T11:56:12.127051Z INFO Node{id=6}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:12.745890Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:12.747275Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:12.748147Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:12.748168Z ERROR Node{id=15}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:12.767509Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:12.767554Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:56:12.767995Z INFO Node{id=19}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:12.768062Z INFO Node{id=19}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:12.768202Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:12.770402Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727772768) } [INFO] [stdout] 2025-09-01T11:56:12.770426Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=19 [INFO] [stdout] 2025-09-01T11:56:12.770900Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:12.770931Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 13 peers to 12 peers [INFO] [stdout] 2025-09-01T11:56:12.771220Z INFO Node{id=19}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:12.771246Z INFO Node{id=19}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:12.771628Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 13) [INFO] [stdout] 2025-09-01T11:56:12.772201Z ERROR Node{id=19}:connect{target=23}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:12.772322Z INFO Node{id=19}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:12.772392Z INFO Node{id=19}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:12.772448Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(23) [INFO] [stdout] 2025-09-01T11:56:12.773026Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:12.773172Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:13.068101Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:13.102430Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:13.110728Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:56:13.110779Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29] peer=29 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:56:13.111466Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:13.111658Z INFO Node{id=28}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:13.111725Z INFO Node{id=28}:connect{target=29}:WritePeer{remote=29}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:13.111841Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:13.254941Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:13.255463Z INFO Node{id=31}:connect{target=28}:ReadPeer{remote=28}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:13.255815Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:13.368534Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:13.368582Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 11 peers to 10 peers [INFO] [stdout] 2025-09-01T11:56:13.368985Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 11) [INFO] [stdout] 2025-09-01T11:56:13.369927Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 11) [INFO] [stdout] 2025-09-01T11:56:13.502161Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:13.505869Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:13.506130Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16]) [INFO] [stdout] 2025-09-01T11:56:13.520478Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:13.521632Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:13.522940Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:13.522962Z ERROR Node{id=7}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:13.671396Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:13.672032Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:56:13.674127Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:13.675232Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:13.675465Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 4]) [INFO] [stdout] 2025-09-01T11:56:13.675879Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:13.772948Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:13.772995Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:56:13.773412Z INFO Node{id=19}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:13.773506Z INFO Node{id=19}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:13.773670Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:13.776177Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727773773) } [INFO] [stdout] 2025-09-01T11:56:13.776483Z INFO Node{id=19}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:13.776508Z INFO Node{id=19}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:13.777574Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:13.778052Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:56:13.778072Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 13, 24] peer=24 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:56:13.779040Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:13.779096Z INFO Node{id=19}:connect{target=24}:WritePeer{remote=24}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:13.779173Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:13.779426Z INFO Node{id=19}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:13.779601Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:14.070379Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:14.071953Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:14.072841Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([15]) [INFO] [stdout] 2025-09-01T11:56:14.113026Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:14.113072Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:56:14.113268Z INFO Node{id=28}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:14.113358Z INFO Node{id=28}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:14.113493Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:14.116360Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727774113) } [INFO] [stdout] 2025-09-01T11:56:14.116384Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=28 [INFO] [stdout] 2025-09-01T11:56:14.116532Z INFO Node{id=28}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:14.116556Z INFO Node{id=28}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:14.118009Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:14.118364Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:56:14.118385Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[10] peer=10 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:56:14.118501Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:14.118557Z INFO Node{id=28}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:14.118634Z INFO Node{id=28}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:14.118853Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:14.119216Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:14.120134Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:14.120170Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:14.120368Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:14.120388Z ERROR Node{id=10}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:14.257372Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:14.257421Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=29 leader=1 [INFO] [stdout] 2025-09-01T11:56:14.257965Z INFO Node{id=31}:connect{target=29}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:14.258050Z INFO Node{id=31}:connect{target=29}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:14.258232Z INFO Node{id=29}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:56:14.527885Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:14.529364Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:14.529730Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5]) [INFO] [stdout] 2025-09-01T11:56:14.585633Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:14.624644Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:14.625716Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:14.626258Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:56:14.729375Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:14.729420Z INFO Node{id=10}: sharedstate::worker::sync_manager: Broadcasted 14 peers to 13 peers [INFO] [stdout] 2025-09-01T11:56:14.749577Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:14.751503Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:14.751816Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:14.751843Z ERROR Node{id=15}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:14.780190Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:14.780238Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:56:14.780782Z INFO Node{id=19}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:14.780872Z INFO Node{id=19}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:14.781018Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:14.783423Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727774780) } [INFO] [stdout] 2025-09-01T11:56:14.783585Z INFO Node{id=19}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:14.783610Z INFO Node{id=19}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:14.783891Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:14.784412Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:56:14.784434Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9] peer=9 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:56:14.784537Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:14.784580Z INFO Node{id=19}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:14.784688Z INFO Node{id=19}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:14.785079Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:14.785134Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:14.985022Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:14.985427Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20]) [INFO] [stdout] 2025-09-01T11:56:15.119725Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:15.119789Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=11 leader=1 [INFO] [stdout] 2025-09-01T11:56:15.120068Z INFO Node{id=28}:connect{target=11}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:15.120154Z INFO Node{id=28}:connect{target=11}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:15.120388Z INFO Node{id=11}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:15.122750Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727775120) } [INFO] [stdout] 2025-09-01T11:56:15.123048Z INFO Node{id=28}:connect{target=11}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:15.123075Z INFO Node{id=28}:connect{target=11}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:15.124457Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:15.124921Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 11 } [INFO] [stdout] 2025-09-01T11:56:15.124954Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 5, 21, 11] peer=11 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:56:15.125100Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:15.126075Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:15.126635Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:15.126963Z INFO Node{id=28}:connect{target=11}:ReadPeer{remote=11}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:15.127037Z INFO Node{id=28}:connect{target=11}:WritePeer{remote=11}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:15.127204Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:15.215573Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:15.215899Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:15.218432Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:15.218960Z INFO Node{id=29}: sequenced_broadcast: 90|Subscriber(4): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 4770 [INFO] [stdout] 2025-09-01T11:56:15.220201Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:15.220260Z INFO Node{id=6}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:15.312176Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:15.402522Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:15.403621Z INFO Node{id=29}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:15.426262Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:15.524111Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:15.525376Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:15.525533Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:15.525555Z ERROR Node{id=7}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:15.631054Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727774258) } [INFO] [stdout] 2025-09-01T11:56:15.631222Z INFO Node{id=31}:connect{target=29}: sharedstate::worker::sync_manager: got pong from peer latency=1373 [INFO] [stdout] 2025-09-01T11:56:15.631251Z INFO Node{id=31}:connect{target=29}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:15.785153Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:15.785201Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:56:15.785497Z INFO Node{id=19}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:15.785590Z INFO Node{id=19}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:15.785703Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:15.786960Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727775785) } [INFO] [stdout] 2025-09-01T11:56:15.787097Z INFO Node{id=19}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:56:15.787124Z INFO Node{id=19}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:15.787465Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:15.787918Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:56:15.787937Z ERROR Node{id=19}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 12] peer=12 leader=1 local=19 [INFO] [stdout] 2025-09-01T11:56:15.788021Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:15.788058Z INFO Node{id=19}:connect{target=12}:WritePeer{remote=12}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:15.788116Z INFO Node{id=19}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:15.788326Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:15.788432Z ERROR ReadClient{remote=19}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:16.036387Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:16.036685Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 28, 13]) [INFO] [stdout] 2025-09-01T11:56:16.073844Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:16.074277Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:16.074502Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([15]) [INFO] [stdout] 2025-09-01T11:56:16.120977Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:16.121488Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:16.121684Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:16.121705Z ERROR Node{id=10}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:16.127049Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:16.127084Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:56:16.127201Z INFO Node{id=28}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:16.127285Z INFO Node{id=28}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:16.127470Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:16.129415Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727776127) } [INFO] [stdout] 2025-09-01T11:56:16.129543Z INFO Node{id=28}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:56:16.129570Z INFO Node{id=28}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:16.129803Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:16.130056Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:56:16.130435Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:16.130560Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:16.130689Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:16.130938Z INFO Node{id=3}: sequenced_broadcast: 121|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:16.131228Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:16.131283Z INFO Node{id=28}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:16.405402Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:16.405533Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:56:16.406162Z INFO Node{id=29}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:16.406238Z INFO Node{id=29}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:16.406310Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:56:16.414014Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:16.415576Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727776406) } [INFO] [stdout] 2025-09-01T11:56:16.416016Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 29 } [INFO] [stdout] 2025-09-01T11:56:16.416038Z ERROR Node{id=31}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=29 leader=1 local=31 [INFO] [stdout] 2025-09-01T11:56:16.416915Z INFO Node{id=29}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=10 [INFO] [stdout] 2025-09-01T11:56:16.416941Z INFO Node{id=29}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:16.417484Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:16.417533Z INFO Node{id=31}:connect{target=29}:WritePeer{remote=29}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:16.417775Z INFO Node{id=31}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:16.418510Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:16.420543Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:16.426507Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:56:16.429855Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:16.430321Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:16.432771Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:16.433407Z INFO Node{id=3}: sequenced_broadcast: 121|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:16.437060Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:16.437127Z INFO Node{id=29}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:16.526534Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:16.527515Z INFO Node{id=15}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:16.528729Z ERROR ReadClient{remote=15}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:16.610323Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:16.617884Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:16.619926Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9]) [INFO] [stdout] 2025-09-01T11:56:16.660510Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:16.667251Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:16.670120Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([26]) [INFO] [stdout] 2025-09-01T11:56:16.789630Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:16.789675Z INFO Node{id=19}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:56:16.790690Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:16.790783Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:16.791839Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=19 [INFO] [stdout] 2025-09-01T11:56:16.801062Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: Ping(1756727776790) } [INFO] [stdout] 2025-09-01T11:56:16.803973Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:56:16.804006Z INFO Node{id=19}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:16.808621Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:16.812578Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:56:16.819193Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:16.820837Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 876, state_accept_seq: 875, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:16.821949Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:16.822595Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, seq(876) < min_allowed(33023) [INFO] [stdout] 2025-09-01T11:56:16.824472Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:16.826359Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber(7): Added, allow_drop: true, next_sequence: 35071, min_allowed_seq: 33023 [INFO] [stdout] 2025-09-01T11:56:16.830524Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 35071) [INFO] [stdout] 2025-09-01T11:56:16.830627Z INFO Node{id=19}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:16.831686Z INFO Node{id=19}: sequenced_broadcast: 133|SequencedBroadcastWorker Started id=133 [INFO] [stdout] 2025-09-01T11:56:16.835597Z INFO Node{id=19}: sequenced_broadcast: 99|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:16.835618Z INFO Node{id=19}: sequenced_broadcast: 99|SequencedBroadcastWorker Stopped id=99 elapsed=51.331914189s iter=4 [INFO] [stdout] 2025-09-01T11:56:17.134895Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:17.135921Z INFO Node{id=4}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:17.137047Z ERROR ReadClient{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:17.260409Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:17.264806Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:17.267517Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5, 21]) [INFO] [stdout] 2025-09-01T11:56:17.419633Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:17.419681Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:56:17.420428Z INFO Node{id=31}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:17.420592Z INFO Node{id=31}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:17.421512Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:56:17.432510Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727777420) } [INFO] [stdout] 2025-09-01T11:56:17.432549Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=31 [INFO] [stdout] 2025-09-01T11:56:17.433787Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:17.433823Z INFO Node{id=21}: sharedstate::worker::sync_manager: Broadcasted 10 peers to 9 peers [INFO] [stdout] 2025-09-01T11:56:17.435424Z INFO Node{id=31}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:56:17.435454Z INFO Node{id=31}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:17.436498Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 10) [INFO] [stdout] 2025-09-01T11:56:17.436545Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 10) [INFO] [stdout] 2025-09-01T11:56:17.437476Z ERROR Node{id=31}:connect{target=21}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:17.437903Z INFO Node{id=31}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:17.438543Z INFO Node{id=31}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:17.438677Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(21) [INFO] [stdout] 2025-09-01T11:56:17.438825Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:17.440230Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:17.527200Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:17.532344Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:17.532389Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:56:17.533284Z INFO Node{id=15}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:17.533364Z INFO Node{id=15}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:17.534444Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:56:17.535190Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:17.538200Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:17.538223Z ERROR Node{id=7}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:17.540653Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727777533) } [INFO] [stdout] 2025-09-01T11:56:17.542994Z INFO Node{id=15}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:56:17.543025Z INFO Node{id=15}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:17.548482Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:17.551934Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 5 } [INFO] [stdout] 2025-09-01T11:56:17.557849Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:17.559025Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:17.560593Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:17.560868Z INFO Node{id=5}: sequenced_broadcast: 105|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:17.564885Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:17.564952Z INFO Node{id=15}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:17.864630Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:17.869217Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:17.872005Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 28, 13, 24]) [INFO] [stdout] 2025-09-01T11:56:18.057519Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:18.057570Z INFO Node{id=1}: sharedstate::worker::sync_manager: Broadcasted 31 peers to 31 peers [INFO] [stdout] 2025-09-01T11:56:18.076051Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:18.079338Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:18.082012Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5, 15]) [INFO] [stdout] 2025-09-01T11:56:18.123009Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:18.126823Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:18.129319Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:18.129344Z ERROR Node{id=10}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:18.142331Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:18.142375Z INFO Node{id=4}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:56:18.143287Z INFO Node{id=4}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:18.143364Z INFO Node{id=4}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:18.144162Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=4 [INFO] [stdout] 2025-09-01T11:56:18.153356Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: Ping(1756727778143) } [INFO] [stdout] 2025-09-01T11:56:18.155570Z INFO Node{id=4}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=12 [INFO] [stdout] 2025-09-01T11:56:18.155598Z INFO Node{id=4}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:18.161337Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:18.164705Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:56:18.168375Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:18.170385Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:18.171312Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:18.171356Z INFO Node{id=32}: sequenced_broadcast: 130|Subscriber rejected, seq(6818) < min_allowed(33685) [INFO] [stdout] 2025-09-01T11:56:18.174238Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:18.174502Z INFO Node{id=32}: sequenced_broadcast: 130|Subscriber rejected, max_seq(35734) < seq(35736) [INFO] [stdout] 2025-09-01T11:56:18.188556Z INFO Node{id=32}: sequenced_broadcast: 130|Subscriber(1): Added, allow_drop: true, next_sequence: 35736, min_allowed_seq: 33688 [INFO] [stdout] 2025-09-01T11:56:18.192853Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 35736) [INFO] [stdout] 2025-09-01T11:56:18.192958Z INFO Node{id=4}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:18.193939Z INFO Node{id=4}: sequenced_broadcast: 134|SequencedBroadcastWorker Started id=134 [INFO] [stdout] 2025-09-01T11:56:18.195188Z INFO Node{id=4}: sequenced_broadcast: 79|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:18.195207Z INFO Node{id=4}: sequenced_broadcast: 79|SequencedBroadcastWorker Stopped id=79 elapsed=84.87910235s iter=2914 [INFO] [stdout] 2025-09-01T11:56:18.198174Z ERROR Node{id=14}:connect{target=4}:ReadPeer{remote=4}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:18.198969Z WARN Node{id=14}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:56:18.198986Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:18.439755Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:18.439821Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:56:18.440820Z INFO Node{id=31}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:18.440895Z INFO Node{id=31}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:18.441902Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:56:18.448468Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727778440) } [INFO] [stdout] 2025-09-01T11:56:18.451067Z INFO Node{id=31}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=11 [INFO] [stdout] 2025-09-01T11:56:18.451105Z INFO Node{id=31}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:18.455468Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:18.458193Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:18.459358Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:56:18.463579Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:18.463949Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:18.465105Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:18.466580Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25]) [INFO] [stdout] 2025-09-01T11:56:18.466617Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:18.467744Z INFO Node{id=27}: sequenced_broadcast: 127|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:18.472886Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:18.472957Z INFO Node{id=31}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:18.601600Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:18.672201Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:18.677044Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:18.679805Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5, 15, 26]) [INFO] [stdout] 2025-09-01T11:56:19.103729Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:19.126860Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:19.131832Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18]) [INFO] [stdout] 2025-09-01T11:56:19.204515Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:19.204565Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:56:19.205492Z INFO Node{id=14}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:19.205574Z INFO Node{id=14}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:19.206516Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:56:19.212718Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727779205) } [INFO] [stdout] 2025-09-01T11:56:19.212772Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=14 [INFO] [stdout] 2025-09-01T11:56:19.217452Z INFO Node{id=14}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=12 [INFO] [stdout] 2025-09-01T11:56:19.217501Z INFO Node{id=14}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:19.221739Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:19.225428Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:56:19.225456Z ERROR Node{id=14}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[10] peer=10 leader=1 local=14 [INFO] [stdout] 2025-09-01T11:56:19.226701Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:19.226778Z INFO Node{id=14}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:19.228235Z INFO Node{id=14}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:19.229187Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:19.229260Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:19.275005Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:19.275877Z INFO Node{id=13}:connect{target=28}:ReadPeer{remote=28}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:19.276974Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:19.539355Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:19.543485Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:19.545180Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 28, 13, 24, 27, 31]) [INFO] [stdout] 2025-09-01T11:56:19.636430Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:19.640529Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:19.642871Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5, 21]) [INFO] [stdout] 2025-09-01T11:56:19.730906Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:19.731066Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:19.731095Z INFO Node{id=10}: sharedstate::worker::sync_manager: Broadcasted 15 peers to 14 peers [INFO] [stdout] 2025-09-01T11:56:19.734395Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:20.119716Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:20.124688Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:20.129193Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16]) [INFO] [stdout] 2025-09-01T11:56:20.131149Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:20.136187Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:20.138724Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 28, 13, 24, 27, 31]) [INFO] [stdout] 2025-09-01T11:56:20.227882Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:20.227938Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:56:20.228907Z INFO Node{id=14}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:20.228974Z INFO Node{id=14}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:20.229855Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:56:20.235910Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727780228) } [INFO] [stdout] 2025-09-01T11:56:20.235940Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=14 [INFO] [stdout] 2025-09-01T11:56:20.238348Z INFO Node{id=14}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=10 [INFO] [stdout] 2025-09-01T11:56:20.238376Z INFO Node{id=14}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:20.238489Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:20.238518Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 12 peers to 11 peers [INFO] [stdout] 2025-09-01T11:56:20.240508Z ERROR Node{id=14}:connect{target=20}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:20.240849Z INFO Node{id=14}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:20.240967Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 12) [INFO] [stdout] 2025-09-01T11:56:20.241073Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 12) [INFO] [stdout] 2025-09-01T11:56:20.241524Z INFO Node{id=14}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:20.241592Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(20) [INFO] [stdout] 2025-09-01T11:56:20.242420Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:20.243449Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:20.279439Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:20.279570Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:56:20.280463Z INFO Node{id=13}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:20.280532Z INFO Node{id=13}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:20.281394Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:20.285970Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:20.287195Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727780280) } [INFO] [stdout] 2025-09-01T11:56:20.289816Z INFO Node{id=13}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:56:20.289847Z INFO Node{id=13}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:20.294134Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:20.297650Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 5 } [INFO] [stdout] 2025-09-01T11:56:20.297676Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 5] peer=5 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:20.298579Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:20.298637Z INFO Node{id=13}:connect{target=5}:WritePeer{remote=5}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:20.299450Z INFO Node{id=13}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:20.299691Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:20.301196Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:20.503343Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:20.505549Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23]) [INFO] [stdout] 2025-09-01T11:56:20.699636Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:20.703661Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:20.706280Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20]) [INFO] [stdout] 2025-09-01T11:56:21.242209Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:21.242253Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:56:21.243055Z INFO Node{id=14}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:21.243202Z INFO Node{id=14}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:21.243961Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:56:21.249749Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727781243) } [INFO] [stdout] 2025-09-01T11:56:21.249792Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=14 [INFO] [stdout] 2025-09-01T11:56:21.252179Z INFO Node{id=14}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=9 [INFO] [stdout] 2025-09-01T11:56:21.252207Z INFO Node{id=14}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:21.258560Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:21.261559Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:56:21.265094Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:21.266223Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:21.268015Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:21.268322Z INFO Node{id=21}: sequenced_broadcast: 107|Subscriber(3): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:21.272463Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:21.272528Z INFO Node{id=14}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:21.303886Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:21.303929Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:56:21.304825Z INFO Node{id=13}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:21.304896Z INFO Node{id=13}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:21.305628Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:21.319457Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727781304) } [INFO] [stdout] 2025-09-01T11:56:21.319492Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=13 [INFO] [stdout] 2025-09-01T11:56:21.322949Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:21.322989Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 14 peers to 13 peers [INFO] [stdout] 2025-09-01T11:56:21.324212Z INFO Node{id=13}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:56:21.324240Z INFO Node{id=13}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:21.325783Z ERROR Node{id=13}:connect{target=23}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:21.325957Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 14) [INFO] [stdout] 2025-09-01T11:56:21.326081Z INFO Node{id=13}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:21.326685Z INFO Node{id=13}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:21.326790Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(23) [INFO] [stdout] 2025-09-01T11:56:21.327049Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:21.340521Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:21.548880Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:21.551114Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3, 29]) [INFO] [stdout] 2025-09-01T11:56:21.714196Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:21.762975Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:21.766133Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17]) [INFO] [stdout] 2025-09-01T11:56:22.328983Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:22.329031Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:56:22.329928Z INFO Node{id=13}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:22.329992Z INFO Node{id=13}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:22.330891Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:22.331070Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:22.338499Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3, 29]) [INFO] [stdout] 2025-09-01T11:56:22.338697Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727782329) } [INFO] [stdout] 2025-09-01T11:56:22.342504Z INFO Node{id=13}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:56:22.342546Z INFO Node{id=13}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:22.350269Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:22.355815Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:56:22.355847Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16] peer=16 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:22.357055Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:22.357108Z INFO Node{id=13}:connect{target=16}:WritePeer{remote=16}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:22.357850Z INFO Node{id=13}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:22.358240Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:22.364239Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:22.435740Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:22.435806Z INFO Node{id=21}: sharedstate::worker::sync_manager: Broadcasted 11 peers to 10 peers [INFO] [stdout] 2025-09-01T11:56:22.438328Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 11) [INFO] [stdout] 2025-09-01T11:56:22.438375Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 11) [INFO] [stdout] 2025-09-01T11:56:22.438416Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 11) [INFO] [stdout] 2025-09-01T11:56:23.358578Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:23.358626Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:56:23.359492Z INFO Node{id=13}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:23.359559Z INFO Node{id=13}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:23.360403Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:23.366606Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727783359) } [INFO] [stdout] 2025-09-01T11:56:23.369416Z INFO Node{id=13}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=10 [INFO] [stdout] 2025-09-01T11:56:23.369447Z INFO Node{id=13}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:23.374106Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:23.377561Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:56:23.377587Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 5, 15, 26] peer=26 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:23.378778Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:23.378833Z INFO Node{id=13}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:23.379433Z INFO Node{id=13}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:23.379831Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:23.381281Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:23.509471Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:23.546287Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:23.550680Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16]) [INFO] [stdout] 2025-09-01T11:56:23.675851Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:24.055838Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:24.058238Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3, 29]) [INFO] [stdout] 2025-09-01T11:56:24.058263Z WARN Node{id=6}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:56:24.059852Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:24.060419Z INFO Node{id=6}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:24.061480Z ERROR ReadClient{remote=6}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:24.068693Z INFO sharedstate::testing::fuzzy_test: node0 sequence updated: 0 => 39086 [INFO] [stdout] 2025-09-01T11:56:24.068710Z INFO sharedstate::testing::fuzzy_test: node1 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068715Z INFO sharedstate::testing::fuzzy_test: node2 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068719Z INFO sharedstate::testing::fuzzy_test: node3 sequence updated: 0 => 39067 [INFO] [stdout] 2025-09-01T11:56:24.068723Z INFO sharedstate::testing::fuzzy_test: node4 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068727Z INFO sharedstate::testing::fuzzy_test: node5 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068731Z INFO sharedstate::testing::fuzzy_test: node6 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068735Z INFO sharedstate::testing::fuzzy_test: node7 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068739Z INFO sharedstate::testing::fuzzy_test: node8 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068743Z INFO sharedstate::testing::fuzzy_test: node9 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068748Z INFO sharedstate::testing::fuzzy_test: node10 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068752Z INFO sharedstate::testing::fuzzy_test: node11 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068774Z INFO sharedstate::testing::fuzzy_test: node12 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068778Z INFO sharedstate::testing::fuzzy_test: node13 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068782Z INFO sharedstate::testing::fuzzy_test: node14 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068786Z INFO sharedstate::testing::fuzzy_test: node15 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068789Z INFO sharedstate::testing::fuzzy_test: node16 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068793Z INFO sharedstate::testing::fuzzy_test: node17 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068796Z INFO sharedstate::testing::fuzzy_test: node18 sequence updated: 0 => 39079 [INFO] [stdout] 2025-09-01T11:56:24.068799Z INFO sharedstate::testing::fuzzy_test: node19 sequence updated: 0 => 39080 [INFO] [stdout] 2025-09-01T11:56:24.068803Z INFO sharedstate::testing::fuzzy_test: node20 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068806Z INFO sharedstate::testing::fuzzy_test: node21 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068810Z INFO sharedstate::testing::fuzzy_test: node22 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068813Z INFO sharedstate::testing::fuzzy_test: node23 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068824Z INFO sharedstate::testing::fuzzy_test: node24 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068828Z INFO sharedstate::testing::fuzzy_test: node25 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068832Z INFO sharedstate::testing::fuzzy_test: node26 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068836Z INFO sharedstate::testing::fuzzy_test: node27 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068840Z INFO sharedstate::testing::fuzzy_test: node28 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068843Z INFO sharedstate::testing::fuzzy_test: node29 sequence updated: 0 => 39073 [INFO] [stdout] 2025-09-01T11:56:24.068846Z INFO sharedstate::testing::fuzzy_test: node30 sequence updated: 0 => 6817 [INFO] [stdout] 2025-09-01T11:56:24.068850Z INFO sharedstate::testing::fuzzy_test: node31 sequence updated: 0 => 39074 [INFO] [stdout] 2025-09-01T11:56:24.380527Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:24.380575Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:56:24.381333Z INFO Node{id=13}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:24.381406Z INFO Node{id=13}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:24.381572Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:24.383811Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727784381) } [INFO] [stdout] 2025-09-01T11:56:24.384745Z INFO Node{id=13}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:24.384791Z INFO Node{id=13}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:24.386364Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:24.386530Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:56:24.386553Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 5, 21, 2] peer=2 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:24.386714Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:24.386784Z INFO Node{id=13}:connect{target=2}:WritePeer{remote=2}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:24.386860Z INFO Node{id=13}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:24.388409Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:24.389184Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:24.529944Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:24.531499Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:24.531625Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5]) [INFO] [stdout] 2025-09-01T11:56:24.626914Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:24.628501Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:24.629013Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:56:24.986095Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:24.987671Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:24.987826Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20]) [INFO] [stdout] 2025-09-01T11:56:25.064092Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:25.064138Z INFO Node{id=6}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:56:25.064996Z INFO Node{id=6}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:25.065069Z INFO Node{id=6}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:25.065127Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=6 [INFO] [stdout] 2025-09-01T11:56:25.394096Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:25.394150Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:56:25.394983Z INFO Node{id=13}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:25.395052Z INFO Node{id=13}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:25.395124Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:25.400779Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727785395) } [INFO] [stdout] 2025-09-01T11:56:25.400806Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=13 [INFO] [stdout] 2025-09-01T11:56:25.400917Z INFO Node{id=13}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=5 [INFO] [stdout] 2025-09-01T11:56:25.400940Z INFO Node{id=13}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:25.401117Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:25.407254Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:56:25.407287Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 13, 24, 27, 31, 10] peer=10 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:25.408039Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:25.408076Z INFO Node{id=10}: sharedstate::worker::sync_manager: Broadcasted 16 peers to 15 peers [INFO] [stdout] 2025-09-01T11:56:25.408352Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:25.409053Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 16) [INFO] [stdout] 2025-09-01T11:56:25.409199Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:25.410004Z ERROR Node{id=13}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: failed to send message to output, stopping read [INFO] [stdout] 2025-09-01T11:56:25.410085Z INFO Node{id=13}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:25.410186Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:25.430067Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: Ping(1756727785065) } [INFO] [stdout] 2025-09-01T11:56:25.430609Z INFO Node{id=6}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=365 [INFO] [stdout] 2025-09-01T11:56:25.430640Z INFO Node{id=6}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:25.767477Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:25.768043Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:56:25.989278Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:25.989838Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:25.990704Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:25.991157Z INFO Node{id=9}: sequenced_broadcast: 117|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:25.991793Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:25.991851Z INFO Node{id=6}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:26.038531Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:26.040266Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:26.040446Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:26.040468Z ERROR Node{id=24}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:26.131028Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:26.410106Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:26.410151Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:56:26.410878Z INFO Node{id=13}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:26.410953Z INFO Node{id=13}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:26.411049Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:26.415364Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727786410) } [INFO] [stdout] 2025-09-01T11:56:26.415393Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=13 [INFO] [stdout] 2025-09-01T11:56:26.415537Z INFO Node{id=13}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=5 [INFO] [stdout] 2025-09-01T11:56:26.415565Z INFO Node{id=13}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:26.416911Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:26.417782Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:56:26.417803Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 5, 21] peer=21 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:26.419049Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:26.419102Z INFO Node{id=13}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:26.419180Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:26.419260Z INFO Node{id=13}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:26.419372Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:26.429102Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:26.621545Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:26.812894Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:26.814374Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:26.817265Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:56:27.269128Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:27.270526Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:27.270647Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5, 21]) [INFO] [stdout] 2025-09-01T11:56:27.420388Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:27.420432Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:56:27.421092Z INFO Node{id=13}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:27.421171Z INFO Node{id=13}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:27.421239Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:27.423725Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727787421) } [INFO] [stdout] 2025-09-01T11:56:27.423845Z INFO Node{id=13}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:56:27.423867Z INFO Node{id=13}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:27.424030Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:27.424855Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:56:27.424877Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 28, 13, 24, 27, 31, 7] peer=7 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:27.424996Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:27.425047Z INFO Node{id=13}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:27.425123Z INFO Node{id=13}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:27.425889Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:27.426590Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:27.437219Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:27.437274Z INFO Node{id=21}: sharedstate::worker::sync_manager: Broadcasted 12 peers to 11 peers [INFO] [stdout] 2025-09-01T11:56:27.438101Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 12) [INFO] [stdout] 2025-09-01T11:56:27.438233Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 12) [INFO] [stdout] 2025-09-01T11:56:27.438267Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 12) [INFO] [stdout] 2025-09-01T11:56:27.554227Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:27.555553Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:27.555697Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5]) [INFO] [stdout] 2025-09-01T11:56:27.653657Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:27.654161Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9]) [INFO] [stdout] 2025-09-01T11:56:27.738074Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:27.738289Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9, 3]) [INFO] [stdout] 2025-09-01T11:56:27.738324Z WARN Node{id=28}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:56:27.738379Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:27.738424Z INFO Node{id=28}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:27.739358Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:27.874391Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:27.875902Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:27.876721Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([24]) [INFO] [stdout] 2025-09-01T11:56:28.040861Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:28.042426Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:28.042584Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:28.042605Z ERROR Node{id=24}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:28.083277Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:28.086788Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:28.086939Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5, 15]) [INFO] [stdout] 2025-09-01T11:56:28.165115Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:28.166680Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:28.166815Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20, 32]) [INFO] [stdout] 2025-09-01T11:56:28.426373Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:28.426420Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:56:28.429237Z INFO Node{id=13}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:28.429356Z INFO Node{id=13}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:28.429437Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:28.431982Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727788429) } [INFO] [stdout] 2025-09-01T11:56:28.432100Z INFO Node{id=13}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:28.432136Z INFO Node{id=13}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:28.432434Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:28.433148Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:56:28.433170Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 5, 21, 14] peer=14 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:28.433288Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:28.433344Z INFO Node{id=13}:connect{target=14}:WritePeer{remote=14}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:28.433422Z INFO Node{id=13}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:28.435014Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:28.435168Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:28.460435Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:28.463850Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:28.463951Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([27]) [INFO] [stdout] 2025-09-01T11:56:28.470578Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:28.478217Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:28.478885Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25]) [INFO] [stdout] 2025-09-01T11:56:28.681016Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:28.682616Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:28.682745Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5, 15, 26]) [INFO] [stdout] 2025-09-01T11:56:28.740932Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:28.740980Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=15 leader=1 [INFO] [stdout] 2025-09-01T11:56:28.741785Z INFO Node{id=28}:connect{target=15}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:28.741954Z INFO Node{id=28}:connect{target=15}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:28.742013Z INFO Node{id=15}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:28.746319Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727788741) } [INFO] [stdout] 2025-09-01T11:56:28.746425Z INFO Node{id=28}:connect{target=15}: sharedstate::worker::sync_manager: got pong from peer latency=5 [INFO] [stdout] 2025-09-01T11:56:28.746451Z INFO Node{id=28}:connect{target=15}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:28.746634Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:28.747344Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 15 } [INFO] [stdout] 2025-09-01T11:56:28.747432Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 5, 15] peer=15 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:56:28.747536Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:28.747576Z INFO Node{id=28}:connect{target=15}:WritePeer{remote=15}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:28.747626Z INFO Node{id=28}:connect{target=15}:ReadPeer{remote=15}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:28.748266Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:28.749068Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:28.919424Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:28.920037Z INFO Node{id=5}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:28.920241Z ERROR ReadClient{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:29.133375Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:29.136591Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:29.137093Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3, 29, 18]) [INFO] [stdout] 2025-09-01T11:56:29.434041Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:29.434086Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:56:29.434686Z INFO Node{id=13}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:29.434796Z INFO Node{id=13}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:29.434863Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:29.436782Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727789434) } [INFO] [stdout] 2025-09-01T11:56:29.436804Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=13 [INFO] [stdout] 2025-09-01T11:56:29.436886Z INFO Node{id=13}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:56:29.436905Z INFO Node{id=13}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:29.437045Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:29.437075Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 13 peers to 12 peers [INFO] [stdout] 2025-09-01T11:56:29.437692Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 13) [INFO] [stdout] 2025-09-01T11:56:29.437727Z ERROR Node{id=13}:connect{target=20}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:29.437803Z INFO Node{id=13}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:29.438657Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:29.438700Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 13) [INFO] [stdout] 2025-09-01T11:56:29.438735Z INFO Node{id=13}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:29.438826Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(20) [INFO] [stdout] 2025-09-01T11:56:29.438956Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:29.546200Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:29.549640Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:29.549802Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([31]) [INFO] [stdout] 2025-09-01T11:56:29.643924Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:29.645259Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:29.645371Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 5, 21]) [INFO] [stdout] 2025-09-01T11:56:29.673266Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:29.673945Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9, 3]) [INFO] [stdout] 2025-09-01T11:56:29.673969Z WARN Node{id=29}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:56:29.675225Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:29.675311Z INFO Node{id=29}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:29.698521Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:29.748915Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:29.748960Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:56:29.749620Z INFO Node{id=28}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:29.749717Z INFO Node{id=28}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:29.749788Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:29.751880Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727789749) } [INFO] [stdout] 2025-09-01T11:56:29.751969Z INFO Node{id=28}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:56:29.751990Z INFO Node{id=28}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:29.752153Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:29.752851Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:56:29.752873Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9, 6] peer=6 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:56:29.752985Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:29.753029Z INFO Node{id=28}:connect{target=6}:WritePeer{remote=6}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:29.753086Z INFO Node{id=28}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:29.753733Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:29.753802Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:29.877143Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:29.879437Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:29.879571Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([24]) [INFO] [stdout] 2025-09-01T11:56:29.921048Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:29.921096Z INFO Node{id=5}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=32 leader=1 [INFO] [stdout] 2025-09-01T11:56:29.921720Z INFO Node{id=5}:connect{target=32}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:29.921838Z INFO Node{id=5}:connect{target=32}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:29.921946Z INFO Node{id=32}: sharedstate::worker::sync_manager: got new client remote=5 [INFO] [stdout] 2025-09-01T11:56:29.923964Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: Ping(1756727789921) } [INFO] [stdout] 2025-09-01T11:56:29.924097Z INFO Node{id=5}:connect{target=32}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:29.924123Z INFO Node{id=5}:connect{target=32}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:29.924311Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:29.925074Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 32 } [INFO] [stdout] 2025-09-01T11:56:29.925638Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:29.925794Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:29.925943Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:29.926696Z INFO Node{id=32}: sequenced_broadcast: 130|Subscriber rejected, seq(6818) < min_allowed(37120) [INFO] [stdout] 2025-09-01T11:56:29.929508Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:29.930929Z INFO Node{id=32}: sequenced_broadcast: 130|Subscriber(2): Added, allow_drop: true, next_sequence: 39168, min_allowed_seq: 37120 [INFO] [stdout] 2025-09-01T11:56:29.931157Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 39168) [INFO] [stdout] 2025-09-01T11:56:29.931238Z INFO Node{id=5}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:29.931262Z INFO Node{id=5}: sequenced_broadcast: 135|SequencedBroadcastWorker Started id=135 [INFO] [stdout] 2025-09-01T11:56:29.932647Z INFO Node{id=5}: sequenced_broadcast: 105|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:29.932666Z INFO Node{id=5}: sequenced_broadcast: 105|SequencedBroadcastWorker Stopped id=105 elapsed=59.719780366s iter=4 [INFO] [stdout] 2025-09-01T11:56:29.933030Z ERROR Node{id=21}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:29.933104Z ERROR Node{id=15}:connect{target=5}:ReadPeer{remote=5}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:29.933161Z WARN Node{id=21}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:56:29.933218Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:29.933294Z WARN Node{id=15}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:56:29.933314Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:30.042910Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:30.044011Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:30.044129Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:30.044150Z ERROR Node{id=24}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:30.069890Z INFO sharedstate::testing::fuzzy_test: node0 sequence updated: 39086 => 39167 [INFO] [stdout] 2025-09-01T11:56:30.069911Z INFO sharedstate::testing::fuzzy_test: node3 sequence updated: 39067 => 39167 [INFO] [stdout] 2025-09-01T11:56:30.069916Z INFO sharedstate::testing::fuzzy_test: node4 sequence updated: 6817 => 39167 [INFO] [stdout] 2025-09-01T11:56:30.069927Z INFO sharedstate::testing::fuzzy_test: node18 sequence updated: 39079 => 39167 [INFO] [stdout] 2025-09-01T11:56:30.069931Z INFO sharedstate::testing::fuzzy_test: node19 sequence updated: 39080 => 39167 [INFO] [stdout] 2025-09-01T11:56:30.069938Z INFO sharedstate::testing::fuzzy_test: node29 sequence updated: 39073 => 39167 [INFO] [stdout] 2025-09-01T11:56:30.069942Z INFO sharedstate::testing::fuzzy_test: node31 sequence updated: 39074 => 39167 [INFO] [stdout] 2025-09-01T11:56:30.130232Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:30.134491Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:30.135635Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16]) [INFO] [stdout] 2025-09-01T11:56:30.139246Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:30.141253Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:30.141534Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([31]) [INFO] [stdout] 2025-09-01T11:56:30.440492Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:30.440553Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:56:30.441266Z INFO Node{id=13}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:30.441385Z INFO Node{id=13}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:30.441465Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:30.465899Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:30.468188Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:30.468371Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([27]) [INFO] [stdout] 2025-09-01T11:56:30.506007Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:30.677064Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:30.677108Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:56:30.677642Z INFO Node{id=29}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:30.677724Z INFO Node{id=29}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:30.677822Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:56:30.680046Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727790677) } [INFO] [stdout] 2025-09-01T11:56:30.680152Z INFO Node{id=29}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:30.680174Z INFO Node{id=29}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:30.680318Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:30.681677Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:30.682430Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:56:30.682451Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[7] peer=7 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:56:30.683492Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:30.683596Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:30.684164Z INFO Node{id=29}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:30.684232Z INFO Node{id=29}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:30.684354Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:30.706963Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:30.708405Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:30.708529Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20]) [INFO] [stdout] 2025-09-01T11:56:30.754183Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:30.754233Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:56:30.754903Z INFO Node{id=28}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:30.754985Z INFO Node{id=28}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:30.755055Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:30.757215Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727790754) } [INFO] [stdout] 2025-09-01T11:56:30.757328Z INFO Node{id=28}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:30.757355Z INFO Node{id=28}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:30.757548Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:30.758282Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:56:30.758304Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 5, 21, 2] peer=2 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:56:30.758417Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:30.758464Z INFO Node{id=28}:connect{target=2}:WritePeer{remote=2}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:30.758545Z INFO Node{id=28}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:30.759191Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:30.759248Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:30.935032Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:30.935081Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:56:30.935143Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:30.935163Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=11 peers=12 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:56:30.935877Z INFO Node{id=15}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:30.935976Z INFO Node{id=15}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:30.936057Z INFO Node{id=21}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:30.936115Z INFO Node{id=21}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:30.936169Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:56:30.936248Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:56:30.938615Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727790936) } [INFO] [stdout] 2025-09-01T11:56:30.938653Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727790935) } [INFO] [stdout] 2025-09-01T11:56:30.938665Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=15 [INFO] [stdout] 2025-09-01T11:56:30.938787Z INFO Node{id=21}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:56:30.938815Z INFO Node{id=21}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:30.938850Z INFO Node{id=15}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:30.938868Z INFO Node{id=15}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:30.940305Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:30.940341Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:30.940501Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 21 } [INFO] [stdout] 2025-09-01T11:56:30.940520Z ERROR Node{id=15}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=21 leader=1 local=15 [INFO] [stdout] 2025-09-01T11:56:30.940621Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:56:30.940640Z ERROR Node{id=21}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[31] peer=31 leader=1 local=21 [INFO] [stdout] 2025-09-01T11:56:30.940782Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:30.940826Z INFO Node{id=21}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:30.940884Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:30.940922Z INFO Node{id=15}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:30.940972Z INFO Node{id=21}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:30.941017Z INFO Node{id=15}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:30.941556Z ERROR ReadClient{remote=21}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:30.941615Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:30.941651Z ERROR ReadClient{remote=15}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:30.942367Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:31.186452Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:31.186870Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23]) [INFO] [stdout] 2025-09-01T11:56:31.261956Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:31.263040Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:31.263167Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:31.263187Z ERROR Node{id=14}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:31.551316Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:31.551951Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:31.552500Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:31.552681Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([31]) [INFO] [stdout] 2025-09-01T11:56:31.619552Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:31.620001Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:31.620021Z ERROR Node{id=22}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:31.684685Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:31.684726Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:56:31.684909Z INFO Node{id=29}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:31.685000Z INFO Node{id=29}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:31.685037Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:56:31.686611Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727791684) } [INFO] [stdout] 2025-09-01T11:56:31.686706Z INFO Node{id=29}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:56:31.686725Z INFO Node{id=29}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:31.687101Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:31.688141Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:31.692692Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:56:31.693073Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[24] peer=24 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:56:31.694374Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:31.694555Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:31.695620Z INFO Node{id=29}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:31.695699Z INFO Node{id=29}:connect{target=24}:WritePeer{remote=24}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:31.695871Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:31.759038Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:31.759085Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:56:31.759693Z INFO Node{id=28}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:31.759793Z INFO Node{id=28}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:31.759847Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:31.761954Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727791759) } [INFO] [stdout] 2025-09-01T11:56:31.762053Z INFO Node{id=28}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:31.762078Z INFO Node{id=28}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:31.762256Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:31.762998Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:56:31.763019Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[14] peer=14 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:56:31.763131Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:31.763184Z INFO Node{id=28}:connect{target=14}:WritePeer{remote=14}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:31.763258Z INFO Node{id=28}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:31.763900Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:31.763944Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:31.767578Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:31.784685Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:31.786421Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17]) [INFO] [stdout] 2025-09-01T11:56:31.787986Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727790441) } [INFO] [stdout] 2025-09-01T11:56:31.788282Z INFO Node{id=13}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=1347 [INFO] [stdout] 2025-09-01T11:56:31.788311Z INFO Node{id=13}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:31.788704Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:31.789169Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:56:31.789201Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9, 3] peer=3 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:31.789317Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:31.789363Z INFO Node{id=13}:connect{target=3}:WritePeer{remote=3}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:31.789438Z INFO Node{id=13}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:31.789895Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:31.790121Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:31.880025Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:31.881134Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:31.881237Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([24]) [INFO] [stdout] 2025-09-01T11:56:31.941956Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:31.942004Z INFO Node{id=15}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:56:31.942068Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:31.942086Z INFO Node{id=21}: sharedstate::worker::sync_manager: found connect target option_count=12 peers=13 target=19 leader=1 [INFO] [stdout] 2025-09-01T11:56:31.942694Z INFO Node{id=15}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:31.942803Z INFO Node{id=15}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:31.942853Z INFO Node{id=21}:connect{target=19}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:31.942903Z INFO Node{id=21}:connect{target=19}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:31.942946Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=15 [INFO] [stdout] 2025-09-01T11:56:31.943001Z INFO Node{id=19}: sharedstate::worker::sync_manager: got new client remote=21 [INFO] [stdout] 2025-09-01T11:56:31.944904Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: Ping(1756727791942) } [INFO] [stdout] 2025-09-01T11:56:31.944982Z INFO Node{id=21}:connect{target=19}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:56:31.945002Z INFO Node{id=21}:connect{target=19}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:31.945142Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:31.945691Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 19 } [INFO] [stdout] 2025-09-01T11:56:31.946315Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:31.946418Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:31.946528Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:31.946959Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: Ping(1756727791942) } [INFO] [stdout] 2025-09-01T11:56:31.947125Z INFO Node{id=15}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=5 [INFO] [stdout] 2025-09-01T11:56:31.947156Z INFO Node{id=15}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:31.947286Z INFO Node{id=19}: sequenced_broadcast: 133|Subscriber rejected, seq(6818) < min_allowed(37120) [INFO] [stdout] 2025-09-01T11:56:31.947861Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:31.948485Z INFO Node{id=19}: sequenced_broadcast: 133|Subscriber(1): Added, allow_drop: true, next_sequence: 39168, min_allowed_seq: 37120 [INFO] [stdout] 2025-09-01T11:56:31.949122Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 39168) [INFO] [stdout] 2025-09-01T11:56:31.949220Z INFO Node{id=21}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:31.949251Z INFO Node{id=21}: sequenced_broadcast: 136|SequencedBroadcastWorker Started id=136 [INFO] [stdout] 2025-09-01T11:56:31.949342Z INFO Node{id=21}: sequenced_broadcast: 107|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:31.949358Z INFO Node{id=21}: sequenced_broadcast: 107|SequencedBroadcastWorker Stopped id=107 elapsed=57.545461958s iter=5 [INFO] [stdout] 2025-09-01T11:56:31.950400Z ERROR Node{id=11}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:31.950471Z ERROR Node{id=2}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:31.950533Z ERROR Node{id=14}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:31.950600Z WARN Node{id=11}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:56:31.950612Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:31.950680Z WARN Node{id=2}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:56:31.950693Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:31.950730Z WARN Node{id=14}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:56:31.950742Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:31.951795Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:31.952713Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:56:31.958062Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:31.958214Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:31.958987Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:31.959138Z INFO Node{id=18}: sequenced_broadcast: 112|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:31.959570Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:31.959659Z INFO Node{id=15}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:32.044844Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:32.046102Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:32.046213Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:32.046245Z ERROR Node{id=24}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:32.144269Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:32.145452Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:32.145584Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([31]) [INFO] [stdout] 2025-09-01T11:56:32.340449Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:32.426234Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:32.426411Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:32.426431Z ERROR Node{id=18}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:32.437893Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:32.437940Z INFO Node{id=21}: sharedstate::worker::sync_manager: Broadcasted 13 peers to 12 peers [INFO] [stdout] 2025-09-01T11:56:32.470180Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:32.471179Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:32.471399Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([27]) [INFO] [stdout] 2025-09-01T11:56:32.695739Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:32.695805Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=2 leader=1 [INFO] [stdout] 2025-09-01T11:56:32.696012Z INFO Node{id=29}:connect{target=2}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:32.696098Z INFO Node{id=29}:connect{target=2}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:32.696157Z INFO Node{id=2}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:56:32.702463Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727792696) } [INFO] [stdout] 2025-09-01T11:56:32.702597Z INFO Node{id=29}:connect{target=2}: sharedstate::worker::sync_manager: got pong from peer latency=6 [INFO] [stdout] 2025-09-01T11:56:32.702624Z INFO Node{id=29}:connect{target=2}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:32.702830Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:32.703924Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:32.704511Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 2 } [INFO] [stdout] 2025-09-01T11:56:32.704532Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=2 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:56:32.704607Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:32.705086Z INFO Node{id=29}:connect{target=2}:ReadPeer{remote=2}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:32.705152Z INFO Node{id=29}:connect{target=2}:WritePeer{remote=2}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:32.705222Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:32.764078Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:32.764123Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:56:32.764809Z INFO Node{id=28}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:32.764876Z INFO Node{id=28}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:32.764928Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:32.777127Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727792764) } [INFO] [stdout] 2025-09-01T11:56:32.777276Z INFO Node{id=28}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:56:32.777307Z INFO Node{id=28}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:32.785716Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:32.786278Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:56:32.786301Z ERROR Node{id=28}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[18] peer=18 leader=1 local=28 [INFO] [stdout] 2025-09-01T11:56:32.786419Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:32.786461Z INFO Node{id=28}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:32.786530Z INFO Node{id=28}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:32.787103Z ERROR ReadClient{remote=28}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:32.789845Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:32.789876Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:56:32.790304Z INFO Node{id=13}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:32.790370Z INFO Node{id=13}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:32.790439Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:32.792126Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727792790) } [INFO] [stdout] 2025-09-01T11:56:32.792247Z INFO Node{id=13}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:56:32.792272Z INFO Node{id=13}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:32.792469Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:32.796140Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:56:32.796165Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9] peer=9 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:32.796307Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:32.796362Z INFO Node{id=13}:connect{target=9}:WritePeer{remote=9}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:32.796449Z INFO Node{id=13}:connect{target=9}:ReadPeer{remote=9}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:32.797081Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:32.797133Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:32.798972Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:32.954050Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:32.954096Z INFO Node{id=11}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=9 leader=1 [INFO] [stdout] 2025-09-01T11:56:32.954159Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:32.954180Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:56:32.954222Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:32.954243Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:56:32.954737Z INFO Node{id=11}:connect{target=9}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:32.959909Z INFO Node{id=11}:connect{target=9}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:32.960000Z INFO Node{id=14}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:32.960060Z INFO Node{id=14}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:32.960111Z INFO Node{id=2}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:32.960171Z INFO Node{id=2}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:32.960275Z INFO Node{id=9}: sharedstate::worker::sync_manager: got new client remote=11 [INFO] [stdout] 2025-09-01T11:56:32.960346Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:56:32.960404Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:56:32.962872Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727792960) } [INFO] [stdout] 2025-09-01T11:56:32.962910Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: Ping(1756727792959) } [INFO] [stdout] 2025-09-01T11:56:32.963073Z INFO Node{id=2}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=3 [INFO] [stdout] 2025-09-01T11:56:32.963097Z INFO Node{id=2}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:32.963123Z INFO Node{id=11}:connect{target=9}: sharedstate::worker::sync_manager: got pong from peer latency=4 [INFO] [stdout] 2025-09-01T11:56:32.963137Z INFO Node{id=11}:connect{target=9}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:32.963568Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727792960) } [INFO] [stdout] 2025-09-01T11:56:32.963601Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=14 [INFO] [stdout] 2025-09-01T11:56:32.966018Z INFO Node{id=14}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=6 [INFO] [stdout] 2025-09-01T11:56:32.966044Z INFO Node{id=14}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:32.966428Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:32.966462Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 15 peers to 14 peers [INFO] [stdout] 2025-09-01T11:56:32.966738Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:32.966803Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:32.967010Z ERROR Node{id=14}:connect{target=23}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:32.967094Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 15) [INFO] [stdout] 2025-09-01T11:56:32.967211Z INFO Node{id=14}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:32.967283Z INFO Node{id=14}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:32.967351Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(23) [INFO] [stdout] 2025-09-01T11:56:32.967406Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 9 } [INFO] [stdout] 2025-09-01T11:56:32.967455Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:56:32.967468Z ERROR Node{id=2}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[27] peer=27 leader=1 local=2 [INFO] [stdout] 2025-09-01T11:56:32.968000Z ERROR ReadClient{remote=14}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:32.968662Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:32.968709Z INFO Node{id=2}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:32.970885Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:32.970939Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:32.971013Z INFO Node{id=2}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:32.971172Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:32.971318Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:32.971547Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:32.971908Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:32.972282Z INFO Node{id=9}: sequenced_broadcast: 117|Subscriber(3): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:32.972882Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:32.972943Z INFO Node{id=11}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:33.551898Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:33.552158Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:33.552283Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16]) [INFO] [stdout] 2025-09-01T11:56:33.553496Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:33.553675Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:33.553793Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([31]) [INFO] [stdout] 2025-09-01T11:56:33.621369Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:33.621664Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:33.621818Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:33.621839Z ERROR Node{id=22}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:33.705461Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:33.705524Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:56:33.705659Z INFO Node{id=29}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:33.705752Z INFO Node{id=29}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:33.705832Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:56:33.706272Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727793705) } [INFO] [stdout] 2025-09-01T11:56:33.706372Z INFO Node{id=29}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:33.706396Z INFO Node{id=29}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:33.706573Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:33.706729Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:56:33.706750Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 5, 15, 26] peer=26 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:56:33.706878Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:33.706927Z INFO Node{id=29}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:33.706993Z INFO Node{id=29}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:33.707085Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:33.707136Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:33.787343Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:33.787406Z INFO Node{id=28}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=4 leader=1 [INFO] [stdout] 2025-09-01T11:56:33.787583Z INFO Node{id=28}:connect{target=4}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:33.787678Z INFO Node{id=28}:connect{target=4}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:33.787738Z INFO Node{id=4}: sharedstate::worker::sync_manager: got new client remote=28 [INFO] [stdout] 2025-09-01T11:56:33.788375Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: Ping(1756727793787) } [INFO] [stdout] 2025-09-01T11:56:33.788473Z INFO Node{id=28}:connect{target=4}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:33.788519Z INFO Node{id=28}:connect{target=4}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:33.788699Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:33.788858Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 4 } [INFO] [stdout] 2025-09-01T11:56:33.789001Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:33.789108Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:33.789202Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:33.789271Z INFO Node{id=4}: sequenced_broadcast: 134|Subscriber rejected, seq(6818) < min_allowed(37120) [INFO] [stdout] 2025-09-01T11:56:33.789321Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:33.789369Z INFO Node{id=4}: sequenced_broadcast: 134|Subscriber(1): Added, allow_drop: true, next_sequence: 39168, min_allowed_seq: 37120 [INFO] [stdout] 2025-09-01T11:56:33.789472Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 39168) [INFO] [stdout] 2025-09-01T11:56:33.789553Z INFO Node{id=28}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:33.789575Z INFO Node{id=28}: sequenced_broadcast: 137|SequencedBroadcastWorker Started id=137 [INFO] [stdout] 2025-09-01T11:56:33.789644Z INFO Node{id=28}: sequenced_broadcast: 96|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:33.789656Z INFO Node{id=28}: sequenced_broadcast: 96|SequencedBroadcastWorker Stopped id=96 elapsed=71.616345939s iter=35 [INFO] [stdout] 2025-09-01T11:56:33.797148Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:33.797199Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:56:33.797296Z INFO Node{id=13}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:33.797372Z INFO Node{id=13}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:33.797426Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:33.797906Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727793797) } [INFO] [stdout] 2025-09-01T11:56:33.797998Z INFO Node{id=13}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:56:33.798023Z INFO Node{id=13}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:33.798185Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:33.798332Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:56:33.798353Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[27] peer=27 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:33.798458Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:33.798506Z INFO Node{id=13}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:33.798580Z INFO Node{id=13}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:33.798665Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:33.798714Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:33.881919Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:33.882190Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:33.882311Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([24]) [INFO] [stdout] 2025-09-01T11:56:33.968662Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:33.968727Z INFO Node{id=14}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:56:33.968842Z INFO Node{id=14}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:33.968926Z INFO Node{id=14}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:33.968977Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=14 [INFO] [stdout] 2025-09-01T11:56:33.969110Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:33.969137Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:56:33.969297Z INFO Node{id=2}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:33.969356Z INFO Node{id=2}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:33.969430Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:56:33.969930Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: Ping(1756727793968) } [INFO] [stdout] 2025-09-01T11:56:33.970044Z INFO Node{id=14}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=2 [INFO] [stdout] 2025-09-01T11:56:33.970069Z INFO Node{id=14}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:33.970247Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:33.970302Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727793969) } [INFO] [stdout] 2025-09-01T11:56:33.970438Z INFO Node{id=2}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:33.970462Z INFO Node{id=2}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:33.970518Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:56:33.970695Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:33.970845Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:33.971016Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:56:33.971036Z ERROR Node{id=2}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=14 leader=1 local=2 [INFO] [stdout] 2025-09-01T11:56:33.971137Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:33.971186Z INFO Node{id=2}:connect{target=14}:WritePeer{remote=14}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:33.971280Z INFO Node{id=2}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:33.971363Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:33.971429Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:33.971468Z ERROR ReadClient{remote=2}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:33.971664Z INFO Node{id=12}: sequenced_broadcast: 124|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:33.971716Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:33.971892Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:33.971944Z INFO Node{id=14}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:34.050874Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:34.051337Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:34.051447Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:34.051469Z ERROR Node{id=24}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:34.146419Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:34.146717Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:34.146917Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([31]) [INFO] [stdout] 2025-09-01T11:56:34.427122Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:34.427372Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:34.427486Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:34.427506Z ERROR Node{id=18}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:34.472042Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:34.472240Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:34.472350Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([27]) [INFO] [stdout] 2025-09-01T11:56:34.629989Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:34.630205Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:34.630333Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:56:34.708143Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:34.708194Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:56:34.708301Z INFO Node{id=29}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:34.708382Z INFO Node{id=29}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:34.708439Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:56:34.708908Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727794708) } [INFO] [stdout] 2025-09-01T11:56:34.708932Z INFO Node{id=8}: sharedstate::worker::sync_manager: Discovered Peer client_id=29 [INFO] [stdout] 2025-09-01T11:56:34.709014Z INFO Node{id=29}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:34.709039Z INFO Node{id=29}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:34.709193Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:34.709222Z INFO Node{id=8}: sharedstate::worker::sync_manager: Broadcasted 9 peers to 8 peers [INFO] [stdout] 2025-09-01T11:56:34.709308Z ERROR Node{id=29}:connect{target=8}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:34.709375Z INFO Node{id=29}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:34.709441Z INFO Node{id=29}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:34.709498Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(8) [INFO] [stdout] 2025-09-01T11:56:34.709532Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:34.709689Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:34.799704Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:34.799773Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:56:34.799878Z INFO Node{id=13}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:34.799956Z INFO Node{id=13}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:34.800008Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:34.800453Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727794799) } [INFO] [stdout] 2025-09-01T11:56:34.800558Z INFO Node{id=13}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:34.800583Z INFO Node{id=13}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:34.800726Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:34.800977Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:56:34.801036Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[18] peer=18 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:34.801157Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:34.801201Z INFO Node{id=13}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:34.801255Z INFO Node{id=13}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:34.801327Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:34.801368Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:34.972001Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:34.972059Z INFO Node{id=2}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=30 leader=1 [INFO] [stdout] 2025-09-01T11:56:34.972159Z INFO Node{id=2}:connect{target=30}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:34.972232Z INFO Node{id=2}:connect{target=30}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:34.972279Z INFO Node{id=30}: sharedstate::worker::sync_manager: got new client remote=2 [INFO] [stdout] 2025-09-01T11:56:34.972663Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: Ping(1756727794972) } [INFO] [stdout] 2025-09-01T11:56:34.972741Z INFO Node{id=2}:connect{target=30}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:56:34.972783Z INFO Node{id=2}:connect{target=30}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:34.972919Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:34.973043Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 30 } [INFO] [stdout] 2025-09-01T11:56:34.973196Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:34.973300Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:34.973399Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:34.973470Z INFO Node{id=30}: sequenced_broadcast: 132|Subscriber rejected, seq(6818) < min_allowed(37120) [INFO] [stdout] 2025-09-01T11:56:34.973519Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:34.973581Z INFO Node{id=30}: sequenced_broadcast: 132|Subscriber(1): Added, allow_drop: true, next_sequence: 39168, min_allowed_seq: 37120 [INFO] [stdout] 2025-09-01T11:56:34.973695Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 39168) [INFO] [stdout] 2025-09-01T11:56:34.973794Z INFO Node{id=2}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:34.973819Z INFO Node{id=2}: sequenced_broadcast: 138|SequencedBroadcastWorker Started id=138 [INFO] [stdout] 2025-09-01T11:56:34.973892Z INFO Node{id=2}: sequenced_broadcast: 123|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:34.973905Z INFO Node{id=2}: sequenced_broadcast: 123|SequencedBroadcastWorker Stopped id=123 elapsed=47.519727486s iter=2 [INFO] [stdout] 2025-09-01T11:56:34.989565Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:34.989835Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:34.989954Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20]) [INFO] [stdout] 2025-09-01T11:56:35.555109Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:35.555463Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:35.555595Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([31]) [INFO] [stdout] 2025-09-01T11:56:35.623362Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:35.623617Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:35.623725Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:35.623745Z ERROR Node{id=22}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:35.710517Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:35.710578Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:56:35.710690Z INFO Node{id=29}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:35.710781Z INFO Node{id=29}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:35.710843Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:56:35.711549Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727795710) } [INFO] [stdout] 2025-09-01T11:56:35.711656Z INFO Node{id=29}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:35.711682Z INFO Node{id=29}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:35.711870Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:35.711987Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:56:35.712004Z ERROR Node{id=29}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[27] peer=27 leader=1 local=29 [INFO] [stdout] 2025-09-01T11:56:35.712091Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:35.712136Z INFO Node{id=29}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:35.712200Z INFO Node{id=29}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:35.712283Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:35.712337Z ERROR ReadClient{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:35.771662Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:35.772078Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:35.772181Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9]) [INFO] [stdout] 2025-09-01T11:56:35.802271Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:35.802332Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:56:35.802522Z INFO Node{id=13}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:35.802607Z INFO Node{id=13}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:35.802656Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:35.803071Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727795802) } [INFO] [stdout] 2025-09-01T11:56:35.803149Z INFO Node{id=13}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:35.803171Z INFO Node{id=13}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:35.803317Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:35.803451Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:56:35.803471Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 12] peer=12 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:35.803568Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:35.803610Z INFO Node{id=13}:connect{target=12}:WritePeer{remote=12}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:35.803663Z INFO Node{id=13}:connect{target=12}:ReadPeer{remote=12}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:35.803735Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:35.804772Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:35.884087Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:35.884364Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:35.884479Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([24]) [INFO] [stdout] 2025-09-01T11:56:36.037594Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:36.037703Z INFO Node{id=24}:connect{target=13}:ReadPeer{remote=13}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:36.038062Z ERROR ReadClient{remote=24}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:36.071607Z INFO sharedstate::testing::fuzzy_test: node1 sequence updated: 6817 => 39167 [INFO] [stdout] 2025-09-01T11:56:36.071648Z INFO sharedstate::testing::fuzzy_test: node20 sequence updated: 6817 => 39167 [INFO] [stdout] 2025-09-01T11:56:36.071656Z INFO sharedstate::testing::fuzzy_test: node27 sequence updated: 6817 => 39167 [INFO] [stdout] 2025-09-01T11:56:36.148656Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:36.149039Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:36.149156Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([31]) [INFO] [stdout] 2025-09-01T11:56:36.428705Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:36.428956Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:36.429065Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:36.429085Z ERROR Node{id=18}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:36.473122Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:36.473355Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:36.473455Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([27]) [INFO] [stdout] 2025-09-01T11:56:36.713716Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:36.713794Z INFO Node{id=29}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=4 leader=1 [INFO] [stdout] 2025-09-01T11:56:36.713903Z INFO Node{id=29}:connect{target=4}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:36.714020Z INFO Node{id=29}:connect{target=4}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:36.714080Z INFO Node{id=4}: sharedstate::worker::sync_manager: got new client remote=29 [INFO] [stdout] 2025-09-01T11:56:36.714543Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: Ping(1756727796714) } [INFO] [stdout] 2025-09-01T11:56:36.714654Z INFO Node{id=29}:connect{target=4}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:56:36.714681Z INFO Node{id=29}:connect{target=4}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:36.714872Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:36.715000Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 4 } [INFO] [stdout] 2025-09-01T11:56:36.715174Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:36.715322Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:36.715490Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:36.715595Z INFO Node{id=4}: sequenced_broadcast: 134|Subscriber rejected, seq(6818) < min_allowed(37120) [INFO] [stdout] 2025-09-01T11:56:36.715673Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:36.715739Z INFO Node{id=4}: sequenced_broadcast: 134|Subscriber(2): Added, allow_drop: true, next_sequence: 39168, min_allowed_seq: 37120 [INFO] [stdout] 2025-09-01T11:56:36.715873Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 39168) [INFO] [stdout] 2025-09-01T11:56:36.715951Z INFO Node{id=29}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:36.715974Z INFO Node{id=29}: sequenced_broadcast: 139|SequencedBroadcastWorker Started id=139 [INFO] [stdout] 2025-09-01T11:56:36.716053Z INFO Node{id=29}: sequenced_broadcast: 90|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:36.716067Z INFO Node{id=29}: sequenced_broadcast: 90|SequencedBroadcastWorker Stopped id=90 elapsed=97.637129141s iter=2378 [INFO] [stdout] 2025-09-01T11:56:36.716376Z ERROR Node{id=18}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:36.716439Z ERROR Node{id=22}:connect{target=29}:ReadPeer{remote=29}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:36.716490Z WARN Node{id=18}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:56:36.716501Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:36.716556Z WARN Node{id=22}: sharedstate::worker::sync_manager: leader read channel closed [INFO] [stdout] 2025-09-01T11:56:36.716567Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:36.804689Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:36.804734Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:56:36.809849Z INFO Node{id=13}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:36.809933Z INFO Node{id=13}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:36.810916Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:36.819684Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727796809) } [INFO] [stdout] 2025-09-01T11:56:36.819716Z INFO Node{id=8}: sharedstate::worker::sync_manager: Discovered Peer client_id=13 [INFO] [stdout] 2025-09-01T11:56:36.822554Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:36.824555Z INFO Node{id=13}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:56:36.824584Z INFO Node{id=13}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:36.830477Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:36.830658Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:36.833672Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:56:36.833943Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:56:36.833966Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 5, 15, 26, 8] peer=8 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:36.836400Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:36.836495Z INFO Node{id=13}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:36.836876Z INFO Node{id=13}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:36.837531Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:36.840359Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:37.040175Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:37.040225Z INFO Node{id=24}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:56:37.041267Z INFO Node{id=24}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:37.041352Z INFO Node{id=24}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:37.042270Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=24 [INFO] [stdout] 2025-09-01T11:56:37.050535Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: Ping(1756727797041) } [INFO] [stdout] 2025-09-01T11:56:37.055334Z INFO Node{id=24}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=14 [INFO] [stdout] 2025-09-01T11:56:37.055371Z INFO Node{id=24}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:37.060093Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:37.063847Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:56:37.068713Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:37.070050Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:37.071545Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:37.072022Z INFO Node{id=3}: sequenced_broadcast: 121|Subscriber(3): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:37.072045Z INFO Node{id=3}: sequenced_broadcast: 1|Subscriber(121): channel closed, dropping [INFO] [stdout] 2025-09-01T11:56:37.072064Z INFO Node{id=3}: sequenced_broadcast: 2|Subscriber(121): channel closed, dropping [INFO] [stdout] 2025-09-01T11:56:37.078595Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:37.078679Z INFO Node{id=24}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:37.556596Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:37.562211Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:37.564197Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([31]) [INFO] [stdout] 2025-09-01T11:56:37.655701Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:37.662148Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:37.664359Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9]) [INFO] [stdout] 2025-09-01T11:56:37.718431Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:37.718486Z INFO Node{id=22}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=5 leader=1 [INFO] [stdout] 2025-09-01T11:56:37.718543Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:37.718565Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:56:37.719350Z INFO Node{id=22}:connect{target=5}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:37.719420Z INFO Node{id=22}:connect{target=5}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:37.719463Z INFO Node{id=18}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:37.719508Z INFO Node{id=18}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:37.722463Z INFO Node{id=5}: sharedstate::worker::sync_manager: got new client remote=22 [INFO] [stdout] 2025-09-01T11:56:37.722535Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:37.727170Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: Ping(1756727797719) } [INFO] [stdout] 2025-09-01T11:56:37.727558Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727797719) } [INFO] [stdout] 2025-09-01T11:56:37.731985Z INFO Node{id=22}:connect{target=5}: sharedstate::worker::sync_manager: got pong from peer latency=12 [INFO] [stdout] 2025-09-01T11:56:37.732019Z INFO Node{id=22}:connect{target=5}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:37.732096Z INFO Node{id=18}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:56:37.732120Z INFO Node{id=18}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:37.736271Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:37.737623Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:37.739537Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:56:37.739560Z ERROR Node{id=18}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[31] peer=31 leader=1 local=18 [INFO] [stdout] 2025-09-01T11:56:37.742839Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:37.742885Z INFO Node{id=18}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:37.743206Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 5 } [INFO] [stdout] 2025-09-01T11:56:37.743788Z INFO Node{id=18}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:37.744633Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:37.745571Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:37.756223Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:37.759070Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:37.759814Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:37.763579Z INFO Node{id=5}: sequenced_broadcast: 135|Subscriber rejected, seq(6818) < min_allowed(39168) [INFO] [stdout] 2025-09-01T11:56:37.766137Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:37.768282Z INFO Node{id=5}: sequenced_broadcast: 135|Subscriber rejected, max_seq(39642) < seq(39643) [INFO] [stdout] 2025-09-01T11:56:37.784868Z INFO Node{id=5}: sequenced_broadcast: 135|Subscriber(1): Added, allow_drop: true, next_sequence: 39643, min_allowed_seq: 39168 [INFO] [stdout] 2025-09-01T11:56:37.789251Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 39643) [INFO] [stdout] 2025-09-01T11:56:37.789490Z INFO Node{id=22}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:37.790821Z INFO Node{id=22}: sequenced_broadcast: 140|SequencedBroadcastWorker Started id=140 [INFO] [stdout] 2025-09-01T11:56:37.792340Z INFO Node{id=22}: sequenced_broadcast: 129|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:37.792358Z INFO Node{id=22}: sequenced_broadcast: 129|SequencedBroadcastWorker Stopped id=129 elapsed=36.263548033s iter=2 [INFO] [stdout] 2025-09-01T11:56:37.837152Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:37.837199Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=17 leader=1 [INFO] [stdout] 2025-09-01T11:56:37.840276Z INFO Node{id=13}:connect{target=17}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:37.840363Z INFO Node{id=13}:connect{target=17}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:37.841214Z INFO Node{id=17}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:37.854851Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727797840) } [INFO] [stdout] 2025-09-01T11:56:37.860803Z INFO Node{id=13}:connect{target=17}: sharedstate::worker::sync_manager: got pong from peer latency=20 [INFO] [stdout] 2025-09-01T11:56:37.860849Z INFO Node{id=13}:connect{target=17}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:37.866277Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:37.873220Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 17 } [INFO] [stdout] 2025-09-01T11:56:37.873258Z ERROR Node{id=13}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17] peer=17 leader=1 local=13 [INFO] [stdout] 2025-09-01T11:56:37.877494Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:37.877924Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:37.877974Z INFO Node{id=13}:connect{target=17}:WritePeer{remote=17}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:37.878546Z INFO Node{id=27}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:37.879650Z INFO Node{id=13}:connect{target=17}:ReadPeer{remote=17}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:37.879838Z ERROR ReadClient{remote=27}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:37.880751Z ERROR ReadClient{remote=13}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:37.881585Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:38.090202Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:38.097194Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:38.099613Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3, 29, 18, 15]) [INFO] [stdout] 2025-09-01T11:56:38.150970Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:38.155918Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:38.158415Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([31]) [INFO] [stdout] 2025-09-01T11:56:38.169259Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:38.178629Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:38.183087Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20, 32]) [INFO] [stdout] 2025-09-01T11:56:38.461148Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:38.462059Z INFO Node{id=31}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:38.463213Z ERROR ReadClient{remote=31}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:38.480818Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:38.487740Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:38.490690Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3, 29, 18, 25]) [INFO] [stdout] 2025-09-01T11:56:38.490719Z WARN Node{id=17}: sharedstate::worker::sync_manager: leader path from peer includes us [INFO] [stdout] 2025-09-01T11:56:38.492047Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:38.492663Z INFO Node{id=17}:connect{target=25}:ReadPeer{remote=25}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:38.493881Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:38.684898Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:38.692528Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:38.695054Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3, 29, 18, 15, 26]) [INFO] [stdout] 2025-09-01T11:56:38.746531Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:38.746580Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:56:38.747439Z INFO Node{id=18}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:38.747503Z INFO Node{id=18}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:38.748164Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:38.758083Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727798747) } [INFO] [stdout] 2025-09-01T11:56:38.762483Z INFO Node{id=18}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:56:38.762518Z INFO Node{id=18}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:38.769269Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:38.774796Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:56:38.774828Z ERROR Node{id=18}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9, 3] peer=3 leader=1 local=18 [INFO] [stdout] 2025-09-01T11:56:38.776032Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:38.776079Z INFO Node{id=18}:connect{target=3}:WritePeer{remote=3}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:38.780921Z INFO Node{id=18}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:38.781256Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:38.783017Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:38.881943Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:38.881990Z INFO Node{id=13}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=1 leader=1 [INFO] [stdout] 2025-09-01T11:56:38.882884Z INFO Node{id=13}:connect{target=1}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:38.882961Z INFO Node{id=13}:connect{target=1}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:38.883934Z INFO Node{id=1}: sharedstate::worker::sync_manager: got new client remote=13 [INFO] [stdout] 2025-09-01T11:56:38.886709Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:38.886742Z INFO Node{id=27}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=12 leader=1 [INFO] [stdout] 2025-09-01T11:56:38.887565Z INFO Node{id=27}:connect{target=12}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:38.887631Z INFO Node{id=27}:connect{target=12}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:38.890795Z INFO Node{id=12}: sharedstate::worker::sync_manager: got new client remote=27 [INFO] [stdout] 2025-09-01T11:56:38.894520Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: Ping(1756727798882) } [INFO] [stdout] 2025-09-01T11:56:38.899830Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: Ping(1756727798887) } [INFO] [stdout] 2025-09-01T11:56:38.900694Z INFO Node{id=13}:connect{target=1}: sharedstate::worker::sync_manager: got pong from peer latency=18 [INFO] [stdout] 2025-09-01T11:56:38.900725Z INFO Node{id=13}:connect{target=1}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:38.902525Z INFO Node{id=27}:connect{target=12}: sharedstate::worker::sync_manager: got pong from peer latency=15 [INFO] [stdout] 2025-09-01T11:56:38.902554Z INFO Node{id=27}:connect{target=12}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:38.908883Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:38.910696Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:38.912690Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 12 } [INFO] [stdout] 2025-09-01T11:56:38.916478Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 1 } [INFO] [stdout] 2025-09-01T11:56:38.918905Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:38.920303Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:38.922950Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:38.923771Z INFO Node{id=12}: sequenced_broadcast: 124|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:38.925833Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:38.930225Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:38.930318Z INFO Node{id=27}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:38.931255Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:38.931835Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:38.933144Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, seq(6818) < min_allowed(38107) [INFO] [stdout] 2025-09-01T11:56:38.950162Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: SubscribeFresh } [INFO] [stdout] 2025-09-01T11:56:38.950903Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber rejected, max_seq(40160) < seq(40162) [INFO] [stdout] 2025-09-01T11:56:38.964732Z INFO Node{id=1}: sequenced_broadcast: 20|Subscriber(8): Added, allow_drop: true, next_sequence: 40162, min_allowed_seq: 38116 [INFO] [stdout] 2025-09-01T11:56:38.970480Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::FreshState(seq: 40162) [INFO] [stdout] 2025-09-01T11:56:38.970591Z INFO Node{id=13}: sharedstate::worker::sync_manager: Connected to leader with FreshState [INFO] [stdout] 2025-09-01T11:56:38.971541Z INFO Node{id=13}: sequenced_broadcast: 141|SequencedBroadcastWorker Started id=141 [INFO] [stdout] 2025-09-01T11:56:38.973139Z INFO Node{id=13}: sequenced_broadcast: 87|Stopping worker due to shutdown [INFO] [stdout] 2025-09-01T11:56:38.973159Z INFO Node{id=13}: sequenced_broadcast: 87|SequencedBroadcastWorker Stopped id=87 elapsed=102.948842482s iter=2688 [INFO] [stdout] 2025-09-01T11:56:39.145081Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:39.150984Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:39.154800Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:39.154835Z ERROR Node{id=25}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:39.469409Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:39.469453Z INFO Node{id=31}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:56:39.470236Z INFO Node{id=31}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:39.470308Z INFO Node{id=31}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:39.471103Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=31 [INFO] [stdout] 2025-09-01T11:56:39.481310Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: Ping(1756727799470) } [INFO] [stdout] 2025-09-01T11:56:39.481345Z INFO Node{id=8}: sharedstate::worker::sync_manager: Discovered Peer client_id=31 [INFO] [stdout] 2025-09-01T11:56:39.483683Z INFO Node{id=31}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:56:39.485803Z INFO Node{id=31}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:39.490533Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:39.497109Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:39.497151Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:56:39.498001Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:56:39.498073Z INFO Node{id=17}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:39.498132Z INFO Node{id=17}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:39.498876Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:39.501898Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:39.503373Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:39.504524Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:39.506425Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727799498) } [INFO] [stdout] 2025-09-01T11:56:39.506637Z INFO Node{id=8}: sequenced_broadcast: 122|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:39.512012Z INFO Node{id=17}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=14 [INFO] [stdout] 2025-09-01T11:56:39.512053Z INFO Node{id=17}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:39.514970Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:39.515044Z INFO Node{id=31}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:39.517434Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:39.523243Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:56:39.523275Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 12, 27] peer=27 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:39.526367Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:39.526423Z INFO Node{id=17}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:39.527718Z INFO Node{id=17}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:39.528729Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:39.528895Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:39.548789Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:39.549675Z INFO Node{id=7}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:39.550682Z ERROR ReadClient{remote=7}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:39.713623Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:39.713671Z INFO Node{id=8}: sharedstate::worker::sync_manager: Broadcasted 11 peers to 10 peers [INFO] [stdout] 2025-09-01T11:56:39.718274Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 11) [INFO] [stdout] 2025-09-01T11:56:39.780395Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:39.780445Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:56:39.781297Z INFO Node{id=18}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:39.781414Z INFO Node{id=18}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:39.782357Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:39.795158Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727799781) } [INFO] [stdout] 2025-09-01T11:56:39.795194Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=18 [INFO] [stdout] 2025-09-01T11:56:39.800416Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:39.800466Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 16 peers to 15 peers [INFO] [stdout] 2025-09-01T11:56:39.802467Z INFO Node{id=18}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=21 [INFO] [stdout] 2025-09-01T11:56:39.802498Z INFO Node{id=18}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:39.803486Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 16) [INFO] [stdout] 2025-09-01T11:56:39.807427Z ERROR Node{id=18}:connect{target=23}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:39.807931Z INFO Node{id=18}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:39.808283Z INFO Node{id=18}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:39.808454Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(23) [INFO] [stdout] 2025-09-01T11:56:39.808997Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:39.810490Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:39.943982Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:40.007753Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:40.025127Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20, 32]) [INFO] [stdout] 2025-09-01T11:56:40.137988Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:40.139619Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:40.140322Z INFO Node{id=10}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:40.142309Z ERROR ReadClient{remote=10}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:40.143730Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 12, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:40.148334Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16]) [INFO] [stdout] 2025-09-01T11:56:40.527411Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:40.527455Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=18 leader=1 [INFO] [stdout] 2025-09-01T11:56:40.528205Z INFO Node{id=17}:connect{target=18}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:40.528306Z INFO Node{id=17}:connect{target=18}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:40.529073Z INFO Node{id=18}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:40.536329Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727800528) } [INFO] [stdout] 2025-09-01T11:56:40.538301Z INFO Node{id=17}:connect{target=18}: sharedstate::worker::sync_manager: got pong from peer latency=10 [INFO] [stdout] 2025-09-01T11:56:40.538330Z INFO Node{id=17}:connect{target=18}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:40.542610Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:40.547576Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 18 } [INFO] [stdout] 2025-09-01T11:56:40.547609Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[] peer=18 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:40.548514Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:40.548583Z INFO Node{id=17}:connect{target=18}:WritePeer{remote=18}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:40.549220Z INFO Node{id=17}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:40.549566Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:40.553854Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:40.555624Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:40.555655Z INFO Node{id=7}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:56:40.556315Z INFO Node{id=7}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:40.556377Z INFO Node{id=7}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:40.557033Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=7 [INFO] [stdout] 2025-09-01T11:56:40.562276Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: Ping(1756727800556) } [INFO] [stdout] 2025-09-01T11:56:40.562304Z INFO Node{id=8}: sharedstate::worker::sync_manager: Discovered Peer client_id=7 [INFO] [stdout] 2025-09-01T11:56:40.564434Z INFO Node{id=7}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=8 [INFO] [stdout] 2025-09-01T11:56:40.564460Z INFO Node{id=7}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:40.570055Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:40.572713Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 8 } [INFO] [stdout] 2025-09-01T11:56:40.575596Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:40.578497Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 7, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:40.579714Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:40.579921Z INFO Node{id=8}: sequenced_broadcast: 122|Subscriber(2): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:40.583147Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:40.583209Z INFO Node{id=7}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:40.710238Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:40.732840Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 30, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:40.735203Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20]) [INFO] [stdout] 2025-09-01T11:56:40.810864Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:40.810909Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=14 leader=1 [INFO] [stdout] 2025-09-01T11:56:40.811585Z INFO Node{id=18}:connect{target=14}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:40.811650Z INFO Node{id=18}:connect{target=14}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:40.812359Z INFO Node{id=14}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:40.820191Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727800811) } [INFO] [stdout] 2025-09-01T11:56:40.824452Z INFO Node{id=18}:connect{target=14}: sharedstate::worker::sync_manager: got pong from peer latency=13 [INFO] [stdout] 2025-09-01T11:56:40.824492Z INFO Node{id=18}:connect{target=14}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:40.828460Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:40.834446Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 14 } [INFO] [stdout] 2025-09-01T11:56:40.834479Z ERROR Node{id=18}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 12, 14] peer=14 leader=1 local=18 [INFO] [stdout] 2025-09-01T11:56:40.835592Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:40.835648Z INFO Node{id=18}:connect{target=14}:WritePeer{remote=14}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:40.836854Z INFO Node{id=18}:connect{target=14}:ReadPeer{remote=14}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:40.837660Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:40.837783Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:41.145481Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:41.145523Z INFO Node{id=10}: sharedstate::worker::sync_manager: found connect target option_count=15 peers=16 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:56:41.146231Z INFO Node{id=10}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:41.146308Z INFO Node{id=10}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:41.147177Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=10 [INFO] [stdout] 2025-09-01T11:56:41.152639Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: Ping(1756727801146) } [INFO] [stdout] 2025-09-01T11:56:41.154708Z INFO Node{id=10}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=8 [INFO] [stdout] 2025-09-01T11:56:41.154735Z INFO Node{id=10}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:41.157421Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:41.160187Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:41.161096Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:41.163345Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:56:41.163391Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:41.163409Z ERROR Node{id=25}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:41.166061Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:41.166951Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 10, message: SubscribeRecovery(RecoverableStateDetails { id: 14075980197196158466, generation: 1, recover_accept_seq: 6818, state_accept_seq: 6817, history: [] }) } [INFO] [stdout] 2025-09-01T11:56:41.170344Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 0) [INFO] [stdout] 2025-09-01T11:56:41.170818Z INFO Node{id=7}: sequenced_broadcast: 116|Subscriber(1): Added, allow_drop: true, next_sequence: 6818, min_allowed_seq: 6818 [INFO] [stdout] 2025-09-01T11:56:41.174232Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::RecoveryAccepted(seq: 6818) [INFO] [stdout] 2025-09-01T11:56:41.174290Z INFO Node{id=10}: sharedstate::worker::sync_manager: Connected to leader with Recovery [INFO] [stdout] 2025-09-01T11:56:41.188716Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:41.192166Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 9, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:41.196428Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23]) [INFO] [stdout] 2025-09-01T11:56:41.549267Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:41.549329Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=6 leader=1 [INFO] [stdout] 2025-09-01T11:56:41.549471Z INFO Node{id=17}:connect{target=6}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:41.549561Z INFO Node{id=17}:connect{target=6}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:41.549621Z INFO Node{id=6}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:41.550133Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727801549) } [INFO] [stdout] 2025-09-01T11:56:41.550226Z INFO Node{id=17}:connect{target=6}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:41.550251Z INFO Node{id=17}:connect{target=6}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:41.550416Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:41.550581Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 6 } [INFO] [stdout] 2025-09-01T11:56:41.550604Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9, 6] peer=6 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:41.550715Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:41.550785Z INFO Node{id=17}:connect{target=6}:WritePeer{remote=6}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:41.550852Z INFO Node{id=17}:connect{target=6}:ReadPeer{remote=6}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:41.550935Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:41.550992Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:41.787171Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:41.787386Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:41.787477Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:41.787493Z ERROR Node{id=16}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:41.835933Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:41.835987Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=24 leader=1 [INFO] [stdout] 2025-09-01T11:56:41.836073Z INFO Node{id=18}:connect{target=24}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:41.836140Z INFO Node{id=18}:connect{target=24}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:41.836179Z INFO Node{id=24}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:41.836548Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727801836) } [INFO] [stdout] 2025-09-01T11:56:41.836628Z INFO Node{id=18}:connect{target=24}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:56:41.836649Z INFO Node{id=18}:connect{target=24}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:41.836795Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:41.836916Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 24 } [INFO] [stdout] 2025-09-01T11:56:41.836935Z ERROR Node{id=18}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9, 3, 24] peer=24 leader=1 local=18 [INFO] [stdout] 2025-09-01T11:56:41.837030Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:41.837067Z INFO Node{id=18}:connect{target=24}:WritePeer{remote=24}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:41.837117Z INFO Node{id=18}:connect{target=24}:ReadPeer{remote=24}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:41.837184Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:41.837219Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:41.946025Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:41.946252Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 21, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:41.946368Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19]) [INFO] [stdout] 2025-09-01T11:56:41.952927Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:41.953128Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:41.953240Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:41.953260Z ERROR Node{id=15}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:42.072065Z INFO sharedstate::testing::fuzzy_test: node0 sequence updated: 39167 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.072100Z INFO sharedstate::testing::fuzzy_test: node1 sequence updated: 39167 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.072105Z INFO sharedstate::testing::fuzzy_test: node3 sequence updated: 39167 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.072109Z INFO sharedstate::testing::fuzzy_test: node4 sequence updated: 39167 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.072116Z INFO sharedstate::testing::fuzzy_test: node12 sequence updated: 6817 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.072122Z INFO sharedstate::testing::fuzzy_test: node18 sequence updated: 39167 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.072126Z INFO sharedstate::testing::fuzzy_test: node19 sequence updated: 39167 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.072129Z INFO sharedstate::testing::fuzzy_test: node20 sequence updated: 39167 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.072133Z INFO sharedstate::testing::fuzzy_test: node21 sequence updated: 6817 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.072138Z INFO sharedstate::testing::fuzzy_test: node27 sequence updated: 39167 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.072141Z INFO sharedstate::testing::fuzzy_test: node28 sequence updated: 6817 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.072145Z INFO sharedstate::testing::fuzzy_test: node29 sequence updated: 39167 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.072149Z INFO sharedstate::testing::fuzzy_test: node31 sequence updated: 39167 => 41215 [INFO] [stdout] 2025-09-01T11:56:42.551719Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:42.551793Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=11 leader=1 [INFO] [stdout] 2025-09-01T11:56:42.551897Z INFO Node{id=17}:connect{target=11}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:42.551982Z INFO Node{id=17}:connect{target=11}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:42.552035Z INFO Node{id=11}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:42.552500Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727802551) } [INFO] [stdout] 2025-09-01T11:56:42.552595Z INFO Node{id=17}:connect{target=11}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:42.552619Z INFO Node{id=17}:connect{target=11}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:42.552794Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:42.552913Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 11 } [INFO] [stdout] 2025-09-01T11:56:42.552930Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9, 11] peer=11 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:42.553005Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:42.553041Z INFO Node{id=17}:connect{target=11}:WritePeer{remote=11}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:42.553100Z INFO Node{id=17}:connect{target=11}:ReadPeer{remote=11}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:42.553162Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:42.553196Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:42.855158Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:42.855212Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:56:42.855416Z INFO Node{id=18}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:42.855498Z INFO Node{id=18}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:42.855589Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:42.856311Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727802855) } [INFO] [stdout] 2025-09-01T11:56:42.856333Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=18 [INFO] [stdout] 2025-09-01T11:56:42.856410Z INFO Node{id=18}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:42.856432Z INFO Node{id=18}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:42.856575Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:42.856614Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 14 peers to 13 peers [INFO] [stdout] 2025-09-01T11:56:42.856796Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 14) [INFO] [stdout] 2025-09-01T11:56:42.856833Z ERROR Node{id=18}:connect{target=20}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:42.856883Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 14) [INFO] [stdout] 2025-09-01T11:56:42.856912Z INFO Node{id=18}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:42.856969Z INFO Node{id=18}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:42.857014Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(20) [INFO] [stdout] 2025-09-01T11:56:42.857044Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:42.857238Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:42.968070Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:42.968314Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 11, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:42.968425Z INFO Node{id=11}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9]) [INFO] [stdout] 2025-09-01T11:56:43.164030Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:43.164389Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:43.164492Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:43.164509Z ERROR Node{id=25}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:43.553538Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:43.553818Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:43.553928Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([16]) [INFO] [stdout] 2025-09-01T11:56:43.553982Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:43.554009Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:56:43.554149Z INFO Node{id=17}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:43.554224Z INFO Node{id=17}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:43.554269Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:43.554653Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727803554) } [INFO] [stdout] 2025-09-01T11:56:43.554749Z INFO Node{id=17}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:56:43.554819Z INFO Node{id=17}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:43.554977Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:43.555121Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:56:43.555142Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3, 29, 18, 15, 26] peer=26 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:43.555244Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:43.555330Z INFO Node{id=17}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:43.555397Z INFO Node{id=17}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:43.555485Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:43.555534Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:43.788577Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:43.788885Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:43.788993Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:43.789013Z ERROR Node{id=16}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:43.790197Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:43.790398Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 28, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:43.790511Z INFO Node{id=28}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20, 32, 4]) [INFO] [stdout] 2025-09-01T11:56:43.858318Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:43.858375Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:56:43.858479Z INFO Node{id=18}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:43.858555Z INFO Node{id=18}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:43.858605Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:43.859035Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727803858) } [INFO] [stdout] 2025-09-01T11:56:43.859057Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=18 [INFO] [stdout] 2025-09-01T11:56:43.859128Z INFO Node{id=18}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:43.859148Z INFO Node{id=18}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:43.859270Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:43.859310Z INFO Node{id=21}: sharedstate::worker::sync_manager: Broadcasted 14 peers to 13 peers [INFO] [stdout] 2025-09-01T11:56:43.859380Z ERROR Node{id=18}:connect{target=21}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:43.859428Z INFO Node{id=18}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:43.859488Z INFO Node{id=18}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:43.859542Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(21) [INFO] [stdout] 2025-09-01T11:56:43.859577Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:43.859736Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:43.954843Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:43.955130Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:43.955244Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:43.955265Z ERROR Node{id=15}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:43.971589Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:43.971867Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 14, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:43.971985Z INFO Node{id=14}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 12]) [INFO] [stdout] 2025-09-01T11:56:44.556596Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:44.556657Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=3 leader=1 [INFO] [stdout] 2025-09-01T11:56:44.556806Z INFO Node{id=17}:connect{target=3}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:44.556899Z INFO Node{id=17}:connect{target=3}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:44.556959Z INFO Node{id=3}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:44.557454Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727804556) } [INFO] [stdout] 2025-09-01T11:56:44.557557Z INFO Node{id=17}:connect{target=3}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:44.557584Z INFO Node{id=17}:connect{target=3}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:44.557741Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:44.557914Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 3 } [INFO] [stdout] 2025-09-01T11:56:44.557936Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9, 3] peer=3 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:44.558043Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:44.558094Z INFO Node{id=17}:connect{target=3}:WritePeer{remote=3}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:44.558167Z INFO Node{id=17}:connect{target=3}:ReadPeer{remote=3}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:44.558254Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:44.558312Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:44.637926Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:44.638185Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 20, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:44.638324Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:56:44.714456Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:44.714518Z INFO Node{id=8}: sharedstate::worker::sync_manager: Broadcasted 12 peers to 11 peers [INFO] [stdout] 2025-09-01T11:56:44.714680Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 12) [INFO] [stdout] 2025-09-01T11:56:44.714734Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 12) [INFO] [stdout] 2025-09-01T11:56:44.859922Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:44.859974Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=26 leader=1 [INFO] [stdout] 2025-09-01T11:56:44.860100Z INFO Node{id=18}:connect{target=26}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:44.860180Z INFO Node{id=18}:connect{target=26}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:44.860258Z INFO Node{id=26}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:44.860730Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727804860) } [INFO] [stdout] 2025-09-01T11:56:44.860841Z INFO Node{id=18}:connect{target=26}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:56:44.860865Z INFO Node{id=18}:connect{target=26}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:44.861006Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:44.861150Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 26 } [INFO] [stdout] 2025-09-01T11:56:44.861172Z ERROR Node{id=18}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3, 29, 18, 15, 26] peer=26 leader=1 local=18 [INFO] [stdout] 2025-09-01T11:56:44.861269Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:44.861327Z INFO Node{id=18}:connect{target=26}:WritePeer{remote=26}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:44.861392Z INFO Node{id=18}:connect{target=26}:ReadPeer{remote=26}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:44.861494Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:44.861546Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:44.973975Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:44.974208Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 2, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:44.974315Z INFO Node{id=2}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20, 30]) [INFO] [stdout] 2025-09-01T11:56:44.991593Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:44.991861Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 32, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:44.991965Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20]) [INFO] [stdout] 2025-09-01T11:56:45.165572Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:45.165836Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:45.165937Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:45.165962Z ERROR Node{id=25}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:45.555422Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:45.555688Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:45.555834Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([16]) [INFO] [stdout] 2025-09-01T11:56:45.559985Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:45.560031Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=31 leader=1 [INFO] [stdout] 2025-09-01T11:56:45.560116Z INFO Node{id=17}:connect{target=31}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:45.560182Z INFO Node{id=17}:connect{target=31}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:45.560223Z INFO Node{id=31}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:45.560707Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727805560) } [INFO] [stdout] 2025-09-01T11:56:45.560805Z INFO Node{id=17}:connect{target=31}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:56:45.560826Z INFO Node{id=17}:connect{target=31}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:45.561017Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:45.561145Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 31 } [INFO] [stdout] 2025-09-01T11:56:45.561164Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3, 29, 18, 15, 26, 8, 31] peer=31 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:45.561253Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:45.561305Z INFO Node{id=17}:connect{target=31}:WritePeer{remote=31}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:45.561370Z INFO Node{id=17}:connect{target=31}:ReadPeer{remote=31}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:45.561440Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:45.561535Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:45.773031Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:45.773240Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 6, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:45.773375Z INFO Node{id=6}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9]) [INFO] [stdout] 2025-09-01T11:56:45.790738Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:45.790996Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:45.791092Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:45.791109Z ERROR Node{id=16}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:45.862581Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:45.862640Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:56:45.862751Z INFO Node{id=18}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:45.862863Z INFO Node{id=18}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:45.862918Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:45.863322Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727805862) } [INFO] [stdout] 2025-09-01T11:56:45.863347Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=18 [INFO] [stdout] 2025-09-01T11:56:45.863434Z INFO Node{id=18}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:45.863460Z INFO Node{id=18}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:45.863629Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:45.863664Z INFO Node{id=10}: sharedstate::worker::sync_manager: Broadcasted 17 peers to 16 peers [INFO] [stdout] 2025-09-01T11:56:45.863770Z ERROR Node{id=18}:connect{target=10}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:45.863833Z INFO Node{id=18}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:45.863892Z INFO Node{id=18}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:45.863947Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(10) [INFO] [stdout] 2025-09-01T11:56:45.863981Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:45.864127Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:45.956909Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:45.957130Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:45.957233Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:45.957250Z ERROR Node{id=15}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:46.562579Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:46.562642Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=10 leader=1 [INFO] [stdout] 2025-09-01T11:56:46.562774Z INFO Node{id=17}:connect{target=10}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:46.562866Z INFO Node{id=17}:connect{target=10}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:46.562926Z INFO Node{id=10}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:46.563402Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727806562) } [INFO] [stdout] 2025-09-01T11:56:46.563426Z INFO Node{id=10}: sharedstate::worker::sync_manager: Discovered Peer client_id=17 [INFO] [stdout] 2025-09-01T11:56:46.563511Z INFO Node{id=17}:connect{target=10}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:46.563535Z INFO Node{id=17}:connect{target=10}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:46.563699Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:46.563883Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 10 } [INFO] [stdout] 2025-09-01T11:56:46.563905Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3, 29, 18, 15, 26, 8, 7, 10] peer=10 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:46.564022Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:46.564078Z INFO Node{id=17}:connect{target=10}:WritePeer{remote=10}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:46.564143Z INFO Node{id=17}:connect{target=10}:ReadPeer{remote=10}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:46.564231Z INFO Node{id=10}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:46.564278Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:46.716074Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:46.716359Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 29, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:46.716488Z INFO Node{id=29}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20, 32, 4]) [INFO] [stdout] 2025-09-01T11:56:46.834903Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:46.835152Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 19, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:46.835259Z INFO Node{id=19}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:56:46.865956Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:46.866014Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=27 leader=1 [INFO] [stdout] 2025-09-01T11:56:46.866118Z INFO Node{id=18}:connect{target=27}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:46.866189Z INFO Node{id=18}:connect{target=27}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:46.866238Z INFO Node{id=27}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:46.866655Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727806866) } [INFO] [stdout] 2025-09-01T11:56:46.866737Z INFO Node{id=18}:connect{target=27}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:56:46.866776Z INFO Node{id=18}:connect{target=27}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:46.866956Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:46.867115Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 27 } [INFO] [stdout] 2025-09-01T11:56:46.867133Z ERROR Node{id=18}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 12, 27] peer=27 leader=1 local=18 [INFO] [stdout] 2025-09-01T11:56:46.867215Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:46.867257Z INFO Node{id=18}:connect{target=27}:WritePeer{remote=27}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:46.867316Z INFO Node{id=18}:connect{target=27}:ReadPeer{remote=27}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:46.867427Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:46.867501Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:47.065339Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:47.065592Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 24, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:47.065725Z INFO Node{id=24}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9, 3]) [INFO] [stdout] 2025-09-01T11:56:47.167361Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:47.167598Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 25, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:47.167694Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:47.167712Z ERROR Node{id=25}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:47.557173Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:47.557465Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:47.557590Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([16]) [INFO] [stdout] 2025-09-01T11:56:47.565890Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:47.565951Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=23 leader=1 [INFO] [stdout] 2025-09-01T11:56:47.566066Z INFO Node{id=17}:connect{target=23}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:47.566153Z INFO Node{id=17}:connect{target=23}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:47.566209Z INFO Node{id=23}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:47.566702Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727807566) } [INFO] [stdout] 2025-09-01T11:56:47.566726Z INFO Node{id=23}: sharedstate::worker::sync_manager: Discovered Peer client_id=17 [INFO] [stdout] 2025-09-01T11:56:47.566842Z INFO Node{id=17}:connect{target=23}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:56:47.566868Z INFO Node{id=17}:connect{target=23}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:47.567034Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:47.567191Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 23 } [INFO] [stdout] 2025-09-01T11:56:47.567214Z ERROR Node{id=17}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[23] peer=23 leader=1 local=17 [INFO] [stdout] 2025-09-01T11:56:47.567336Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:47.567388Z INFO Node{id=17}:connect{target=23}:WritePeer{remote=23}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:47.567453Z INFO Node{id=17}:connect{target=23}:ReadPeer{remote=23}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:47.567546Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:47.567581Z INFO Node{id=23}: sharedstate::worker::sync_manager: Broadcasted 17 peers to 16 peers [INFO] [stdout] 2025-09-01T11:56:47.567615Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:47.567864Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 17) [INFO] [stdout] 2025-09-01T11:56:47.567960Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:47.665747Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:47.665983Z INFO Node{id=9}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 3, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:47.666100Z INFO Node{id=3}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 23, 9]) [INFO] [stdout] 2025-09-01T11:56:47.744249Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:47.744523Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 22, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:47.744648Z INFO Node{id=22}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20, 32, 5]) [INFO] [stdout] 2025-09-01T11:56:47.792057Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:47.792308Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:47.792409Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:47.792427Z ERROR Node{id=16}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:47.868414Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:47.868473Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=7 leader=1 [INFO] [stdout] 2025-09-01T11:56:47.868584Z INFO Node{id=18}:connect{target=7}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:47.868673Z INFO Node{id=18}:connect{target=7}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:47.868734Z INFO Node{id=7}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:47.869205Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727807868) } [INFO] [stdout] 2025-09-01T11:56:47.869314Z INFO Node{id=18}:connect{target=7}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:47.869339Z INFO Node{id=18}:connect{target=7}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:47.869501Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:47.869656Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 7 } [INFO] [stdout] 2025-09-01T11:56:47.869677Z ERROR Node{id=18}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[1, 17, 14, 6, 4, 19, 32, 30, 16, 23, 9, 3, 29, 18, 15, 26, 8, 7] peer=7 leader=1 local=18 [INFO] [stdout] 2025-09-01T11:56:47.869798Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:47.869849Z INFO Node{id=18}:connect{target=7}:WritePeer{remote=7}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:47.869913Z INFO Node{id=18}:connect{target=7}:ReadPeer{remote=7}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:47.869988Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:47.870029Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:47.958604Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:47.958899Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:47.959010Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:47.959029Z ERROR Node{id=15}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:48.058076Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:48.058136Z INFO Node{id=1}: sharedstate::worker::sync_manager: Broadcasted 31 peers to 31 peers [INFO] [stdout] 2025-09-01T11:56:48.073585Z INFO sharedstate::testing::fuzzy_test: leader sequence stable [INFO] [stdout] 2025-09-01T11:56:48.100526Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:48.100816Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 26, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:48.100921Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([15]) [INFO] [stdout] 2025-09-01T11:56:48.184879Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:48.185122Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 4, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:48.185229Z INFO Node{id=4}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20, 32]) [INFO] [stdout] 2025-09-01T11:56:48.568831Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:48.568885Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=20 leader=1 [INFO] [stdout] 2025-09-01T11:56:48.568987Z INFO Node{id=17}:connect{target=20}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:48.569058Z INFO Node{id=17}:connect{target=20}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:48.569108Z INFO Node{id=20}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:48.569502Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727808569) } [INFO] [stdout] 2025-09-01T11:56:48.569532Z INFO Node{id=20}: sharedstate::worker::sync_manager: Discovered Peer client_id=17 [INFO] [stdout] 2025-09-01T11:56:48.569604Z INFO Node{id=17}:connect{target=20}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:56:48.569626Z INFO Node{id=17}:connect{target=20}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:48.569749Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:48.569805Z INFO Node{id=20}: sharedstate::worker::sync_manager: Broadcasted 15 peers to 14 peers [INFO] [stdout] 2025-09-01T11:56:48.569952Z ERROR Node{id=17}:connect{target=20}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:48.570016Z INFO Node{id=30}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 15) [INFO] [stdout] 2025-09-01T11:56:48.570057Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 15) [INFO] [stdout] 2025-09-01T11:56:48.570093Z INFO Node{id=17}:connect{target=20}:WritePeer{remote=20}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:48.570156Z INFO Node{id=17}:connect{target=20}:ReadPeer{remote=20}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:48.570223Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(20) [INFO] [stdout] 2025-09-01T11:56:48.570264Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:48.570502Z INFO Node{id=20}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:48.697893Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:48.698135Z INFO Node{id=26}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 8, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:48.698241Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([26]) [INFO] [stdout] 2025-09-01T11:56:48.871710Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:48.871785Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=16 leader=1 [INFO] [stdout] 2025-09-01T11:56:48.871995Z INFO Node{id=18}:connect{target=16}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:48.872084Z INFO Node{id=18}:connect{target=16}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:48.872187Z INFO Node{id=16}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:48.872697Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727808872) } [INFO] [stdout] 2025-09-01T11:56:48.872817Z INFO Node{id=18}:connect{target=16}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:56:48.872842Z INFO Node{id=18}:connect{target=16}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:48.872991Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:48.873118Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::UpdatedFollow { remote: 16 } [INFO] [stdout] 2025-09-01T11:56:48.873136Z ERROR Node{id=18}: sharedstate::worker::sync_manager: peer we're following has invalid leader path path=[16] peer=16 leader=1 local=18 [INFO] [stdout] 2025-09-01T11:56:48.873213Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(RejectPeer) [INFO] [stdout] 2025-09-01T11:56:48.873254Z INFO Node{id=18}:connect{target=16}:WritePeer{remote=16}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:48.873315Z INFO Node{id=18}:connect{target=16}:ReadPeer{remote=16}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:48.873411Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: SendMePeers } [INFO] [stdout] 2025-09-01T11:56:48.873458Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:48.916434Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:48.916678Z INFO Node{id=12}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 27, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:48.916812Z INFO Node{id=27}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 19, 32, 30, 28, 13, 29, 18, 25, 17, 16, 12]) [INFO] [stdout] 2025-09-01T11:56:48.916859Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:48.917051Z INFO Node{id=1}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 13, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:48.917137Z INFO Node{id=13}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1]) [INFO] [stdout] 2025-09-01T11:56:49.138201Z INFO Node{id=25}: sharedstate::worker::sync_manager: Next Event: Timer(RequireVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:49.138371Z INFO Node{id=25}:connect{target=18}:ReadPeer{remote=18}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:49.138635Z ERROR ReadClient{remote=25}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:49.499551Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:49.499824Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 31, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:49.499935Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([8]) [INFO] [stdout] 2025-09-01T11:56:49.558415Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:49.558726Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 23, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:49.558862Z INFO Node{id=23}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([16]) [INFO] [stdout] 2025-09-01T11:56:49.571254Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:49.571323Z INFO Node{id=17}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=21 leader=1 [INFO] [stdout] 2025-09-01T11:56:49.571442Z INFO Node{id=17}:connect{target=21}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:49.571561Z INFO Node{id=17}:connect{target=21}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:49.571623Z INFO Node{id=21}: sharedstate::worker::sync_manager: got new client remote=17 [INFO] [stdout] 2025-09-01T11:56:49.572119Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: Ping(1756727809571) } [INFO] [stdout] 2025-09-01T11:56:49.572153Z INFO Node{id=21}: sharedstate::worker::sync_manager: Discovered Peer client_id=17 [INFO] [stdout] 2025-09-01T11:56:49.572240Z INFO Node{id=17}:connect{target=21}: sharedstate::worker::sync_manager: got pong from peer latency=1 [INFO] [stdout] 2025-09-01T11:56:49.572262Z INFO Node{id=17}:connect{target=21}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:49.572416Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:49.572449Z INFO Node{id=21}: sharedstate::worker::sync_manager: Broadcasted 15 peers to 14 peers [INFO] [stdout] 2025-09-01T11:56:49.572523Z ERROR Node{id=17}:connect{target=21}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:49.572578Z INFO Node{id=17}:connect{target=21}:WritePeer{remote=21}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:49.572643Z INFO Node{id=17}:connect{target=21}:ReadPeer{remote=21}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:49.572698Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(21) [INFO] [stdout] 2025-09-01T11:56:49.572730Z ERROR ReadClient{remote=17}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:49.572883Z INFO Node{id=21}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 17, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:49.794916Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:49.795476Z INFO Node{id=17}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 16, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:49.795585Z INFO Node{id=16}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:49.795606Z ERROR Node{id=16}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:49.874896Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: Timer(ConnectToPeer) [INFO] [stdout] 2025-09-01T11:56:49.874959Z INFO Node{id=18}: sharedstate::worker::sync_manager: found connect target option_count=30 peers=32 target=8 leader=1 [INFO] [stdout] 2025-09-01T11:56:49.875121Z INFO Node{id=18}:connect{target=8}: sharedstate::worker::sync_manager: connected to target [INFO] [stdout] 2025-09-01T11:56:49.875216Z INFO Node{id=18}:connect{target=8}: sharedstate::worker::sync_manager: send ping to peer, waiting for pong [INFO] [stdout] 2025-09-01T11:56:49.875340Z INFO Node{id=8}: sharedstate::worker::sync_manager: got new client remote=18 [INFO] [stdout] 2025-09-01T11:56:49.875890Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: Ping(1756727809875) } [INFO] [stdout] 2025-09-01T11:56:49.875913Z INFO Node{id=8}: sharedstate::worker::sync_manager: Discovered Peer client_id=18 [INFO] [stdout] 2025-09-01T11:56:49.875987Z INFO Node{id=18}:connect{target=8}: sharedstate::worker::sync_manager: got pong from peer latency=0 [INFO] [stdout] 2025-09-01T11:56:49.876009Z INFO Node{id=18}:connect{target=8}: sharedstate::worker::sync_manager: request leader path from peer [INFO] [stdout] 2025-09-01T11:56:49.876151Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: Timer(BroadcastPeers) [INFO] [stdout] 2025-09-01T11:56:49.876186Z INFO Node{id=8}: sharedstate::worker::sync_manager: Broadcasted 13 peers to 12 peers [INFO] [stdout] 2025-09-01T11:56:49.876359Z INFO Node{id=7}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 13) [INFO] [stdout] 2025-09-01T11:56:49.876411Z INFO Node{id=31}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::Peers(count: 13) [INFO] [stdout] 2025-09-01T11:56:49.876439Z ERROR Node{id=18}:connect{target=8}: sharedstate::worker::sync_manager: got unexpected leader path response [INFO] [stdout] 2025-09-01T11:56:49.876486Z INFO Node{id=18}:connect{target=8}:WritePeer{remote=8}: sharedstate::net::message_channels: input closed, ending write [INFO] [stdout] 2025-09-01T11:56:49.876541Z INFO Node{id=18}:connect{target=8}:ReadPeer{remote=8}: sharedstate::net::message_channels: output closed, stopping read [INFO] [stdout] 2025-09-01T11:56:49.876599Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ConnectionUpdate::ConnectFailed(8) [INFO] [stdout] 2025-09-01T11:56:49.876632Z ERROR ReadClient{remote=18}: sharedstate::net::message_channels: failed to read from network error=SizeReadError(Custom { kind: UnexpectedEof, error: "early eof" }) [INFO] [stdout] 2025-09-01T11:56:49.876859Z INFO Node{id=8}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 18, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:49.960846Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:49.961094Z INFO Node{id=18}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 15, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:49.961210Z INFO Node{id=15}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([]) [INFO] [stdout] 2025-09-01T11:56:49.961231Z ERROR Node{id=15}: sharedstate::worker::sync_manager: got empty leader path from peer [INFO] [stdout] 2025-09-01T11:56:50.026747Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: Timer(SendVerifyLeader) [INFO] [stdout] 2025-09-01T11:56:50.027078Z INFO Node{id=32}: sharedstate::worker::sync_manager: Next Event: ClientMessage { client: 5, message: ShareLeaderPath } [INFO] [stdout] 2025-09-01T11:56:50.027218Z INFO Node{id=5}: sharedstate::worker::sync_manager: Next Event: LeaderMessage::LeaderPath([1, 20, 32]) [INFO] [stdout] [INFO] [stdout] thread 'testing::fuzzy_test::fuzzy_test' (21) panicked at src/testing/fuzzy_test.rs:173:13: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: TestState { sequence: 6817, numbers: [2196, 489, 0, 0, 34, 43, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 1119, 1119, 870, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183] } [INFO] [stdout] right: TestState { sequence: 41215, numbers: [9394, 2532, 25, 302, 769, 854, 8, 259, 248, 220, 1378, 98, 537, 2239, 114, 167, 3855, 3828, 3383, 3292, 27, 246, 354, 394, 0, 81, 245, 435, 979, 2119, 172, 2661] } [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x622e8134e652 - std::backtrace_rs::backtrace::libunwind::trace::h589a96ef7638b383 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x622e8134e652 - std::backtrace_rs::backtrace::trace_unsynchronized::h23a30a7548c40de4 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x622e8134e652 - std::sys::backtrace::_print_fmt::h87dabd6535c8c07a [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x622e8134e652 - ::fmt::hffd20ad4e5eca8ab [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x622e8136007f - core::fmt::rt::Argument::fmt::h75c83e3fd5ce2419 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x622e8136007f - core::fmt::write::h6d9d3a7cfd7b84f5 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/fmt/mod.rs:1468:25 [INFO] [stdout] 6: 0x622e8131b7e3 - std::io::default_write_fmt::he11a713685e2464d [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x622e8131b7e3 - std::io::Write::write_fmt::h9d08f7e050bd2612 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x622e81327762 - std::sys::backtrace::BacktraceLock::print::hb28797143397220e [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x622e8132c47f - std::panicking::default_hook::{{closure}}::h7555113b62983743 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x622e8132c311 - std::panicking::default_hook::h2714b564abe8d914 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x622e811fb94e - as core::ops::function::Fn>::call::h0e9de8cceb22406e [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 12: 0x622e811fb94e - test::test_main_with_exit_callback::{{closure}}::h8125bd66cf4739d0 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x622e8132cbce - as core::ops::function::Fn>::call::h1fd0a0802eaec16b [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 14: 0x622e8132cbce - std::panicking::panic_with_hook::h190dc82263685ec5 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x622e8132c8ea - std::panicking::panic_handler::{{closure}}::h8b036a1b366643ac [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:707:13 [INFO] [stdout] 16: 0x622e81327899 - std::sys::backtrace::__rust_end_short_backtrace::h45affcfc0c830da8 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x622e8130f9ed - __rustc[9b67c8562bba447b]::rust_begin_unwind [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x622e81368000 - core::panicking::panic_fmt::hc084f85b1e76c16d [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/panicking.rs:75:14 [INFO] [stdout] 19: 0x622e81367db3 - core::panicking::assert_failed_inner::h1215dc8d5a90adff [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/panicking.rs:439:17 [INFO] [stdout] 20: 0x622e8106d94f - core::panicking::assert_failed::hb914cd0a21113cea [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/panicking.rs:394:5 [INFO] [stdout] 21: 0x622e81037811 - sharedstate::testing::fuzzy_test::_fuzzy_test::{{closure}}::h1651148d0e7a67b7 [INFO] [stdout] at /opt/rustwide/workdir/src/testing/fuzzy_test.rs:173:13 [INFO] [stdout] 22: 0x622e81033e2c - sharedstate::testing::fuzzy_test::fuzzy_test::{{closure}}::h6bab00edd3c94014 [INFO] [stdout] at /opt/rustwide/workdir/src/testing/fuzzy_test.rs:14:19 [INFO] [stdout] 23: 0x622e8117aa32 - as core::future::future::Future>::poll::h434771155369610d [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/future/future.rs:133:9 [INFO] [stdout] 24: 0x622e8117acad - as core::future::future::Future>::poll::hbe9d51c1c46c50d2 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/future/future.rs:133:9 [INFO] [stdout] 25: 0x622e8112dd2d - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}}::h861f04b604752326 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:733:70 [INFO] [stdout] 26: 0x622e8112dacb - tokio::task::coop::with_budget::h3a66d55b903ed416 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 27: 0x622e8112dacb - tokio::task::coop::budget::h4d67359ec161d2eb [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 28: 0x622e8112dacb - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::hb36547f4cec533aa [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:733:25 [INFO] [stdout] 29: 0x622e8112b110 - tokio::runtime::scheduler::current_thread::Context::enter::ha6520ea8c29a0fb2 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 30: 0x622e8112c70d - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::h20073c87d705c8c6 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:732:44 [INFO] [stdout] 31: 0x622e8112c3b4 - tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}}::h1e82ec09a1872b67 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:820:68 [INFO] [stdout] 32: 0x622e810f6d7b - tokio::runtime::context::scoped::Scoped::set::h369e9b2b649b1006 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 33: 0x622e810f6739 - tokio::runtime::context::set_scheduler::{{closure}}::h979451ff4983ce38 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/context.rs:176:38 [INFO] [stdout] 34: 0x622e8110ced2 - std::thread::local::LocalKey::try_with::hcdd93df4c96bae24 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 35: 0x622e811088de - std::thread::local::LocalKey::with::h8bfb936e63e66b2a [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 36: 0x622e810f66ad - tokio::runtime::context::set_scheduler::h213924d01cfe4b18 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/context.rs:176:17 [INFO] [stdout] 37: 0x622e8112bd40 - tokio::runtime::scheduler::current_thread::CoreGuard::enter::h82b8ade9c9fbebfd [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:820:27 [INFO] [stdout] 38: 0x622e8112c453 - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::hb4512cd83fef745d [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:720:24 [INFO] [stdout] 39: 0x622e811231e0 - tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}}::h1cea886e3d7f7a52 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 40: 0x622e81122420 - tokio::runtime::context::runtime::enter_runtime::h8e61f20f6a0c6d28 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 41: 0x622e81122fa1 - tokio::runtime::scheduler::current_thread::CurrentThread::block_on::h72812b79ba1b2125 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 42: 0x622e811736f4 - tokio::runtime::runtime::Runtime::block_on_inner::h58e48cc6e8071fa9 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 43: 0x622e811739c7 - tokio::runtime::runtime::Runtime::block_on::h68b292f20b827fe1 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.45.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 44: 0x622e81033c73 - sharedstate::testing::fuzzy_test::fuzzy_test::ha5f6e80dca72e690 [INFO] [stdout] at /opt/rustwide/workdir/src/testing/fuzzy_test.rs:14:24 [INFO] [stdout] 45: 0x622e81033ec7 - sharedstate::testing::fuzzy_test::fuzzy_test::{{closure}}::hf6ff91f68503d6f8 [INFO] [stdout] at /opt/rustwide/workdir/src/testing/fuzzy_test.rs:13:22 [INFO] [stdout] 46: 0x622e80fe9fd6 - core::ops::function::FnOnce::call_once::hab33f53ba8bbd495 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/ops/function.rs:253:5 [INFO] [stdout] 47: 0x622e811fb70b - core::ops::function::FnOnce::call_once::hdbf42be2a49fb464 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/ops/function.rs:253:5 [INFO] [stdout] 48: 0x622e811fb70b - test::__rust_begin_short_backtrace::h8ae08814d38cb356 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/test/src/lib.rs:663:18 [INFO] [stdout] 49: 0x622e812111f5 - test::run_test_in_process::{{closure}}::h54574ee4f414d690 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/test/src/lib.rs:686:74 [INFO] [stdout] 50: 0x622e812111f5 - as core::ops::function::FnOnce<()>>::call_once::h01a0b7e7aad1a501 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/panic/unwind_safe.rs:272:9 [INFO] [stdout] 51: 0x622e812111f5 - std::panicking::catch_unwind::do_call::h23820e817c5612cd [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:590:40 [INFO] [stdout] 52: 0x622e812111f5 - std::panicking::catch_unwind::h89c73a1febce5587 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:553:19 [INFO] [stdout] 53: 0x622e812111f5 - std::panic::catch_unwind::h6f55caf410861914 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panic.rs:359:14 [INFO] [stdout] 54: 0x622e812111f5 - test::run_test_in_process::h4db851878e5d3983 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/test/src/lib.rs:686:27 [INFO] [stdout] 55: 0x622e812111f5 - test::run_test::{{closure}}::h57e8c5acaceacd24 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/test/src/lib.rs:607:43 [INFO] [stdout] 56: 0x622e811e7be4 - test::run_test::{{closure}}::he775ccc3a9d3b97e [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/test/src/lib.rs:637:41 [INFO] [stdout] 57: 0x622e811e7be4 - std::sys::backtrace::__rust_begin_short_backtrace::hcd94d85d7765225a [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 58: 0x622e811eb42a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::ha814de8393f07830 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 59: 0x622e811eb42a - as core::ops::function::FnOnce<()>>::call_once::h8deb00af0abdeaf1 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/panic/unwind_safe.rs:272:9 [INFO] [stdout] 60: 0x622e811eb42a - std::panicking::catch_unwind::do_call::h1917979b0c279ea2 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:590:40 [INFO] [stdout] 61: 0x622e811eb42a - std::panicking::catch_unwind::h791ad91fee6ce34e [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panicking.rs:553:19 [INFO] [stdout] 62: 0x622e811eb42a - std::panic::catch_unwind::h982b8f40ed9c3cce [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/panic.rs:359:14 [INFO] [stdout] 63: 0x622e811eb42a - std::thread::Builder::spawn_unchecked_::{{closure}}::h545e54700c945db4 [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 64: 0x622e811eb42a - core::ops::function::FnOnce::call_once{{vtable.shim}}::hb427481a5227d78a [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/core/src/ops/function.rs:253:5 [INFO] [stdout] 65: 0x622e81321b1f - as core::ops::function::FnOnce>::call_once::h1d452951a81e9d0a [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 66: 0x622e81321b1f - std::sys::pal::unix::thread::Thread::new::thread_start::h29c84e109630689a [INFO] [stdout] at /rustc/b83b707f97d809763b7861afa7638871f3339a33/library/std/src/sys/pal/unix/thread.rs:118:17 [INFO] [stdout] 67: 0x7bfdf2a7caa4 - [INFO] [stdout] 68: 0x7bfdf2b09a34 - clone [INFO] [stdout] 69: 0x0 - [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] testing::fuzzy_test::fuzzy_test [INFO] [stdout] testing::state_tests::slow_state_deadlock_test [INFO] [stdout] [INFO] [stdout] test result: FAILED. 14 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out; finished in 123.70s [INFO] [stdout] [INFO] [stderr] error: test failed, to rerun pass `--lib` [INFO] running `Command { std: "docker" "inspect" "593d2bde7e94d6db5f1f22cb023b34dfa28ed30f62328215421da1d66ab0e72f", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "593d2bde7e94d6db5f1f22cb023b34dfa28ed30f62328215421da1d66ab0e72f", kill_on_drop: false }` [INFO] [stdout] 593d2bde7e94d6db5f1f22cb023b34dfa28ed30f62328215421da1d66ab0e72f