[INFO] fetching crate tender 0.1.1... [INFO] testing tender-0.1.1 against master#159ba8a92c9e2fa4121f106176309521f4af87e9+rustflags=-Copt-level=3 for pr-107051 [INFO] extracting crate tender 0.1.1 into /workspace/builds/worker-5-tc1/source [INFO] validating manifest of crates.io crate tender 0.1.1 on toolchain 159ba8a92c9e2fa4121f106176309521f4af87e9 [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+159ba8a92c9e2fa4121f106176309521f4af87e9" "metadata" "--manifest-path" "Cargo.toml" "--no-deps", kill_on_drop: false }` [INFO] started tweaking crates.io crate tender 0.1.1 [INFO] finished tweaking crates.io crate tender 0.1.1 [INFO] tweaked toml for crates.io crate tender 0.1.1 written to /workspace/builds/worker-5-tc1/source/Cargo.toml [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" __CARGO_TEST_CHANNEL_OVERRIDE_DO_NOT_USE_THIS="nightly" "/workspace/cargo-home/bin/cargo" "+159ba8a92c9e2fa4121f106176309521f4af87e9" "generate-lockfile" "--manifest-path" "Cargo.toml" "-Zno-index-update", kill_on_drop: false }` [INFO] [stderr] Blocking waiting for file lock on package cache [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+159ba8a92c9e2fa4121f106176309521f4af87e9" "fetch" "--manifest-path" "Cargo.toml", kill_on_drop: false }` [INFO] [stderr] Downloading crates ... [INFO] [stderr] Downloaded watch v0.2.2 [INFO] [stderr] Downloaded sync-wait-group v0.1.1 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-5-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-5-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7c93e6f9a94679f34221d1005849d4c83862f85b14a264fa0883942f4d043a66" "/opt/rustwide/cargo-home/bin/cargo" "+159ba8a92c9e2fa4121f106176309521f4af87e9" "metadata" "--no-deps" "--format-version=1", kill_on_drop: false }` [INFO] [stdout] 98c00d721adde8c730dacf829434a96ef98508b4e2a2fc31a76991070e622674 [INFO] running `Command { std: "docker" "start" "-a" "98c00d721adde8c730dacf829434a96ef98508b4e2a2fc31a76991070e622674", kill_on_drop: false }` [INFO] running `Command { std: "docker" "inspect" "98c00d721adde8c730dacf829434a96ef98508b4e2a2fc31a76991070e622674", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "98c00d721adde8c730dacf829434a96ef98508b4e2a2fc31a76991070e622674", kill_on_drop: false }` [INFO] [stdout] 98c00d721adde8c730dacf829434a96ef98508b4e2a2fc31a76991070e622674 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-5-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-5-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid -Copt-level=3" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7c93e6f9a94679f34221d1005849d4c83862f85b14a264fa0883942f4d043a66" "/opt/rustwide/cargo-home/bin/cargo" "+159ba8a92c9e2fa4121f106176309521f4af87e9" "build" "--frozen" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] 17115d37f9e60673349d469bad5f33ff2cbe111e3bd93969f67a04782a24351d [INFO] running `Command { std: "docker" "start" "-a" "17115d37f9e60673349d469bad5f33ff2cbe111e3bd93969f67a04782a24351d", kill_on_drop: false }` [INFO] [stderr] Compiling byteorder v1.4.3 [INFO] [stderr] Compiling sync-wait-group v0.1.1 [INFO] [stderr] Compiling watch v0.2.2 [INFO] [stderr] Compiling fxhash v0.2.1 [INFO] [stderr] Compiling tender v0.1.1 (/opt/rustwide/workdir) [INFO] [stderr] Finished dev [unoptimized + debuginfo] target(s) in 1.62s [INFO] running `Command { std: "docker" "inspect" "17115d37f9e60673349d469bad5f33ff2cbe111e3bd93969f67a04782a24351d", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "17115d37f9e60673349d469bad5f33ff2cbe111e3bd93969f67a04782a24351d", kill_on_drop: false }` [INFO] [stdout] 17115d37f9e60673349d469bad5f33ff2cbe111e3bd93969f67a04782a24351d [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-5-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-5-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid -Copt-level=3" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7c93e6f9a94679f34221d1005849d4c83862f85b14a264fa0883942f4d043a66" "/opt/rustwide/cargo-home/bin/cargo" "+159ba8a92c9e2fa4121f106176309521f4af87e9" "test" "--frozen" "--no-run" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] 344c4dd547ed5c56f365b005cabf2642e595470a5aae1683743c051d92fd8883 [INFO] running `Command { std: "docker" "start" "-a" "344c4dd547ed5c56f365b005cabf2642e595470a5aae1683743c051d92fd8883", kill_on_drop: false }` [INFO] [stderr] Compiling regex-syntax v0.6.28 [INFO] [stderr] Compiling termcolor v1.2.0 [INFO] [stderr] Compiling tender v0.1.1 (/opt/rustwide/workdir) [INFO] [stderr] Compiling regex v1.7.1 [INFO] [stderr] Compiling env_logger v0.9.3 [INFO] [stderr] Finished test [unoptimized + debuginfo] target(s) in 31.74s [INFO] running `Command { std: "docker" "inspect" "344c4dd547ed5c56f365b005cabf2642e595470a5aae1683743c051d92fd8883", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "344c4dd547ed5c56f365b005cabf2642e595470a5aae1683743c051d92fd8883", kill_on_drop: false }` [INFO] [stdout] 344c4dd547ed5c56f365b005cabf2642e595470a5aae1683743c051d92fd8883 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-5-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-5-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid -Copt-level=3" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7c93e6f9a94679f34221d1005849d4c83862f85b14a264fa0883942f4d043a66" "/opt/rustwide/cargo-home/bin/cargo" "+159ba8a92c9e2fa4121f106176309521f4af87e9" "test" "--frozen", kill_on_drop: false }` [INFO] [stdout] e18c9815d3add59ac3788f1d03171b71705971da52fd9743bb50acd3ed80c6c6 [INFO] running `Command { std: "docker" "start" "-a" "e18c9815d3add59ac3788f1d03171b71705971da52fd9743bb50acd3ed80c6c6", kill_on_drop: false }` [INFO] [stderr] Finished test [unoptimized + debuginfo] target(s) in 0.07s [INFO] [stderr] Running unittests src/lib.rs (/opt/rustwide/target/debug/deps/tender-5cdc797f653071b0) [INFO] [stdout] [INFO] [stdout] running 0 tests [INFO] [stdout] [INFO] [stdout] test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s [INFO] [stdout] [INFO] [stderr] Running tests/fixtures.rs (/opt/rustwide/target/debug/deps/fixtures-ccadb762ada5e302) [INFO] [stdout] [INFO] [stderr] Running tests/group_election.rs (/opt/rustwide/target/debug/deps/group_election-7a0a68daea8c2461) [INFO] [stdout] running 0 tests [INFO] [stdout] [INFO] [stdout] test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] running 6 tests [INFO] [stdout] test test_election_with_initial_candidate ... ok [INFO] [stdout] test test_step_up_and_down ... FAILED [INFO] [stdout] test test_move_leader ... ok [INFO] [stdout] test test_election_with_quorum ... ok [INFO] [stdout] test test_election_with_observer ... ok [INFO] [stdout] test test_election ... ok [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] [INFO] [stdout] ---- test_step_up_and_down stdout ---- [INFO] [stdout] [2023-01-21T11:33:25Z INFO tender::core] [NodeId(1000-1001)][Term(0)] start election main thread [INFO] [stdout] [2023-01-21T11:33:25Z INFO tender::core::startup] [NodeId(1000-1001)][Term(0)] start the startup loop [INFO] [stdout] [2023-01-21T11:33:25Z INFO group_election::fixtures] [Node(NodeId(1000-1001))] an event has happened: Startup [INFO] [stdout] [2023-01-21T11:33:25Z INFO tender::core] [NodeId(1000-1003)][Term(0)] start election main thread [INFO] [stdout] [2023-01-21T11:33:25Z INFO tender::core::startup] [NodeId(1000-1003)][Term(0)] start the startup loop [INFO] [stdout] [2023-01-21T11:33:25Z INFO tender::core] [NodeId(1000-1002)][Term(0)] start election main thread [INFO] [stdout] [2023-01-21T11:33:25Z INFO group_election::fixtures] [Node(NodeId(1000-1003))] an event has happened: Startup [INFO] [stdout] [2023-01-21T11:33:25Z INFO tender::core::startup] [NodeId(1000-1002)][Term(0)] start the startup loop [INFO] [stdout] [2023-01-21T11:33:25Z INFO group_election::fixtures] [Node(NodeId(1000-1002))] an event has happened: Startup [INFO] [stdout] [2023-01-21T11:33:26Z INFO tender::core::startup] [NodeId(1000-1001)][Term(1)] this node is forced to be leader [INFO] [stdout] [2023-01-21T11:33:26Z INFO tender::core::leader] [NodeId(1000-1001)][Term(1)] start the leader loop [INFO] [stdout] [2023-01-21T11:33:26Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1003)) [INFO] [stdout] [2023-01-21T11:33:26Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1002)) [INFO] [stdout] [2023-01-21T11:33:26Z INFO tender::core::startup] [NodeId(1000-1002)][Term(0)] this node is initialized with 3 members, and transit to follower [INFO] [stdout] [2023-01-21T11:33:26Z INFO tender::core::follower] [NodeId(1000-1002)][Term(0)] start the follower loop [INFO] [stdout] [2023-01-21T11:33:26Z TRACE tender::core::follower] [NodeId(1000-1002)][Term(0)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1002 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:26Z INFO tender::core] [NodeId(1000-1002)][Term(1)] change leader to NodeId(1000-1001) [INFO] [stdout] [2023-01-21T11:33:26Z INFO tender::core::startup] [NodeId(1000-1003)][Term(0)] this node is initialized with 3 members, and transit to follower [INFO] [stdout] [2023-01-21T11:33:26Z INFO tender::core::follower] [NodeId(1000-1003)][Term(0)] start the follower loop [INFO] [stdout] [2023-01-21T11:33:26Z INFO group_election::fixtures] [Node(NodeId(1000-1002))] an event has happened: ChangeLeader(NodeId { group_id: 1000, group_node_id: 1001 }) [INFO] [stdout] [2023-01-21T11:33:26Z INFO group_election::fixtures] [Node(NodeId(1000-1002))] an event has happened: TransitToFollower { term: 0, prev_state: Startup, caused_by_step_down: false } [INFO] [stdout] [2023-01-21T11:33:26Z DEBUG tender::core::follower] [NodeId(1000-1002)][Term(1)] event (ChangeLeader(NodeId { group_id: 1000, group_node_id: 1001 })) in term 1 is handled [INFO] [stdout] [2023-01-21T11:33:26Z INFO group_election::fixtures] [Node(NodeId(1000-1001))] an event has happened: TransitToLeader { term: 1, caused_by_step_up: false } [INFO] [stdout] [2023-01-21T11:33:26Z TRACE tender::core::follower] [NodeId(1000-1003)][Term(0)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1003 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:26Z INFO tender::core] [NodeId(1000-1003)][Term(1)] change leader to NodeId(1000-1001) [INFO] [stdout] [2023-01-21T11:33:26Z INFO group_election::fixtures] [Node(NodeId(1000-1003))] an event has happened: TransitToFollower { term: 0, prev_state: Startup, caused_by_step_down: false } [INFO] [stdout] [2023-01-21T11:33:26Z INFO group_election::fixtures] [Node(NodeId(1000-1003))] an event has happened: ChangeLeader(NodeId { group_id: 1000, group_node_id: 1001 }) [INFO] [stdout] [2023-01-21T11:33:26Z DEBUG tender::core::follower] [NodeId(1000-1003)][Term(1)] event (ChangeLeader(NodeId { group_id: 1000, group_node_id: 1001 })) in term 1 is handled [INFO] [stdout] [2023-01-21T11:33:26Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1003)) [INFO] [stdout] [2023-01-21T11:33:26Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1002)) [INFO] [stdout] [2023-01-21T11:33:26Z TRACE tender::core::follower] [NodeId(1000-1002)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1002 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:26Z TRACE tender::core::follower] [NodeId(1000-1003)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1003 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:27Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1003)) [INFO] [stdout] [2023-01-21T11:33:27Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1002)) [INFO] [stdout] [2023-01-21T11:33:27Z TRACE tender::core::follower] [NodeId(1000-1003)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1003 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:27Z TRACE tender::core::follower] [NodeId(1000-1002)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1002 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:27Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1003)) [INFO] [stdout] [2023-01-21T11:33:27Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1002)) [INFO] [stdout] [2023-01-21T11:33:27Z TRACE tender::core::follower] [NodeId(1000-1002)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1002 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:27Z TRACE tender::core::follower] [NodeId(1000-1003)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1003 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:28Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1003)) [INFO] [stdout] [2023-01-21T11:33:28Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1002)) [INFO] [stdout] [2023-01-21T11:33:28Z TRACE tender::core::follower] [NodeId(1000-1003)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1003 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:28Z TRACE tender::core::follower] [NodeId(1000-1002)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1002 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:28Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1003)) [INFO] [stdout] [2023-01-21T11:33:28Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1002)) [INFO] [stdout] [2023-01-21T11:33:28Z TRACE tender::core::follower] [NodeId(1000-1003)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1003 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:28Z TRACE tender::core::follower] [NodeId(1000-1002)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1002 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:29Z INFO group_election] --- node NodeId(1000-1001) step down to follower [INFO] [stdout] [2023-01-21T11:33:29Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1003)) [INFO] [stdout] [2023-01-21T11:33:29Z TRACE tender::core::leader] [NodeId(1000-1001)][Term(1)] send heartbeat to node(NodeId(1000-1002)) [INFO] [stdout] [2023-01-21T11:33:29Z DEBUG tender::core::leader] [NodeId(1000-1001)][Term(1)] step down to follower [INFO] [stdout] [2023-01-21T11:33:29Z INFO tender::core::follower] [NodeId(1000-1001)][Term(1)] start the follower loop [INFO] [stdout] [2023-01-21T11:33:29Z INFO group_election] --- node NodeId(1000-1002) step up to leader [INFO] [stdout] [2023-01-21T11:33:29Z INFO group_election::fixtures] [Node(NodeId(1000-1001))] an event has happened: TransitToFollower { term: 1, prev_state: Leader, caused_by_step_down: true } [INFO] [stdout] [2023-01-21T11:33:29Z TRACE tender::core::follower] [NodeId(1000-1003)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1003 }, leader_id: NodeId { group_id: 1000, group_node_id: 1001 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:29Z DEBUG tender::core::follower] [NodeId(1000-1002)][Term(1)] step up to leader [INFO] [stdout] [2023-01-21T11:33:29Z INFO tender::core::leader] [NodeId(1000-1002)][Term(1)] start the leader loop [INFO] [stdout] [2023-01-21T11:33:29Z TRACE tender::core::leader] [NodeId(1000-1002)][Term(1)] send heartbeat to node(NodeId(1000-1001)) [INFO] [stdout] [2023-01-21T11:33:29Z TRACE tender::core::leader] [NodeId(1000-1002)][Term(1)] send heartbeat to node(NodeId(1000-1003)) [INFO] [stdout] [2023-01-21T11:33:29Z INFO tender::core] [NodeId(1000-1002)][Term(1)] change leader from NodeId(1000-1002) to NodeId(1000-1001) [INFO] [stdout] [2023-01-21T11:33:29Z INFO tender::core] [NodeId(1000-1002)][Term(1)] received valid heartbeat in Leader state, so transit to follower [INFO] [stdout] [2023-01-21T11:33:29Z INFO tender::core::follower] [NodeId(1000-1002)][Term(1)] start the follower loop [INFO] [stdout] [2023-01-21T11:33:29Z INFO group_election::fixtures] [Node(NodeId(1000-1002))] an event has happened: TransitToLeader { term: 1, caused_by_step_up: true } [INFO] [stdout] [2023-01-21T11:33:29Z INFO group_election::fixtures] [Node(NodeId(1000-1002))] an event has happened: ChangeLeader(NodeId { group_id: 1000, group_node_id: 1001 }) [INFO] [stdout] [2023-01-21T11:33:29Z INFO group_election::fixtures] [Node(NodeId(1000-1002))] an event has happened: TransitToFollower { term: 1, prev_state: Leader, caused_by_step_down: false } [INFO] [stdout] [2023-01-21T11:33:29Z TRACE tender::core::follower] [NodeId(1000-1001)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1001 }, leader_id: NodeId { group_id: 1000, group_node_id: 1002 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:29Z INFO tender::core] [NodeId(1000-1001)][Term(1)] change leader from NodeId(1000-1001) to NodeId(1000-1002) [INFO] [stdout] [2023-01-21T11:33:29Z TRACE tender::core::follower] [NodeId(1000-1003)][Term(1)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1003 }, leader_id: NodeId { group_id: 1000, group_node_id: 1002 }, term: 1 } [INFO] [stdout] [2023-01-21T11:33:29Z INFO tender::core] [NodeId(1000-1003)][Term(1)] change leader from NodeId(1000-1001) to NodeId(1000-1002) [INFO] [stdout] [2023-01-21T11:33:29Z DEBUG tender::core::follower] [NodeId(1000-1002)][Term(1)] event (TransitToLeader { term: 1, caused_by_step_up: true }) in term 1 is handled [INFO] [stdout] [2023-01-21T11:33:29Z DEBUG tender::core::follower] [NodeId(1000-1002)][Term(1)] event (ChangeLeader(NodeId { group_id: 1000, group_node_id: 1001 })) in term 1 is handled [INFO] [stdout] [2023-01-21T11:33:29Z INFO group_election::fixtures] [Node(NodeId(1000-1001))] an event has happened: ChangeLeader(NodeId { group_id: 1000, group_node_id: 1002 }) [INFO] [stdout] [2023-01-21T11:33:29Z INFO group_election::fixtures] [Node(NodeId(1000-1003))] an event has happened: ChangeLeader(NodeId { group_id: 1000, group_node_id: 1002 }) [INFO] [stdout] [2023-01-21T11:33:29Z DEBUG tender::core::follower] [NodeId(1000-1001)][Term(1)] event (ChangeLeader(NodeId { group_id: 1000, group_node_id: 1002 })) in term 1 is handled [INFO] [stdout] [2023-01-21T11:33:29Z DEBUG tender::core::follower] [NodeId(1000-1003)][Term(1)] event (ChangeLeader(NodeId { group_id: 1000, group_node_id: 1002 })) in term 1 is handled [INFO] [stdout] [2023-01-21T11:33:30Z INFO tender::core::follower] [NodeId(1000-1003)][Term(1)] an election timeout is hit, need to transit to pre-candidate [INFO] [stdout] [2023-01-21T11:33:30Z INFO tender::core::candidate] [NodeId(1000-1003)][Term(1)] start the pre-candidate loop [INFO] [stdout] [2023-01-21T11:33:30Z DEBUG tender::core::candidate] [NodeId(1000-1003)][Term(1)] quorum is Major, votes granted(1/2) [INFO] [stdout] [2023-01-21T11:33:30Z DEBUG tender::core::candidate] [NodeId(1000-1003)][Term(1)] start to send pre-vote request to all nodes(2) [INFO] [stdout] [2023-01-21T11:33:30Z INFO group_election::fixtures] [Node(NodeId(1000-1003))] an event has happened: TransitToPreCandidate [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core::follower] [NodeId(1000-1001)][Term(1)] an election timeout is hit, need to transit to pre-candidate [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core::candidate] [NodeId(1000-1001)][Term(1)] start the pre-candidate loop [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1001)][Term(1)] quorum is Major, votes granted(1/2) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1001)][Term(1)] start to send pre-vote request to all nodes(2) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1001)][Term(1)] received vote request: VoteRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1001 }, candidate_id: NodeId { group_id: 1000, group_node_id: 1003 }, vote_id: 1, term: 1, factor: MemVoteFactor { priority: 0, _marker: PhantomData }, pre_vote: true, move_leader: false } [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core] [NodeId(1000-1001)][Term(1)] voted for pre-candidate(NodeId(1000-1003)) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1003)][Term(1)] received vote response : VoteResponse { node_id: NodeId { group_id: 1000, group_node_id: 1001 }, candidate_id: NodeId { group_id: 1000, group_node_id: 1003 }, vote_id: 1, term: 1, pre_vote: true, vote_result: Granted } [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1003)][Term(1)] votes granted(2/2) [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core::candidate] [NodeId(1000-1003)][Term(1)] minimum number of pre-votes have been received, so transit to candidate [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core::candidate] [NodeId(1000-1003)][Term(1)] start the candidate loop [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1003)][Term(1)] quorum is Major, votes granted(1/2) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1003)][Term(2)] start to send vote request to all nodes(2) [INFO] [stdout] [2023-01-21T11:33:31Z INFO group_election::fixtures] [Node(NodeId(1000-1003))] an event has happened: TransitToCandidate [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core] [NodeId(1000-1002)][Term(1)] voted for pre-candidate(NodeId(1000-1003)) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1001)][Term(1)] received vote request: VoteRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1001 }, candidate_id: NodeId { group_id: 1000, group_node_id: 1003 }, vote_id: 2, term: 2, factor: MemVoteFactor { priority: 0, _marker: PhantomData }, pre_vote: false, move_leader: false } [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core] [NodeId(1000-1001)][Term(2)] vote request term(2) is greater than current term(2), so transit to follower [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core] [NodeId(1000-1001)][Term(2)] granted vote for candidate(NodeId(1000-1003)) [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core::follower] [NodeId(1000-1001)][Term(2)] start the follower loop [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core] [NodeId(1000-1002)][Term(2)] vote request term(2) is greater than current term(2) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core] [NodeId(1000-1002)][Term(2)] granted vote for candidate(NodeId(1000-1003)) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1003)][Term(2)] received vote response : VoteResponse { node_id: NodeId { group_id: 1000, group_node_id: 1002 }, candidate_id: NodeId { group_id: 1000, group_node_id: 1003 }, vote_id: 1, term: 1, pre_vote: true, vote_result: Granted } [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1003)][Term(2)] vote id is 2, so ignore vote response: VoteResponse { node_id: NodeId { group_id: 1000, group_node_id: 1002 }, candidate_id: NodeId { group_id: 1000, group_node_id: 1003 }, vote_id: 1, term: 1, pre_vote: true, vote_result: Granted } [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1003)][Term(2)] received vote response : VoteResponse { node_id: NodeId { group_id: 1000, group_node_id: 1001 }, candidate_id: NodeId { group_id: 1000, group_node_id: 1003 }, vote_id: 2, term: 2, pre_vote: false, vote_result: Granted } [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1003)][Term(2)] votes granted(2/2) [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core::follower] [NodeId(1000-1002)][Term(2)] an election timeout is hit, need to transit to pre-candidate [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core::candidate] [NodeId(1000-1003)][Term(2)] minimum number of votes have been received, so transit to leader [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core::leader] [NodeId(1000-1003)][Term(2)] start the leader loop [INFO] [stdout] [2023-01-21T11:33:31Z TRACE tender::core::leader] [NodeId(1000-1003)][Term(2)] send heartbeat to node(NodeId(1000-1001)) [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core::candidate] [NodeId(1000-1002)][Term(2)] start the pre-candidate loop [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1002)][Term(2)] quorum is Major, votes granted(1/2) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::candidate] [NodeId(1000-1002)][Term(2)] start to send pre-vote request to all nodes(2) [INFO] [stdout] [2023-01-21T11:33:31Z TRACE tender::core::leader] [NodeId(1000-1003)][Term(2)] send heartbeat to node(NodeId(1000-1002)) [INFO] [stdout] [2023-01-21T11:33:31Z INFO group_election::fixtures] [Node(NodeId(1000-1003))] an event has happened: TransitToLeader { term: 2, caused_by_step_up: false } [INFO] [stdout] [2023-01-21T11:33:31Z INFO group_election::fixtures] [Node(NodeId(1000-1001))] an event has happened: TransitToFollower { term: 2, prev_state: PreCandidate, caused_by_step_down: false } [INFO] [stdout] [2023-01-21T11:33:31Z INFO group_election::fixtures] [Node(NodeId(1000-1002))] an event has happened: TransitToPreCandidate [INFO] [stdout] [2023-01-21T11:33:31Z TRACE tender::core::follower] [NodeId(1000-1001)][Term(2)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1001 }, leader_id: NodeId { group_id: 1000, group_node_id: 1003 }, term: 2 } [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core] [NodeId(1000-1001)][Term(2)] change leader to NodeId(1000-1003) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core] [NodeId(1000-1003)][Term(2)] voted for pre-candidate(NodeId(1000-1002)) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core] [NodeId(1000-1003)][Term(2)] vote term(1) from candidate(NodeId(1000-1001)) is less than current term(2) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core] [NodeId(1000-1001)][Term(2)] reject vote request received within election timeout minimum [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core] [NodeId(1000-1002)][Term(2)] change leader to NodeId(1000-1003) [INFO] [stdout] [2023-01-21T11:33:31Z INFO group_election::fixtures] [Node(NodeId(1000-1001))] an event has happened: TransitToPreCandidate [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core] [NodeId(1000-1002)][Term(2)] received valid heartbeat in PreCandidate state, so transit to follower [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::follower] [NodeId(1000-1001)][Term(2)] event (TransitToPreCandidate) in term 1 is handled [INFO] [stdout] [2023-01-21T11:33:31Z INFO tender::core::follower] [NodeId(1000-1002)][Term(2)] start the follower loop [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core] [NodeId(1000-1002)][Term(2)] vote term(1) from candidate(NodeId(1000-1001)) is less than current term(2) [INFO] [stdout] [2023-01-21T11:33:31Z INFO group_election::fixtures] [Node(NodeId(1000-1002))] an event has happened: ChangeLeader(NodeId { group_id: 1000, group_node_id: 1003 }) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::follower] [NodeId(1000-1002)][Term(2)] event (ChangeLeader(NodeId { group_id: 1000, group_node_id: 1003 })) in term 2 is handled [INFO] [stdout] [2023-01-21T11:33:31Z INFO group_election::fixtures] [Node(NodeId(1000-1002))] an event has happened: TransitToFollower { term: 2, prev_state: PreCandidate, caused_by_step_down: false } [INFO] [stdout] [2023-01-21T11:33:31Z INFO group_election::fixtures] [Node(NodeId(1000-1001))] an event has happened: ChangeLeader(NodeId { group_id: 1000, group_node_id: 1003 }) [INFO] [stdout] [2023-01-21T11:33:31Z DEBUG tender::core::follower] [NodeId(1000-1001)][Term(2)] event (ChangeLeader(NodeId { group_id: 1000, group_node_id: 1003 })) in term 2 is handled [INFO] [stdout] [2023-01-21T11:33:31Z TRACE tender::core::leader] [NodeId(1000-1003)][Term(2)] send heartbeat to node(NodeId(1000-1001)) [INFO] [stdout] [2023-01-21T11:33:31Z TRACE tender::core::leader] [NodeId(1000-1003)][Term(2)] send heartbeat to node(NodeId(1000-1002)) [INFO] [stdout] [2023-01-21T11:33:31Z TRACE tender::core::follower] [NodeId(1000-1001)][Term(2)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1001 }, leader_id: NodeId { group_id: 1000, group_node_id: 1003 }, term: 2 } [INFO] [stdout] [2023-01-21T11:33:31Z TRACE tender::core::follower] [NodeId(1000-1002)][Term(2)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1002 }, leader_id: NodeId { group_id: 1000, group_node_id: 1003 }, term: 2 } [INFO] [stdout] [2023-01-21T11:33:32Z TRACE tender::core::leader] [NodeId(1000-1003)][Term(2)] send heartbeat to node(NodeId(1000-1001)) [INFO] [stdout] [2023-01-21T11:33:32Z TRACE tender::core::leader] [NodeId(1000-1003)][Term(2)] send heartbeat to node(NodeId(1000-1002)) [INFO] [stdout] [2023-01-21T11:33:32Z TRACE tender::core::follower] [NodeId(1000-1002)][Term(2)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1002 }, leader_id: NodeId { group_id: 1000, group_node_id: 1003 }, term: 2 } [INFO] [stdout] [2023-01-21T11:33:32Z TRACE tender::core::follower] [NodeId(1000-1001)][Term(2)] received heartbeat: HeartbeatRequest { target_node_id: NodeId { group_id: 1000, group_node_id: 1001 }, leader_id: NodeId { group_id: 1000, group_node_id: 1003 }, term: 2 } [INFO] [stdout] thread 'test_step_up_and_down' panicked at 'assertion failed: `(left == right)` [INFO] [stdout] left: `2`, [INFO] [stdout] right: `1`', tests/fixtures.rs:280:9 [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x5555592cc3da - std::backtrace_rs::backtrace::libunwind::trace::h66ee1baad70f162c [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5 [INFO] [stdout] 1: 0x5555592cc3da - std::backtrace_rs::backtrace::trace_unsynchronized::h3125ffde7ca4d99d [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5 [INFO] [stdout] 2: 0x5555592cc3da - std::sys_common::backtrace::_print_fmt::hbdbfa23eed07e688 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/sys_common/backtrace.rs:65:5 [INFO] [stdout] 3: 0x5555592cc3da - ::fmt::h9040fa46c3c21247 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/sys_common/backtrace.rs:44:22 [INFO] [stdout] 4: 0x5555592f03ae - core::fmt::write::h19a3b12aeae2c539 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/core/src/fmt/mod.rs:1213:17 [INFO] [stdout] 5: 0x5555592c97c5 - std::io::Write::write_fmt::h09ebe1f248736025 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/io/mod.rs:1682:15 [INFO] [stdout] 6: 0x5555592cc1a5 - std::sys_common::backtrace::_print::hd1ad2573e05197ac [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/sys_common/backtrace.rs:47:5 [INFO] [stdout] 7: 0x5555592cc1a5 - std::sys_common::backtrace::print::h6c0cd3c6b401ec72 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/sys_common/backtrace.rs:34:9 [INFO] [stdout] 8: 0x5555592cdf9f - std::panicking::default_hook::{{closure}}::hc09dc9f114375efd [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/panicking.rs:267:22 [INFO] [stdout] 9: 0x5555592cdc5f - std::panicking::default_hook::hfef666cfe5862d3a [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/panicking.rs:283:9 [INFO] [stdout] 10: 0x55555929ce77 - as core::ops::function::Fn>::call::he38c0640087feffe [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/alloc/src/boxed.rs:2002:9 [INFO] [stdout] 11: 0x55555929ce77 - test::test_main::{{closure}}::h071363f8c705b257 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/test/src/lib.rs:135:21 [INFO] [stdout] 12: 0x5555592ce6cd - as core::ops::function::Fn>::call::hc536ff91d276a16f [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/alloc/src/boxed.rs:2002:9 [INFO] [stdout] 13: 0x5555592ce6cd - std::panicking::rust_panic_with_hook::h09cdc27f5c472cc8 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/panicking.rs:692:13 [INFO] [stdout] 14: 0x5555592ce449 - std::panicking::begin_panic_handler::{{closure}}::h5f38ac8f9e3f3f74 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/panicking.rs:579:13 [INFO] [stdout] 15: 0x5555592cc88c - std::sys_common::backtrace::__rust_end_short_backtrace::hb85d55d699513ebd [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/sys_common/backtrace.rs:137:18 [INFO] [stdout] 16: 0x5555592ce152 - rust_begin_unwind [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/panicking.rs:575:5 [INFO] [stdout] 17: 0x5555591c0c03 - core::panicking::panic_fmt::hf0f695a991f65334 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/core/src/panicking.rs:64:14 [INFO] [stdout] 18: 0x5555591c0f3f - core::panicking::assert_failed_inner::h2c1788cc8f33879c [INFO] [stdout] 19: 0x5555591b7d0b - core::panicking::assert_failed::h18cb34d6eefd4e33 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/core/src/panicking.rs:212:5 [INFO] [stdout] 20: 0x5555591d7fac - group_election::fixtures::MemRouter::assert_node_state::hfac573a9859c5151 [INFO] [stdout] at /opt/rustwide/workdir/tests/fixtures.rs:280:9 [INFO] [stdout] 21: 0x5555591e9c90 - group_election::test_step_up_and_down::hb5b80bd1df647a45 [INFO] [stdout] at /opt/rustwide/workdir/tests/group_election.rs:272:5 [INFO] [stdout] 22: 0x5555591e9c90 - group_election::test_step_up_and_down::{{closure}}::h9418d6e9f24ec7a6 [INFO] [stdout] at /opt/rustwide/workdir/tests/group_election.rs:235:28 [INFO] [stdout] 23: 0x5555591e9c90 - core::ops::function::FnOnce::call_once::hfd01653e64d9bbe6 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x5555592a230f - core::ops::function::FnOnce::call_once::hd02ae4551b605d89 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x5555592a230f - test::__rust_begin_short_backtrace::he067d58dbbe4eff7 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/test/src/lib.rs:656:18 [INFO] [stdout] 26: 0x555559273b2c - test::run_test::{{closure}}::h99317ada5450d4d1 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/test/src/lib.rs:647:30 [INFO] [stdout] 27: 0x555559273b2c - core::ops::function::FnOnce::call_once{{vtable.shim}}::hdbccb09ec15aa75d [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 28: 0x5555592a129a - as core::ops::function::FnOnce>::call_once::hc712263f4579e1e7 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/alloc/src/boxed.rs:1988:9 [INFO] [stdout] 29: 0x5555592a129a - as core::ops::function::FnOnce<()>>::call_once::h3fdee2221c1ed465 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/core/src/panic/unwind_safe.rs:271:9 [INFO] [stdout] 30: 0x5555592a129a - std::panicking::try::do_call::h974a87241ba6f765 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/panicking.rs:483:40 [INFO] [stdout] 31: 0x5555592a129a - std::panicking::try::hf1858fb4e3e6e1ba [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/panicking.rs:447:19 [INFO] [stdout] 32: 0x5555592a129a - std::panic::catch_unwind::h3fc10dff34553685 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/panic.rs:140:14 [INFO] [stdout] 33: 0x5555592a129a - test::run_test_in_process::h44fc06fdd04e007f [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/test/src/lib.rs:679:27 [INFO] [stdout] 34: 0x5555592a129a - test::run_test::run_test_inner::{{closure}}::hc67e39cdf6272f40 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/test/src/lib.rs:573:39 [INFO] [stdout] 35: 0x55555926e1f1 - test::run_test::run_test_inner::{{closure}}::h1d1c6318c71d3046 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/test/src/lib.rs:600:37 [INFO] [stdout] 36: 0x55555926e1f1 - std::sys_common::backtrace::__rust_begin_short_backtrace::h4e2185d7b9c32705 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/sys_common/backtrace.rs:121:18 [INFO] [stdout] 37: 0x555559273bbb - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hf11e5bf7faafa3a6 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/thread/mod.rs:558:17 [INFO] [stdout] 38: 0x555559273bbb - as core::ops::function::FnOnce<()>>::call_once::h2a1d6b9f209466ea [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/core/src/panic/unwind_safe.rs:271:9 [INFO] [stdout] 39: 0x555559273bbb - std::panicking::try::do_call::he469b6a8fbe95fec [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/panicking.rs:483:40 [INFO] [stdout] 40: 0x555559273bbb - std::panicking::try::h1f733fcf13a98585 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/panicking.rs:447:19 [INFO] [stdout] 41: 0x555559273bbb - std::panic::catch_unwind::hecc42615914d8beb [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/panic.rs:140:14 [INFO] [stdout] 42: 0x555559273bbb - std::thread::Builder::spawn_unchecked_::{{closure}}::h17668fadfa667920 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 43: 0x555559273bbb - core::ops::function::FnOnce::call_once{{vtable.shim}}::hfeec135cddf64dce [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 44: 0x5555592d2d53 - as core::ops::function::FnOnce>::call_once::h092bd1b4a0b89f07 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/alloc/src/boxed.rs:1988:9 [INFO] [stdout] 45: 0x5555592d2d53 - as core::ops::function::FnOnce>::call_once::h54d7347fe39f45fd [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/alloc/src/boxed.rs:1988:9 [INFO] [stdout] 46: 0x5555592d2d53 - std::sys::unix::thread::Thread::new::thread_start::hc8954c4997518988 [INFO] [stdout] at /rustc/159ba8a92c9e2fa4121f106176309521f4af87e9/library/std/src/sys/unix/thread.rs:108:17 [INFO] [stdout] 47: 0x7f1cb3e63b43 - [INFO] [stdout] 48: 0x7f1cb3ef4bb4 - __clone [INFO] [stdout] 49: 0x0 - [INFO] [stdout] [2023-01-21T11:33:32Z TRACE tender::core::leader] [NodeId(1000-1003)][Term(2)] send heartbeat to node(NodeId(1000-1001)) [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] test_step_up_and_down [INFO] [stdout] [INFO] [stdout] test result: FAILED. 5 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 10.13s [INFO] [stdout] [INFO] [stderr] error: test failed, to rerun pass `--test group_election` [INFO] running `Command { std: "docker" "inspect" "e18c9815d3add59ac3788f1d03171b71705971da52fd9743bb50acd3ed80c6c6", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "e18c9815d3add59ac3788f1d03171b71705971da52fd9743bb50acd3ed80c6c6", kill_on_drop: false }` [INFO] [stdout] e18c9815d3add59ac3788f1d03171b71705971da52fd9743bb50acd3ed80c6c6