[INFO] fetching crate chitchat 0.7.0... [INFO] testing chitchat-0.7.0 against 1.81.0 for beta-1.82-1 [INFO] extracting crate chitchat 0.7.0 into /workspace/builds/worker-3-tc1/source [INFO] validating manifest of crates.io crate chitchat 0.7.0 on toolchain 1.81.0 [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+1.81.0" "metadata" "--manifest-path" "Cargo.toml" "--no-deps", kill_on_drop: false }` [INFO] started tweaking crates.io crate chitchat 0.7.0 [INFO] finished tweaking crates.io crate chitchat 0.7.0 [INFO] tweaked toml for crates.io crate chitchat 0.7.0 written to /workspace/builds/worker-3-tc1/source/Cargo.toml [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+1.81.0" "generate-lockfile" "--manifest-path" "Cargo.toml", kill_on_drop: false }` [INFO] [stderr] Updating crates.io index [INFO] [stderr] Blocking waiting for file lock on package cache [INFO] [stderr] Locking 75 packages to latest compatible versions [INFO] [stderr] Adding addr2line v0.22.0 (latest: v0.24.1) [INFO] [stderr] Adding gimli v0.29.0 (latest: v0.31.0) [INFO] [stderr] Adding hermit-abi v0.3.9 (latest: v0.4.0) [INFO] [stderr] Adding itertools v0.10.5 (latest: v0.13.0) [INFO] [stderr] Adding miniz_oxide v0.7.4 (latest: v0.8.0) [INFO] [stderr] Adding mock_instant v0.2.1 (latest: v0.5.1) [INFO] [stderr] Adding nu-ansi-term v0.46.0 (latest: v0.50.1) [INFO] [stderr] Adding wasi v0.11.0+wasi-snapshot-preview1 (latest: v0.13.2+wasi-0.2.1) [INFO] [stderr] Adding windows-sys v0.52.0 (latest: v0.59.0) [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+1.81.0" "fetch" "--manifest-path" "Cargo.toml", kill_on_drop: false }` [INFO] [stderr] Blocking waiting for file lock on package cache [INFO] [stderr] Downloading crates ... [INFO] [stderr] Downloaded mock_instant v0.2.1 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-3-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-3-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:96c0aa5895ecf336b730ef27bcdde6e21027415df19888aad3cf1448e5d65d90" "/opt/rustwide/cargo-home/bin/cargo" "+1.81.0" "metadata" "--no-deps" "--format-version=1", kill_on_drop: false }` [INFO] [stdout] 3d65c47e7322ea6dab5320553c87c6b66fbe5a15fe2a1f18720f4d93b0f66711 [INFO] running `Command { std: "docker" "start" "-a" "3d65c47e7322ea6dab5320553c87c6b66fbe5a15fe2a1f18720f4d93b0f66711", kill_on_drop: false }` [INFO] running `Command { std: "docker" "inspect" "3d65c47e7322ea6dab5320553c87c6b66fbe5a15fe2a1f18720f4d93b0f66711", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "3d65c47e7322ea6dab5320553c87c6b66fbe5a15fe2a1f18720f4d93b0f66711", kill_on_drop: false }` [INFO] [stdout] 3d65c47e7322ea6dab5320553c87c6b66fbe5a15fe2a1f18720f4d93b0f66711 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-3-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-3-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=warn" "-e" "RUSTDOCFLAGS=--cap-lints=warn" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:96c0aa5895ecf336b730ef27bcdde6e21027415df19888aad3cf1448e5d65d90" "/opt/rustwide/cargo-home/bin/cargo" "+1.81.0" "build" "--frozen" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] 5cb7a3cc6bf1cb0ec96a4fc9f7b02b52c12ffa1e6594d79c267ba2782d9ec567 [INFO] running `Command { std: "docker" "start" "-a" "5cb7a3cc6bf1cb0ec96a4fc9f7b02b52c12ffa1e6594d79c267ba2782d9ec567", kill_on_drop: false }` [INFO] [stderr] Compiling futures-sink v0.3.30 [INFO] [stderr] Compiling anyhow v1.0.87 [INFO] [stderr] Compiling futures-core v0.3.30 [INFO] [stderr] Compiling tracing-core v0.1.32 [INFO] [stderr] Compiling either v1.13.0 [INFO] [stderr] Compiling syn v2.0.77 [INFO] [stderr] Compiling itertools v0.10.5 [INFO] [stderr] Compiling zerocopy-derive v0.7.35 [INFO] [stderr] Compiling tokio-macros v2.4.0 [INFO] [stderr] Compiling tracing-attributes v0.1.27 [INFO] [stderr] Compiling serde_derive v1.0.210 [INFO] [stderr] Compiling async-trait v0.1.82 [INFO] [stderr] Compiling tokio v1.40.0 [INFO] [stderr] Compiling zerocopy v0.7.35 [INFO] [stderr] Compiling tracing v0.1.40 [INFO] [stderr] Compiling ppv-lite86 v0.2.20 [INFO] [stderr] Compiling rand_chacha v0.3.1 [INFO] [stderr] Compiling rand v0.8.5 [INFO] [stderr] Compiling serde v1.0.210 [INFO] [stderr] Compiling tokio-util v0.7.12 [INFO] [stderr] Compiling tokio-stream v0.1.16 [INFO] [stderr] Compiling chitchat v0.7.0 (/opt/rustwide/workdir) [INFO] [stdout] warning: method `serialize_to_vec` is never used [INFO] [stdout] --> src/serialize.rs:16:8 [INFO] [stdout] | [INFO] [stdout] 13 | pub trait Serializable: Sized { [INFO] [stdout] | ------------ method in this trait [INFO] [stdout] ... [INFO] [stdout] 16 | fn serialize_to_vec(&self) -> Vec { [INFO] [stdout] | ^^^^^^^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(dead_code)]` on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: 1 warning emitted [INFO] [stdout] [INFO] [stdout] [INFO] [stderr] Finished `dev` profile [unoptimized + debuginfo] target(s) in 11.39s [INFO] running `Command { std: "docker" "inspect" "5cb7a3cc6bf1cb0ec96a4fc9f7b02b52c12ffa1e6594d79c267ba2782d9ec567", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "5cb7a3cc6bf1cb0ec96a4fc9f7b02b52c12ffa1e6594d79c267ba2782d9ec567", kill_on_drop: false }` [INFO] [stdout] 5cb7a3cc6bf1cb0ec96a4fc9f7b02b52c12ffa1e6594d79c267ba2782d9ec567 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-3-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-3-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=warn" "-e" "RUSTDOCFLAGS=--cap-lints=warn" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:96c0aa5895ecf336b730ef27bcdde6e21027415df19888aad3cf1448e5d65d90" "/opt/rustwide/cargo-home/bin/cargo" "+1.81.0" "test" "--frozen" "--no-run" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] 4f2646a13e8cb5e6746d1a4eef0cb11ebc9f0efdd5c47b5a87575de29410d5a2 [INFO] running `Command { std: "docker" "start" "-a" "4f2646a13e8cb5e6746d1a4eef0cb11ebc9f0efdd5c47b5a87575de29410d5a2", kill_on_drop: false }` [INFO] [stderr] Compiling tracing-core v0.1.32 [INFO] [stderr] Compiling log v0.4.22 [INFO] [stderr] Compiling overload v0.1.1 [INFO] [stderr] Compiling thread_local v1.1.8 [INFO] [stderr] Compiling sharded-slab v0.1.7 [INFO] [stderr] Compiling mock_instant v0.2.1 [INFO] [stderr] Compiling serde_json v1.0.128 [INFO] [stderr] Compiling nu-ansi-term v0.46.0 [INFO] [stderr] Compiling tracing v0.1.40 [INFO] [stderr] Compiling tracing-log v0.2.0 [INFO] [stderr] Compiling chitchat v0.7.0 (/opt/rustwide/workdir) [INFO] [stderr] Compiling tracing-subscriber v0.3.18 [INFO] [stderr] Compiling assert-json-diff v2.0.2 [INFO] [stdout] warning: method `serialize_to_vec` is never used [INFO] [stdout] --> src/serialize.rs:16:8 [INFO] [stdout] | [INFO] [stdout] 13 | pub trait Serializable: Sized { [INFO] [stdout] | ------------ method in this trait [INFO] [stdout] ... [INFO] [stdout] 16 | fn serialize_to_vec(&self) -> Vec { [INFO] [stdout] | ^^^^^^^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(dead_code)]` on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: 1 warning emitted [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: method `serialize_to_vec` is never used [INFO] [stdout] --> src/serialize.rs:16:8 [INFO] [stdout] | [INFO] [stdout] 13 | pub trait Serializable: Sized { [INFO] [stdout] | ------------ method in this trait [INFO] [stdout] ... [INFO] [stdout] 16 | fn serialize_to_vec(&self) -> Vec { [INFO] [stdout] | ^^^^^^^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(dead_code)]` on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: 1 warning emitted [INFO] [stdout] [INFO] [stdout] [INFO] [stderr] Finished `test` profile [unoptimized + debuginfo] target(s) in 4.88s [INFO] running `Command { std: "docker" "inspect" "4f2646a13e8cb5e6746d1a4eef0cb11ebc9f0efdd5c47b5a87575de29410d5a2", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "4f2646a13e8cb5e6746d1a4eef0cb11ebc9f0efdd5c47b5a87575de29410d5a2", kill_on_drop: false }` [INFO] [stdout] 4f2646a13e8cb5e6746d1a4eef0cb11ebc9f0efdd5c47b5a87575de29410d5a2 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-3-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-3-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=warn" "-e" "RUSTDOCFLAGS=--cap-lints=warn" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:96c0aa5895ecf336b730ef27bcdde6e21027415df19888aad3cf1448e5d65d90" "/opt/rustwide/cargo-home/bin/cargo" "+1.81.0" "test" "--frozen", kill_on_drop: false }` [INFO] [stdout] d64fadade85f9131ce58cf19db44b02bc5dd784e1a1dfa85d7872fb2d972150a [INFO] running `Command { std: "docker" "start" "-a" "d64fadade85f9131ce58cf19db44b02bc5dd784e1a1dfa85d7872fb2d972150a", kill_on_drop: false }` [INFO] [stderr] warning: method `serialize_to_vec` is never used [INFO] [stderr] --> src/serialize.rs:16:8 [INFO] [stderr] | [INFO] [stderr] 13 | pub trait Serializable: Sized { [INFO] [stderr] | ------------ method in this trait [INFO] [stderr] ... [INFO] [stderr] 16 | fn serialize_to_vec(&self) -> Vec { [INFO] [stderr] | ^^^^^^^^^^^^^^^^ [INFO] [stderr] | [INFO] [stderr] = note: `#[warn(dead_code)]` on by default [INFO] [stderr] [INFO] [stderr] warning: `chitchat` (lib) generated 1 warning [INFO] [stderr] warning: `chitchat` (lib test) generated 1 warning (1 duplicate) [INFO] [stderr] Finished `test` profile [unoptimized + debuginfo] target(s) in 0.04s [INFO] [stderr] Running unittests src/lib.rs (/opt/rustwide/target/debug/deps/chitchat-770f0d47812b1ef4) [INFO] [stdout] [INFO] [stdout] running 59 tests [INFO] [stdout] test delta::tests::test_delta_serialization_default ... ok [INFO] [stdout] test delta::tests::test_delta_serialization_exceed_mtu_on_add_kv ... ok [INFO] [stdout] test delta::tests::test_delta_serialization_exceed_mtu_on_add_node_to_reset ... ok [INFO] [stdout] test delta::tests::test_delta_serialization_exceed_mtu_on_add_node ... ok [INFO] [stdout] test delta::tests::test_delta_serialization_simple_foo ... ok [INFO] [stdout] test failure_detector::tests::test_bounded_array_stats ... ok [INFO] [stdout] test delta::tests::test_delta_serialization_simple_node ... ok [INFO] [stdout] test delta::tests::test_delta_serialization_simple_with_nodes_to_reset ... ok [INFO] [stdout] test failure_detector::tests::test_failure_detector_node_state_after_initial_interval ... ok [INFO] [stdout] test failure_detector::tests::test_sampling_window ... ok [INFO] [stdout] test message::tests::test_ack ... ok [INFO] [stdout] test message::tests::test_bad_cluster ... ok [INFO] [stdout] test message::tests::test_syn ... ok [INFO] [stdout] test message::tests::test_syn_ack ... ok [INFO] [stdout] test serialize::tests::test_serialize_bool ... ok [INFO] [stdout] test serialize::tests::test_serialize_chitchat_id ... ok [INFO] [stdout] test serialize::tests::test_serialize_heartbeat ... ok [INFO] [stdout] test serialize::tests::test_serialize_ip ... ok [INFO] [stdout] test serialize::tests::test_serialize_option_u64 ... ok [INFO] [stdout] test server::tests::test_gossip_no_dead_node_no_seed_nodes ... ok [INFO] [stdout] test server::tests::test_gossip_dead_and_seed_node ... ok [INFO] [stdout] test server::tests::test_select_nodes_for_gossip ... ok [INFO] [stdout] test state::tests::test_cluster_state_apply_delta ... ok [INFO] [stdout] test server::tests::test_syn_ack ... ok [INFO] [stdout] test server::tests::test_syn_bad_cluster ... ok [INFO] [stdout] test state::tests::test_cluster_state_compute_digest ... ok [INFO] [stdout] test server::tests::test_seeding ... ok [INFO] [stdout] test state::tests::test_cluster_state_missing_node ... ok [INFO] [stdout] test state::tests::test_cluster_state_set ... ok [INFO] [stdout] test server::tests::test_syn ... ok [INFO] [stdout] test server::tests::test_heartbeat ... ok [INFO] [stdout] test state::tests::test_cluster_state_set_and_mark_for_deletion ... ok [INFO] [stdout] test state::tests::test_cluster_state_gc_keys_marked_for_deletion ... ok [INFO] [stdout] test state::tests::test_cluster_state_first_version_is_one ... ok [INFO] [stdout] test state::tests::test_cluster_state_compute_delta_with_old_node_state_that_needs_reset ... ok [INFO] [stdout] test state::tests::test_iter_prefix ... ok [INFO] [stdout] test state::tests::test_sorted_stale_nodes_empty ... ok [INFO] [stdout] test state::tests::test_cluster_state_set_with_same_value_updates_version ... ok [INFO] [stdout] test state::tests::test_sorted_stale_nodes_offer ... ok [INFO] [stdout] test state::tests::test_sorted_stale_nodes_into_iter ... ok [INFO] [stdout] test state::tests::test_sorted_stale_nodes_insert ... ok [INFO] [stdout] test state::tests::test_cluster_state_compute_delta_should_ignore_dead_nodes ... ok [INFO] [stdout] test state::tests::test_cluster_state_compute_delta_depth_first_chitchat ... ok [INFO] [stdout] test state::tests::test_stale_node_iter_stale_key_values ... ok [INFO] [stdout] test state::tests::test_cluster_state_compute_delta_depth_first_single_node ... ok [INFO] [stdout] test state::tests::test_cluster_state_compute_delta_missing_node ... ok [INFO] [stdout] test tests::test_chitchat_handshake ... ok [INFO] [stdout] test failure_detector::tests::test_failure_detector_node_state_from_live_to_down_to_live ... ok [INFO] [stdout] test failure_detector::tests::test_failure_detector ... ok [INFO] [stdout] test delta::tests::test_delta_serialization_panic_if_add_after_exceed - should panic ... ok [INFO] [stdout] test transport::tests::test_udp_transport_ignore_invalid_payload ... ok [INFO] [stdout] test server::tests::test_member_change_event_is_broadcasted ... ok [INFO] [stdout] test tests::test_multiple_nodes ... ok [INFO] [stdout] test transport::tests::test_transport_in_mem ... ok [INFO] [stdout] test transport::tests::test_transport_udp ... ok [INFO] [stdout] test tests::test_network_partition_nodes ... ok [INFO] [stdout] test tests::test_dead_node_should_not_be_gossiped_when_node_joins ... ok [INFO] [stdout] test tests::test_node_goes_from_live_to_down_to_live ... ok [INFO] [stdout] test tests::test_dead_node_garbage_collection ... ok [INFO] [stdout] [INFO] [stdout] test result: ok. 59 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 25.81s [INFO] [stdout] [INFO] [stderr] Running tests/cluster_test.rs (/opt/rustwide/target/debug/deps/cluster_test-f0bff949e35d994e) [INFO] [stdout] [INFO] [stdout] running 4 tests [INFO] [stdout] 2024-09-08T21:54:34.558043Z  INFO chitchat::server: initial_seed_addrs={} [INFO] [stdout] 2024-09-08T21:54:34.558043Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:43049, 127.0.0.1:42049, 127.0.0.1:45705} [INFO] [stdout] 2024-09-08T21:54:34.558169Z  INFO chitchat::server: initial_seed_addrs={} [INFO] [stdout] 2024-09-08T21:54:34.558551Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:42643} [INFO] [stdout] 2024-09-08T21:54:34.558564Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:36981} [INFO] [stdout] 2024-09-08T21:54:34.558584Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:43049, 127.0.0.1:42049, 127.0.0.1:45705} [INFO] [stdout] 2024-09-08T21:54:34.558633Z  INFO cluster_test: Waiting for node state to be present. node_id=node-1 state_snapshot=ClusterStateSnapshot { node_state_snapshots: [NodeStateSnapshot { chitchat_id: ChitchatId { node_id: "node-2", generation_id: 0, gossip_advertise_addr: 127.0.0.1:40983 }, node_state: NodeState { heartbeat: Heartbeat(1), key_values: {}, max_version: 0, last_heartbeat: Instant { tv_sec: 60289, tv_nsec: 167046505 } } }], seed_addrs: {127.0.0.1:36981} } [INFO] [stdout] 2024-09-08T21:54:34.558678Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:43049, 127.0.0.1:42049, 127.0.0.1:45705} [INFO] [stdout] 2024-09-08T21:54:34.558786Z  INFO cluster_test: Waiting for node state to be present. node_id=node-1 state_snapshot=ClusterStateSnapshot { node_state_snapshots: [NodeStateSnapshot { chitchat_id: ChitchatId { node_id: "node-2", generation_id: 0, gossip_advertise_addr: 127.0.0.1:43049 }, node_state: NodeState { heartbeat: Heartbeat(1), key_values: {}, max_version: 0, last_heartbeat: Instant { tv_sec: 60289, tv_nsec: 167087565 } } }], seed_addrs: {127.0.0.1:43049, 127.0.0.1:42049, 127.0.0.1:45705} } [INFO] [stdout] 2024-09-08T21:54:34.559357Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:41383, 127.0.0.1:35969, 127.0.0.1:46067} [INFO] [stdout] 2024-09-08T21:54:34.559436Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:46067, 127.0.0.1:35969, 127.0.0.1:41383} [INFO] [stdout] 2024-09-08T21:54:34.559581Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:35969, 127.0.0.1:46067, 127.0.0.1:41383} [INFO] [stdout] 2024-09-08T21:54:34.559739Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:35969, 127.0.0.1:41383, 127.0.0.1:46067} [INFO] [stdout] 2024-09-08T21:54:34.559884Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:46067, 127.0.0.1:41383, 127.0.0.1:35969} [INFO] [stdout] 2024-09-08T21:54:34.559976Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:46067, 127.0.0.1:35969, 127.0.0.1:41383} [INFO] [stdout] 2024-09-08T21:54:34.560082Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:46067, 127.0.0.1:41383, 127.0.0.1:35969} [INFO] [stdout] 2024-09-08T21:54:34.560193Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:41383, 127.0.0.1:46067, 127.0.0.1:35969} [INFO] [stdout] 2024-09-08T21:54:34.560355Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:46067, 127.0.0.1:35969, 127.0.0.1:41383} [INFO] [stdout] 2024-09-08T21:54:34.560468Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:41383, 127.0.0.1:35969, 127.0.0.1:46067} [INFO] [stdout] 2024-09-08T21:54:34.560564Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:46067, 127.0.0.1:35969, 127.0.0.1:41383} [INFO] [stdout] 2024-09-08T21:54:34.560660Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:46067, 127.0.0.1:35969, 127.0.0.1:41383} [INFO] [stdout] 2024-09-08T21:54:34.560753Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:35969, 127.0.0.1:46067, 127.0.0.1:41383} [INFO] [stdout] 2024-09-08T21:54:34.560845Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:41383, 127.0.0.1:35969, 127.0.0.1:46067} [INFO] [stdout] 2024-09-08T21:54:34.560934Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:46067, 127.0.0.1:35969, 127.0.0.1:41383} [INFO] [stdout] 2024-09-08T21:54:34.561049Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:41383, 127.0.0.1:46067, 127.0.0.1:35969} [INFO] [stdout] 2024-09-08T21:54:34.561135Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:35969, 127.0.0.1:41383, 127.0.0.1:46067} [INFO] [stdout] 2024-09-08T21:54:34.561217Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:46067, 127.0.0.1:35969, 127.0.0.1:41383} [INFO] [stdout] 2024-09-08T21:54:34.561280Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:46067, 127.0.0.1:35969, 127.0.0.1:41383} [INFO] [stdout] 2024-09-08T21:54:34.561367Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:41383, 127.0.0.1:46067, 127.0.0.1:35969} [INFO] [stdout] 2024-09-08T21:54:34.558644Z  INFO cluster_test: Waiting for node state to be present. node_id=node-1 state_snapshot=ClusterStateSnapshot { node_state_snapshots: [NodeStateSnapshot { chitchat_id: ChitchatId { node_id: "node-2", generation_id: 0, gossip_advertise_addr: 127.0.0.1:40665 }, node_state: NodeState { heartbeat: Heartbeat(1), key_values: {"key_b": VersionedValue { value: "1", version: 1, tombstone: None }}, max_version: 1, last_heartbeat: Instant { tv_sec: 60289, tv_nsec: 167042715 } } }], seed_addrs: {127.0.0.1:42643} } [INFO] [stdout] 2024-09-08T21:54:34.660508Z  INFO chitchat::transport::channel: close addr=127.0.0.1:40983 [INFO] [stdout] 2024-09-08T21:54:34.660594Z  INFO chitchat::transport::channel: close addr=127.0.0.1:36981 [INFO] [stdout] 2024-09-08T21:54:34.660605Z  INFO cluster_test: Waiting for node state to be present. node_id=node-1 state_snapshot=ClusterStateSnapshot { node_state_snapshots: [NodeStateSnapshot { chitchat_id: ChitchatId { node_id: "node-2", generation_id: 0, gossip_advertise_addr: 127.0.0.1:43049 }, node_state: NodeState { heartbeat: Heartbeat(2), key_values: {}, max_version: 0, last_heartbeat: Instant { tv_sec: 60289, tv_nsec: 167087565 } } }, NodeStateSnapshot { chitchat_id: ChitchatId { node_id: "node-3", generation_id: 0, gossip_advertise_addr: 127.0.0.1:45705 }, node_state: NodeState { heartbeat: Heartbeat(2), key_values: {}, max_version: 0, last_heartbeat: Instant { tv_sec: 60289, tv_nsec: 170775544 } } }], seed_addrs: {127.0.0.1:43049, 127.0.0.1:42049, 127.0.0.1:45705} } [INFO] [stdout] test test_simple_simulation_with_network_partition ... ok [INFO] [stdout] 2024-09-08T21:54:34.662451Z  INFO chitchat::transport::channel: close addr=127.0.0.1:42643 [INFO] [stdout] 2024-09-08T21:54:34.662502Z  INFO chitchat::transport::channel: close addr=127.0.0.1:40665 [INFO] [stdout] test test_simple_simulation_insert ... ok [INFO] [stdout] 2024-09-08T21:54:34.761563Z  INFO cluster_test: Waiting for predicate to be true. node_id=node-1 [INFO] [stdout] 2024-09-08T21:54:36.863038Z  INFO chitchat::server: initial_seed_addrs={127.0.0.1:45705} [INFO] [stdout] 2024-09-08T21:54:37.364099Z  INFO cluster_test: Waiting for predicate to be true. node_id=node-1 [INFO] [stdout] 2024-09-08T21:54:37.460882Z  WARN chitchat::state: Node to reset ChitchatId { node_id: "node-3", generation_id: 0, gossip_advertise_addr: 127.0.0.1:45705 } [INFO] [stdout] 2024-09-08T21:54:37.461055Z  WARN chitchat::state: Node to reset ChitchatId { node_id: "node-1", generation_id: 0, gossip_advertise_addr: 127.0.0.1:42049 } [INFO] [stdout] 2024-09-08T21:54:37.461082Z  WARN chitchat::state: Node to reset ChitchatId { node_id: "node-2", generation_id: 0, gossip_advertise_addr: 127.0.0.1:43049 } [INFO] [stdout] 2024-09-08T21:54:37.461468Z  WARN chitchat::state: Node to reset ChitchatId { node_id: "node-1", generation_id: 0, gossip_advertise_addr: 127.0.0.1:42049 } [INFO] [stdout] 2024-09-08T21:54:37.461499Z  WARN chitchat::state: Node to reset ChitchatId { node_id: "node-2", generation_id: 0, gossip_advertise_addr: 127.0.0.1:43049 } [INFO] [stdout] 2024-09-08T21:54:37.464723Z  INFO cluster_test: Waiting for predicate to be true. node_id=node-1 [INFO] [stdout] 2024-09-08T21:54:37.560536Z  WARN chitchat::state: Node to reset ChitchatId { node_id: "node-3", generation_id: 0, gossip_advertise_addr: 127.0.0.1:45705 } [INFO] [stdout] 2024-09-08T21:54:37.560639Z  WARN chitchat::state: Node to reset ChitchatId { node_id: "node-3", generation_id: 0, gossip_advertise_addr: 127.0.0.1:45705 } [INFO] [stdout] 2024-09-08T21:54:37.560702Z  WARN chitchat::state: Node to reset ChitchatId { node_id: "node-3", generation_id: 0, gossip_advertise_addr: 127.0.0.1:45705 } [INFO] [stdout] 2024-09-08T21:54:37.566460Z  INFO cluster_test: Waiting for predicate to be true. node_id=node-1 [INFO] [stdout] 2024-09-08T21:54:37.667758Z  INFO cluster_test: Waiting for predicate to be true. node_id=node-1 [INFO] [stdout] 2024-09-08T21:54:37.769621Z  INFO cluster_test: Waiting for predicate to be true. node_id=node-1 [INFO] [stdout] 2024-09-08T21:54:38.032179Z  INFO chitchat::transport::channel: close addr=127.0.0.1:43049 [INFO] [stdout] 2024-09-08T21:54:38.032240Z  INFO chitchat::transport::channel: close addr=127.0.0.1:42049 [INFO] [stdout] 2024-09-08T21:54:38.032266Z  INFO chitchat::transport::channel: close addr=127.0.0.1:45705 [INFO] [stdout] 2024-09-08T21:54:38.032289Z  INFO chitchat::transport::channel: close addr=127.0.0.1:34951 [INFO] [stdout] test test_marked_for_deletion_gc_with_network_partition ... FAILED [INFO] [stdout] 2024-09-08T21:54:44.598931Z  INFO cluster_test: Checking keys are present... [INFO] [stdout] 2024-09-08T21:54:54.647464Z  INFO cluster_test: Checking keys are deleted... [INFO] [stdout] 2024-09-08T21:54:54.661123Z  INFO chitchat::transport::channel: close addr=127.0.0.1:34769 [INFO] [stdout] 2024-09-08T21:54:54.661430Z  INFO chitchat::transport::channel: close addr=127.0.0.1:46559 [INFO] [stdout] 2024-09-08T21:54:54.661680Z  INFO chitchat::transport::channel: close addr=127.0.0.1:33425 [INFO] [stdout] 2024-09-08T21:54:54.662807Z  INFO chitchat::transport::channel: close addr=127.0.0.1:41443 [INFO] [stdout] 2024-09-08T21:54:54.663268Z  INFO chitchat::transport::channel: close addr=127.0.0.1:34189 [INFO] [stdout] 2024-09-08T21:54:54.663409Z  INFO chitchat::transport::channel: close addr=127.0.0.1:41383 [INFO] [stdout] 2024-09-08T21:54:54.664320Z  INFO chitchat::transport::channel: close addr=127.0.0.1:36299 [INFO] [stdout] 2024-09-08T21:54:54.664602Z  INFO chitchat::transport::channel: close addr=127.0.0.1:40313 [INFO] [stdout] 2024-09-08T21:54:54.664960Z  INFO chitchat::transport::channel: close addr=127.0.0.1:46067 [INFO] [stdout] 2024-09-08T21:54:54.665652Z  INFO chitchat::transport::channel: close addr=127.0.0.1:46617 [INFO] [stdout] 2024-09-08T21:54:54.665936Z  INFO chitchat::transport::channel: close addr=127.0.0.1:35969 [INFO] [stdout] 2024-09-08T21:54:54.666479Z  INFO chitchat::transport::channel: close addr=127.0.0.1:42177 [INFO] [stdout] 2024-09-08T21:54:54.667057Z  INFO chitchat::transport::channel: close addr=127.0.0.1:39505 [INFO] [stdout] 2024-09-08T21:54:54.667388Z  INFO chitchat::transport::channel: close addr=127.0.0.1:41533 [INFO] [stdout] 2024-09-08T21:54:54.667593Z  INFO chitchat::transport::channel: close addr=127.0.0.1:45707 [INFO] [stdout] 2024-09-08T21:54:54.668358Z  INFO chitchat::transport::channel: close addr=127.0.0.1:34613 [INFO] [stdout] 2024-09-08T21:54:54.668670Z  INFO chitchat::transport::channel: close addr=127.0.0.1:46253 [INFO] [stdout] 2024-09-08T21:54:54.668833Z  INFO chitchat::transport::channel: close addr=127.0.0.1:41467 [INFO] [stdout] 2024-09-08T21:54:54.669967Z  INFO chitchat::transport::channel: close addr=127.0.0.1:33725 [INFO] [stdout] 2024-09-08T21:54:54.670149Z  INFO chitchat::transport::channel: close addr=127.0.0.1:42019 [INFO] [stdout] test test_simple_simulation_heavy_insert_delete ... ok [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] [INFO] [stdout] ---- test_marked_for_deletion_gc_with_network_partition stdout ---- [INFO] [stdout] thread 'test_marked_for_deletion_gc_with_network_partition' panicked at tests/cluster_test.rs:151:28: [INFO] [stdout] called `Result::unwrap()` on an `Err` value: Predicate timeout on chitchat_id=node-1 [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.87/src/backtrace.rs:27:14 [INFO] [stdout] 1: cluster_test::Simulator::execute::{{closure}}::{{closure}} [INFO] [stdout] at ./tests/cluster_test.rs:150:29 [INFO] [stdout] 2: core::result::Result::map_err [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/result.rs:854:27 [INFO] [stdout] 3: cluster_test::Simulator::execute::{{closure}} [INFO] [stdout] at ./tests/cluster_test.rs:134:25 [INFO] [stdout] 4: cluster_test::test_marked_for_deletion_gc_with_network_partition::{{closure}} [INFO] [stdout] at ./tests/cluster_test.rs:443:35 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:696:57 [INFO] [stdout] 8: tokio::runtime::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:107:5 [INFO] [stdout] 9: tokio::runtime::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:73:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:696:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:423:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:695:36 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:774:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context.rs:180:26 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:283:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:260:9 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context.rs:180:9 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:774:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:683:19 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:191:28 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:179:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/runtime.rs:361:47 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/runtime.rs:335:13 [INFO] [stdout] 26: cluster_test::test_marked_for_deletion_gc_with_network_partition [INFO] [stdout] at ./tests/cluster_test.rs:443:5 [INFO] [stdout] 27: cluster_test::test_marked_for_deletion_gc_with_network_partition::{{closure}} [INFO] [stdout] at ./tests/cluster_test.rs:328:62 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/test/src/lib.rs:624:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/test/src/lib.rs:647:60 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 [INFO] [stdout] 33: std::panicking::try::do_call [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 [INFO] [stdout] 34: std::panicking::try [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/test/src/lib.rs:647:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/test/src/lib.rs:568:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/test/src/lib.rs:598:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:152:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:538:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 [INFO] [stdout] 42: std::panicking::try::do_call [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 [INFO] [stdout] 43: std::panicking::try [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:537:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9 [INFO] [stdout] 49: std::sys::pal::unix::thread::Thread::new::thread_start [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/pal/unix/thread.rs:108:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x60321d797035 - std::backtrace_rs::backtrace::libunwind::trace::h649ab3318d3445c5 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/../../backtrace/src/backtrace/libunwind.rs:116:5 [INFO] [stdout] 1: 0x60321d797035 - std::backtrace_rs::backtrace::trace_unsynchronized::hf4bb60c3387150c3 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5 [INFO] [stdout] 2: 0x60321d797035 - std::sys::backtrace::_print_fmt::hd9186c800e44bd00 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:65:5 [INFO] [stdout] 3: 0x60321d797035 - ::fmt::h1b9dad2a88e955ff [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:40:26 [INFO] [stdout] 4: 0x60321d7c040b - core::fmt::rt::Argument::fmt::h351a7824f737a6a0 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x60321d7c040b - core::fmt::write::h4b5a1270214bc4a7 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/fmt/mod.rs:1182:21 [INFO] [stdout] 6: 0x60321d79435f - std::io::Write::write_fmt::h9d1e399061051a36 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/io/mod.rs:1827:15 [INFO] [stdout] 7: 0x60321d798851 - std::sys::backtrace::BacktraceLock::print::h68d41b51481bce5c [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:43:9 [INFO] [stdout] 8: 0x60321d798851 - std::panicking::default_hook::{{closure}}::h96ab15e9936be7ed [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:269:22 [INFO] [stdout] 9: 0x60321d7983f8 - std::panicking::default_hook::h3cacb9c27561ad33 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:293:9 [INFO] [stdout] 10: 0x60321d5ed51a - as core::ops::function::Fn>::call::hd212b1446b2b2077 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2084:9 [INFO] [stdout] 11: 0x60321d5ed51a - test::test_main::{{closure}}::hd15ff34f3f68988b [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/test/src/lib.rs:136:21 [INFO] [stdout] 12: 0x60321d7990ef - as core::ops::function::Fn>::call::hce7569f4ca5d1b64 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2084:9 [INFO] [stdout] 13: 0x60321d7990ef - std::panicking::rust_panic_with_hook::hfe205f6954b2c97b [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:808:13 [INFO] [stdout] 14: 0x60321d798d17 - std::panicking::begin_panic_handler::{{closure}}::h6cb44b3a50f28c44 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:674:13 [INFO] [stdout] 15: 0x60321d7974f9 - std::sys::backtrace::__rust_end_short_backtrace::hf1c1f2a92799bb0e [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:168:18 [INFO] [stdout] 16: 0x60321d7989a4 - rust_begin_unwind [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:665:5 [INFO] [stdout] 17: 0x60321d4e9c93 - core::panicking::panic_fmt::h3d8fc78294164da7 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panicking.rs:74:14 [INFO] [stdout] 18: 0x60321d4ea206 - core::result::unwrap_failed::hfa79a499befff387 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/result.rs:1679:5 [INFO] [stdout] 19: 0x60321d4fd6f8 - core::result::Result::unwrap::h547da3909af27f33 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/result.rs:1102:23 [INFO] [stdout] 20: 0x60321d4fd6f8 - cluster_test::Simulator::execute::{{closure}}::h5dcdeea841c59c12 [INFO] [stdout] at /opt/rustwide/workdir/tests/cluster_test.rs:134:25 [INFO] [stdout] 21: 0x60321d509749 - cluster_test::test_marked_for_deletion_gc_with_network_partition::{{closure}}::h73bbbf7d632ccdad [INFO] [stdout] at /opt/rustwide/workdir/tests/cluster_test.rs:443:35 [INFO] [stdout] 22: 0x60321d533f32 - as core::future::future::Future>::poll::h87b784a5b2832d74 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9 [INFO] [stdout] 23: 0x60321d533fad - as core::future::future::Future>::poll::hff8fd0b206e4b3c4 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9 [INFO] [stdout] 24: 0x60321d54a1bf - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}}::h1f50fddfc1041b6c [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:696:57 [INFO] [stdout] 25: 0x60321d549f55 - tokio::runtime::coop::with_budget::hd9249d68f37a8480 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:107:5 [INFO] [stdout] 26: 0x60321d549f55 - tokio::runtime::coop::budget::hf910c75ae0eef1a7 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/coop.rs:73:5 [INFO] [stdout] 27: 0x60321d549f55 - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::h974418dabf45be46 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:696:25 [INFO] [stdout] 28: 0x60321d547da0 - tokio::runtime::scheduler::current_thread::Context::enter::hd0ebf9225425060d [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:423:19 [INFO] [stdout] 29: 0x60321d5497cb - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::hc11db7768bd8392a [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:695:36 [INFO] [stdout] 30: 0x60321d548c64 - tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}}::h6ae37cca156143d9 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:774:68 [INFO] [stdout] 31: 0x60321d552edb - tokio::runtime::context::scoped::Scoped::set::h93ad51a71e31090c [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 32: 0x60321d51dc49 - tokio::runtime::context::set_scheduler::{{closure}}::h209db782feb79a7e [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context.rs:180:26 [INFO] [stdout] 33: 0x60321d52038a - std::thread::local::LocalKey::try_with::he96a662ec3795e64 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:283:12 [INFO] [stdout] 34: 0x60321d51ee2a - std::thread::local::LocalKey::with::h5a917e2fabcfeb9c [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/local.rs:260:9 [INFO] [stdout] 35: 0x60321d51dbbd - tokio::runtime::context::set_scheduler::h5266710d8e8f2f24 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context.rs:180:9 [INFO] [stdout] 36: 0x60321d5489f0 - tokio::runtime::scheduler::current_thread::CoreGuard::enter::hf53592daad3819c4 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:774:27 [INFO] [stdout] 37: 0x60321d548ce3 - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::hcb0fd15a5151b711 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:683:19 [INFO] [stdout] 38: 0x60321d545f2a - tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}}::h009afe95fd58a95f [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:191:28 [INFO] [stdout] 39: 0x60321d5379c6 - tokio::runtime::context::runtime::enter_runtime::hcdc3bd00e172b534 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 40: 0x60321d545c31 - tokio::runtime::scheduler::current_thread::CurrentThread::block_on::h2c3e83fa509cc545 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/scheduler/current_thread/mod.rs:179:9 [INFO] [stdout] 41: 0x60321d53375d - tokio::runtime::runtime::Runtime::block_on_inner::hc0897932f2b603bf [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/runtime.rs:361:47 [INFO] [stdout] 42: 0x60321d533875 - tokio::runtime::runtime::Runtime::block_on::h451399fa6847bc40 [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.40.0/src/runtime/runtime.rs:335:13 [INFO] [stdout] 43: 0x60321d505dc0 - cluster_test::test_marked_for_deletion_gc_with_network_partition::ha75b3793096c27be [INFO] [stdout] at /opt/rustwide/workdir/tests/cluster_test.rs:443:5 [INFO] [stdout] 44: 0x60321d505c57 - cluster_test::test_marked_for_deletion_gc_with_network_partition::{{closure}}::h628d89ea4c24c276 [INFO] [stdout] at /opt/rustwide/workdir/tests/cluster_test.rs:328:62 [INFO] [stdout] 45: 0x60321d50e3c6 - core::ops::function::FnOnce::call_once::h7ac9bfa425260191 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 46: 0x60321d5f1b5b - core::ops::function::FnOnce::call_once::h81f56a195fe4862e [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: 0x60321d5f1b5b - test::__rust_begin_short_backtrace::h919c79c8b896f9e2 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/test/src/lib.rs:624:18 [INFO] [stdout] 48: 0x60321d5f1405 - test::run_test_in_process::{{closure}}::h7b3d5751c5b4dd75 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/test/src/lib.rs:647:60 [INFO] [stdout] 49: 0x60321d5f1405 - as core::ops::function::FnOnce<()>>::call_once::hdabd61465e4dbd80 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 [INFO] [stdout] 50: 0x60321d5f1405 - std::panicking::try::do_call::hc813c79fd64b0a90 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 [INFO] [stdout] 51: 0x60321d5f1405 - std::panicking::try::h055c5de7e7bfc209 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 [INFO] [stdout] 52: 0x60321d5f1405 - std::panic::catch_unwind::h4265d6525195c807 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 [INFO] [stdout] 53: 0x60321d5f1405 - test::run_test_in_process::he72c277a35f96567 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/test/src/lib.rs:647:27 [INFO] [stdout] 54: 0x60321d5f1405 - test::run_test::{{closure}}::h974e632522c0fbcf [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/test/src/lib.rs:568:43 [INFO] [stdout] 55: 0x60321d5b93f4 - test::run_test::{{closure}}::hdc2c89ce8b601dda [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/test/src/lib.rs:598:41 [INFO] [stdout] 56: 0x60321d5b93f4 - std::sys::backtrace::__rust_begin_short_backtrace::h342cb8e53aeb2076 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:152:18 [INFO] [stdout] 57: 0x60321d5bcb22 - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::h67b1b5c1709ad95b [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:538:17 [INFO] [stdout] 58: 0x60321d5bcb22 - as core::ops::function::FnOnce<()>>::call_once::hd8c7a030ea8b7676 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9 [INFO] [stdout] 59: 0x60321d5bcb22 - std::panicking::try::do_call::h512c2ab2c15b7d31 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40 [INFO] [stdout] 60: 0x60321d5bcb22 - std::panicking::try::h5c2903f8937bc868 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19 [INFO] [stdout] 61: 0x60321d5bcb22 - std::panic::catch_unwind::h242c80217c2dbece [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14 [INFO] [stdout] 62: 0x60321d5bcb22 - std::thread::Builder::spawn_unchecked_::{{closure}}::h6cb4494ebdd8caf7 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:537:30 [INFO] [stdout] 63: 0x60321d5bcb22 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h42193b008049ba94 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 64: 0x60321d79d43b - as core::ops::function::FnOnce>::call_once::ha1963004222e7822 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9 [INFO] [stdout] 65: 0x60321d79d43b - as core::ops::function::FnOnce>::call_once::h1086ced1f7c494c2 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9 [INFO] [stdout] 66: 0x60321d79d43b - std::sys::pal::unix::thread::Thread::new::thread_start::ha8af9c992ef0b208 [INFO] [stdout] at /rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/pal/unix/thread.rs:108:17 [INFO] [stdout] 67: 0x7a23b8787ac3 - [INFO] [stdout] 68: 0x7a23b8818a04 - clone [INFO] [stdout] 69: 0x0 - [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] test_marked_for_deletion_gc_with_network_partition [INFO] [stdout] [INFO] [stdout] test result: FAILED. 3 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 20.12s [INFO] [stdout] [INFO] [stderr] error: test failed, to rerun pass `--test cluster_test` [INFO] running `Command { std: "docker" "inspect" "d64fadade85f9131ce58cf19db44b02bc5dd784e1a1dfa85d7872fb2d972150a", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "d64fadade85f9131ce58cf19db44b02bc5dd784e1a1dfa85d7872fb2d972150a", kill_on_drop: false }` [INFO] [stdout] d64fadade85f9131ce58cf19db44b02bc5dd784e1a1dfa85d7872fb2d972150a