[INFO] fetching crate ai-workbench-lib 0.4.0... [INFO] testing ai-workbench-lib-0.4.0 against try#dec9417b8611e34e787a3e4c37686b5131f9e5c5 for pr-154210 [INFO] extracting crate ai-workbench-lib 0.4.0 into /workspace/builds/worker-6-tc2/source [INFO] started tweaking crates.io crate ai-workbench-lib 0.4.0 [INFO] finished tweaking crates.io crate ai-workbench-lib 0.4.0 [INFO] tweaked toml for crates.io crate ai-workbench-lib 0.4.0 written to /workspace/builds/worker-6-tc2/source/Cargo.toml [INFO] validating manifest of crates.io crate ai-workbench-lib 0.4.0 on toolchain dec9417b8611e34e787a3e4c37686b5131f9e5c5 [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+dec9417b8611e34e787a3e4c37686b5131f9e5c5" "metadata" "--manifest-path" "Cargo.toml" "--no-deps", kill_on_drop: false }` [INFO] crate crates.io crate ai-workbench-lib 0.4.0 already has a lockfile, it will not be regenerated [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+dec9417b8611e34e787a3e4c37686b5131f9e5c5" "fetch" "--manifest-path" "Cargo.toml", kill_on_drop: false }` [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:d429b63d4308055ea97f60fb1d3dfca48854a00942f1bd2ad806beaf015945ec" "/opt/rustwide/cargo-home/bin/cargo" "+dec9417b8611e34e787a3e4c37686b5131f9e5c5" "metadata" "--no-deps" "--format-version=1", kill_on_drop: false }` [INFO] [stdout] 3fd9a44529d589fdd205b705c4e560823c2cd52e44ec0902a76e4700ddc1c8b0 [INFO] running `Command { std: "docker" "start" "-a" "3fd9a44529d589fdd205b705c4e560823c2cd52e44ec0902a76e4700ddc1c8b0", kill_on_drop: false }` [INFO] running `Command { std: "docker" "inspect" "3fd9a44529d589fdd205b705c4e560823c2cd52e44ec0902a76e4700ddc1c8b0", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "3fd9a44529d589fdd205b705c4e560823c2cd52e44ec0902a76e4700ddc1c8b0", kill_on_drop: false }` [INFO] [stdout] 3fd9a44529d589fdd205b705c4e560823c2cd52e44ec0902a76e4700ddc1c8b0 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:d429b63d4308055ea97f60fb1d3dfca48854a00942f1bd2ad806beaf015945ec" "/opt/rustwide/cargo-home/bin/cargo" "+dec9417b8611e34e787a3e4c37686b5131f9e5c5" "build" "--frozen" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] 4b55db1f92045f89b9e70fed6458122d94e1febb1077ea93c8e36b93b2e5c535 [INFO] running `Command { std: "docker" "start" "-a" "4b55db1f92045f89b9e70fed6458122d94e1febb1077ea93c8e36b93b2e5c535", kill_on_drop: false }` [INFO] [stderr] Compiling libc v0.2.175 [INFO] [stderr] Compiling proc-macro2 v1.0.101 [INFO] [stderr] Compiling unicode-ident v1.0.18 [INFO] [stderr] Compiling cfg-if v1.0.3 [INFO] [stderr] Compiling bytes v1.10.1 [INFO] [stderr] Compiling lock_api v0.4.13 [INFO] [stderr] Compiling parking_lot_core v0.9.11 [INFO] [stderr] Compiling itoa v1.0.15 [INFO] [stderr] Compiling zeroize v1.8.1 [INFO] [stderr] Compiling memchr v2.7.5 [INFO] [stderr] Compiling find-msvc-tools v0.1.0 [INFO] [stderr] Compiling typenum v1.18.0 [INFO] [stderr] Compiling tracing-core v0.1.34 [INFO] [stderr] Compiling num-traits v0.2.19 [INFO] [stderr] Compiling powerfmt v0.2.0 [INFO] [stderr] Compiling ryu v1.0.20 [INFO] [stderr] Compiling http v1.3.1 [INFO] [stderr] Compiling http v0.2.12 [INFO] [stderr] Compiling deranged v0.5.3 [INFO] [stderr] Compiling quote v1.0.40 [INFO] [stderr] Compiling signal-hook-registry v1.4.6 [INFO] [stderr] Compiling socket2 v0.6.0 [INFO] [stderr] Compiling syn v2.0.106 [INFO] [stderr] Compiling parking_lot v0.12.4 [INFO] [stderr] Compiling mio v1.0.4 [INFO] [stderr] Compiling jobserver v0.1.34 [INFO] [stderr] Compiling getrandom v0.2.16 [INFO] [stderr] Compiling cc v1.2.35 [INFO] [stderr] Compiling http-body v1.0.1 [INFO] [stderr] Compiling generic-array v0.14.7 [INFO] [stderr] Compiling http-body v0.4.6 [INFO] [stderr] Compiling outref v0.5.2 [INFO] [stderr] Compiling either v1.15.0 [INFO] [stderr] Compiling num-conv v0.1.0 [INFO] [stderr] Compiling time-core v0.1.5 [INFO] [stderr] Compiling vsimd v0.8.0 [INFO] [stderr] Compiling http-body-util v0.1.3 [INFO] [stderr] Compiling time v0.3.42 [INFO] [stderr] Compiling num-integer v0.1.46 [INFO] [stderr] Compiling block-buffer v0.10.4 [INFO] [stderr] Compiling crypto-common v0.1.6 [INFO] [stderr] Compiling bytes-utils v0.1.4 [INFO] [stderr] Compiling digest v0.10.7 [INFO] [stderr] Compiling fs_extra v1.3.0 [INFO] [stderr] Compiling dunce v1.0.5 [INFO] [stderr] Compiling cmake v0.1.54 [INFO] [stderr] Compiling foldhash v0.1.5 [INFO] [stderr] Compiling base64-simd v0.8.0 [INFO] [stderr] Compiling rand_core v0.6.4 [INFO] [stderr] Compiling aws-lc-rs v1.13.3 [INFO] [stderr] Compiling getrandom v0.3.3 [INFO] [stderr] Compiling hashbrown v0.15.5 [INFO] [stderr] Compiling stable_deref_trait v1.2.0 [INFO] [stderr] Compiling der v0.6.1 [INFO] [stderr] Compiling rustls-pki-types v1.12.0 [INFO] [stderr] Compiling log v0.4.27 [INFO] [stderr] Compiling base64ct v1.8.0 [INFO] [stderr] Compiling rustls v0.21.12 [INFO] [stderr] Compiling rustls v0.23.31 [INFO] [stderr] Compiling semver v1.0.26 [INFO] [stderr] Compiling crc32fast v1.5.0 [INFO] [stderr] Compiling crypto-bigint v0.4.9 [INFO] [stderr] Compiling ring v0.17.14 [INFO] [stderr] Compiling aws-lc-sys v0.30.0 [INFO] [stderr] Compiling indexmap v2.11.0 [INFO] [stderr] Compiling ff v0.12.1 [INFO] [stderr] Compiling base16ct v0.1.1 [INFO] [stderr] Compiling base64 v0.21.7 [INFO] [stderr] Compiling spki v0.6.0 [INFO] [stderr] Compiling pkcs8 v0.9.0 [INFO] [stderr] Compiling rustls-pemfile v1.0.4 [INFO] [stderr] Compiling group v0.12.1 [INFO] [stderr] Compiling hmac v0.12.1 [INFO] [stderr] Compiling sec1 v0.3.0 [INFO] [stderr] Compiling socket2 v0.5.10 [INFO] [stderr] Compiling rfc6979 v0.3.1 [INFO] [stderr] Compiling rustc_version v0.4.1 [INFO] [stderr] Compiling rustls-native-certs v0.6.3 [INFO] [stderr] Compiling sha2 v0.10.9 [INFO] [stderr] Compiling elliptic-curve v0.12.3 [INFO] [stderr] Compiling rustls-native-certs v0.8.1 [INFO] [stderr] Compiling signature v1.6.4 [INFO] [stderr] Compiling tower v0.5.2 [INFO] [stderr] Compiling aws-types v1.3.8 [INFO] [stderr] Compiling litemap v0.8.0 [INFO] [stderr] Compiling writeable v0.6.1 [INFO] [stderr] Compiling crypto-bigint v0.5.5 [INFO] [stderr] Compiling ecdsa v0.14.8 [INFO] [stderr] Compiling zerocopy v0.8.26 [INFO] [stderr] Compiling icu_properties_data v2.0.1 [INFO] [stderr] Compiling icu_normalizer_data v2.0.0 [INFO] [stderr] Compiling uuid v1.18.1 [INFO] [stderr] Compiling p256 v0.11.1 [INFO] [stderr] Compiling serde v1.0.219 [INFO] [stderr] Compiling synstructure v0.13.2 [INFO] [stderr] Compiling regex-lite v0.1.7 [INFO] [stderr] Compiling rand_core v0.9.3 [INFO] [stderr] Compiling aho-corasick v1.1.3 [INFO] [stderr] Compiling regex-syntax v0.8.6 [INFO] [stderr] Compiling crc-catalog v2.4.0 [INFO] [stderr] Compiling xmlparser v0.13.6 [INFO] [stderr] Compiling aws-smithy-xml v0.60.10 [INFO] [stderr] Compiling crc v3.3.0 [INFO] [stderr] Compiling urlencoding v2.1.3 [INFO] [stderr] Compiling sha1 v0.10.6 [INFO] [stderr] Compiling ppv-lite86 v0.2.21 [INFO] [stderr] Compiling md-5 v0.10.6 [INFO] [stderr] Compiling tokio-macros v2.5.0 [INFO] [stderr] Compiling futures-macro v0.3.31 [INFO] [stderr] Compiling tracing-attributes v0.1.30 [INFO] [stderr] Compiling zerofrom-derive v0.1.6 [INFO] [stderr] Compiling tokio v1.47.1 [INFO] [stderr] Compiling futures-util v0.3.31 [INFO] [stderr] Compiling tracing v0.1.41 [INFO] [stderr] Compiling yoke-derive v0.8.0 [INFO] [stderr] Compiling zerofrom v0.1.6 [INFO] [stderr] Compiling zerovec-derive v0.11.1 [INFO] [stderr] Compiling sct v0.7.1 [INFO] [stderr] Compiling rustls-webpki v0.101.7 [INFO] [stderr] Compiling displaydoc v0.2.5 [INFO] [stderr] Compiling yoke v0.8.0 [INFO] [stderr] Compiling serde_derive v1.0.219 [INFO] [stderr] Compiling zerotrie v0.2.2 [INFO] [stderr] Compiling zerovec v0.11.4 [INFO] [stderr] Compiling regex-automata v0.4.10 [INFO] [stderr] Compiling rand_chacha v0.9.0 [INFO] [stderr] Compiling rand v0.9.2 [INFO] [stderr] Compiling serde_json v1.0.143 [INFO] [stderr] Compiling anyhow v1.0.99 [INFO] [stderr] Compiling tinystr v0.8.1 [INFO] [stderr] Compiling potential_utf v0.1.3 [INFO] [stderr] Compiling icu_collections v2.0.0 [INFO] [stderr] Compiling icu_locale_core v2.0.0 [INFO] [stderr] Compiling sharded-slab v0.1.7 [INFO] [stderr] Compiling futures-executor v0.3.31 [INFO] [stderr] Compiling icu_provider v2.0.0 [INFO] [stderr] Compiling lru v0.12.5 [INFO] [stderr] Compiling tracing-log v0.2.0 [INFO] [stderr] Compiling icu_normalizer v2.0.0 [INFO] [stderr] Compiling icu_properties v2.0.1 [INFO] [stderr] Compiling csv-core v0.1.12 [INFO] [stderr] Compiling thread_local v1.1.9 [INFO] [stderr] Compiling regex v1.11.2 [INFO] [stderr] Compiling nu-ansi-term v0.50.1 [INFO] [stderr] Compiling tokio-util v0.7.16 [INFO] [stderr] Compiling aws-smithy-async v1.2.5 [INFO] [stderr] Compiling tokio-rustls v0.24.1 [INFO] [stderr] Compiling unicode-id v0.3.5 [INFO] [stderr] Compiling markdown v1.0.0 [INFO] [stderr] Compiling aws-smithy-types v1.3.2 [INFO] [stderr] Compiling h2 v0.4.12 [INFO] [stderr] Compiling h2 v0.3.27 [INFO] [stderr] Compiling aws-smithy-runtime-api v1.9.0 [INFO] [stderr] Compiling aws-smithy-eventstream v0.60.11 [INFO] [stderr] Compiling aws-smithy-json v0.61.5 [INFO] [stderr] Compiling idna_adapter v1.2.1 [INFO] [stderr] Compiling idna v1.1.0 [INFO] [stderr] Compiling crc-fast v1.3.0 [INFO] [stderr] Compiling aws-smithy-http v0.62.3 [INFO] [stderr] Compiling aws-credential-types v1.2.6 [INFO] [stderr] Compiling aws-smithy-observability v0.1.3 [INFO] [stderr] Compiling hyper v1.7.0 [INFO] [stderr] Compiling aws-sigv4 v1.3.4 [INFO] [stderr] Compiling url v2.5.7 [INFO] [stderr] Compiling aws-smithy-query v0.60.7 [INFO] [stderr] Compiling hyper v0.14.32 [INFO] [stderr] Compiling aws-smithy-checksums v0.63.8 [INFO] [stderr] Compiling hyper-util v0.1.16 [INFO] [stderr] Compiling csv v1.3.1 [INFO] [stderr] Compiling tracing-subscriber v0.3.20 [INFO] [stderr] Compiling futures v0.3.31 [INFO] [stderr] Compiling hyper-rustls v0.24.2 [INFO] [stderr] Compiling rustls-webpki v0.103.4 [INFO] [stderr] Compiling tokio-rustls v0.26.2 [INFO] [stderr] Compiling hyper-rustls v0.27.7 [INFO] [stderr] Compiling aws-smithy-http-client v1.1.1 [INFO] [stderr] Compiling aws-smithy-runtime v1.9.1 [INFO] [stderr] Compiling aws-runtime v1.5.10 [INFO] [stderr] Compiling aws-sdk-sts v1.85.0 [INFO] [stderr] Compiling aws-sdk-sso v1.83.0 [INFO] [stderr] Compiling aws-sdk-ssooidc v1.84.0 [INFO] [stderr] Compiling aws-sdk-s3 v1.104.0 [INFO] [stderr] Compiling aws-sdk-bedrockruntime v1.105.0 [INFO] [stderr] Compiling aws-config v1.8.6 [INFO] [stderr] Compiling ai-workbench-lib v0.4.0 (/opt/rustwide/workdir) [INFO] [stderr] Finished `dev` profile [unoptimized + debuginfo] target(s) in 5m 01s [INFO] running `Command { std: "docker" "inspect" "4b55db1f92045f89b9e70fed6458122d94e1febb1077ea93c8e36b93b2e5c535", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "4b55db1f92045f89b9e70fed6458122d94e1febb1077ea93c8e36b93b2e5c535", kill_on_drop: false }` [INFO] [stdout] 4b55db1f92045f89b9e70fed6458122d94e1febb1077ea93c8e36b93b2e5c535 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:d429b63d4308055ea97f60fb1d3dfca48854a00942f1bd2ad806beaf015945ec" "/opt/rustwide/cargo-home/bin/cargo" "+dec9417b8611e34e787a3e4c37686b5131f9e5c5" "test" "--frozen" "--no-run" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] a46585c6ab9151f43d7684ea1856b43f8bd3f063298fc96e29d7cbe8ff34e29e [INFO] running `Command { std: "docker" "start" "-a" "a46585c6ab9151f43d7684ea1856b43f8bd3f063298fc96e29d7cbe8ff34e29e", kill_on_drop: false }` [INFO] [stderr] Compiling ai-workbench-lib v0.4.0 (/opt/rustwide/workdir) [INFO] [stderr] Finished `test` profile [unoptimized + debuginfo] target(s) in 9.19s [INFO] running `Command { std: "docker" "inspect" "a46585c6ab9151f43d7684ea1856b43f8bd3f063298fc96e29d7cbe8ff34e29e", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "a46585c6ab9151f43d7684ea1856b43f8bd3f063298fc96e29d7cbe8ff34e29e", kill_on_drop: false }` [INFO] [stdout] a46585c6ab9151f43d7684ea1856b43f8bd3f063298fc96e29d7cbe8ff34e29e [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:d429b63d4308055ea97f60fb1d3dfca48854a00942f1bd2ad806beaf015945ec" "/opt/rustwide/cargo-home/bin/cargo" "+dec9417b8611e34e787a3e4c37686b5131f9e5c5" "test" "--frozen", kill_on_drop: false }` [INFO] [stdout] 80dbf007c83d42125fcce01c6b49d9fa3702a2c36ed41331a256f5346949dc74 [INFO] running `Command { std: "docker" "start" "-a" "80dbf007c83d42125fcce01c6b49d9fa3702a2c36ed41331a256f5346949dc74", kill_on_drop: false }` [INFO] [stderr] Finished `test` profile [unoptimized + debuginfo] target(s) in 0.37s [INFO] [stderr] Running unittests src/lib.rs (/opt/rustwide/target/debug/deps/ai_workbench_lib-ef2dcd3deb1be477) [INFO] [stdout] [INFO] [stdout] running 37 tests [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_file_info_structure ... ok [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_large_csv_file_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_edge_cases ... FAILED [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_mixed_file_types ... FAILED [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_processing_summary ... ok [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_single_file_discovery ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_binary_file_rejection ... FAILED [INFO] [stdout] test modules::file_splitter::splitter::tests::test_empty_file ... ok [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_file_filtering ... ok [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_folder_discovery ... ok [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_large_folder_discovery ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_invalid_csv_fallback ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_json_file_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_csv_format_validation ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_csv_multiple_chunks_with_headers ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_csv_single_chunk ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_record_boundary_preservation ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_tsv_format ... ok [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_large_text_file_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection ... FAILED [INFO] [stdout] test modules::file_splitter::splitter_types::code_splitter::tests::test_large_file_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_json_lines_detection ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::text_splitter::tests::test_large_file_multiple_chunks ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_json_lines_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection ... FAILED [INFO] [stdout] test modules::file_splitter::splitter_types::code_splitter::tests::test_simple_code_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_file_type_detection_by_extension ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection ... FAILED [INFO] [stdout] test modules::file_splitter::splitter_types::text_splitter::tests::test_line_boundary_preservation ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::text_splitter::tests::test_small_file_single_chunk ... ok [INFO] [stdout] 2026-03-31T21:32:30.788969Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/script.py': dispatch failure [INFO] [stdout] 2026-03-31T21:32:30.789023Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2026-03-31T21:32:30.789055Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2026-03-31T21:32:30.789066Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/script.py' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.789082Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2026-03-31T21:32:30.789093Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2026-03-31T21:32:30.789103Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2026-03-31T21:32:30.789111Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2026-03-31T21:32:30.789193Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/script.py': Failed to get metadata for test-data/script.py [INFO] [stdout] 2026-03-31T21:32:30.789219Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2026-03-31T21:32:30.789230Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/script.py [INFO] [stdout] 2026-03-31T21:32:30.789242Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2026-03-31T21:32:30.789255Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2026-03-31T21:32:30.789263Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2026-03-31T21:32:30.789275Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/script.py [INFO] [stdout] 2026-03-31T21:32:30.801486Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.801539Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/' [INFO] [stdout] 2026-03-31T21:32:30.801551Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.801562Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2026-03-31T21:32:30.804389Z  INFO ai_workbench_lib::modules::job_processor::core: Starting job execution for job_id: test-large-dbbf6208-a507-461c-8d85-26c53ec27f8b [INFO] [stdout] 2026-03-31T21:32:30.804647Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.804709Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'large_files/large_document.txt' [INFO] [stdout] 2026-03-31T21:32:30.804743Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.804871Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large_files/large_document.txt [INFO] [stdout] 2026-03-31T21:32:30.804892Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2026-03-31T21:32:30.805126Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2026-03-31T21:32:30.805361Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.805379Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'large_files/large_document.txt' [INFO] [stdout] 2026-03-31T21:32:30.805389Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.805401Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2026-03-31T21:32:30.805434Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.805446Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'large_files/large_document.txt' [INFO] [stdout] 2026-03-31T21:32:30.805455Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large_files/large_document.txt [INFO] [stdout] 2026-03-31T21:32:30.805471Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2026-03-31T21:32:30.805501Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.805514Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'large_files/large_document.txt' [INFO] [stdout] 2026-03-31T21:32:30.805524Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.805533Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large_files/large_document.txt [INFO] [stdout] 2026-03-31T21:32:30.805544Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2026-03-31T21:32:30.813703Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'large_files/large_document.txt': dispatch failure [INFO] [stdout] 2026-03-31T21:32:30.815608Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2026-03-31T21:32:30.815691Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2026-03-31T21:32:30.815728Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'large_files/large_document.txt' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.815810Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2026-03-31T21:32:30.815846Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2026-03-31T21:32:30.815894Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2026-03-31T21:32:30.815923Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2026-03-31T21:32:30.817503Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.817561Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/sample.txt' [INFO] [stdout] 2026-03-31T21:32:30.817609Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.817663Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/sample.txt [INFO] [stdout] 2026-03-31T21:32:30.817696Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2026-03-31T21:32:30.818048Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2026-03-31T21:32:30.818101Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.818137Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/sample.txt' [INFO] [stdout] 2026-03-31T21:32:30.818196Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.818297Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2026-03-31T21:32:30.818381Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.818484Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'text_files/sample.txt' [INFO] [stdout] 2026-03-31T21:32:30.818542Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/sample.txt [INFO] [stdout] 2026-03-31T21:32:30.818988Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2026-03-31T21:32:30.819136Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.822370Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'large_files/large_document.txt': Failed to get metadata for large_files/large_document.txt [INFO] [stdout] 2026-03-31T21:32:30.824676Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2026-03-31T21:32:30.824614Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'text_files/sample.txt' [INFO] [stdout] 2026-03-31T21:32:30.824759Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.824793Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large_files/large_document.txt [INFO] [stdout] 2026-03-31T21:32:30.824866Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2026-03-31T21:32:30.824895Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/sample.txt [INFO] [stdout] 2026-03-31T21:32:30.824969Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2026-03-31T21:32:30.824953Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2026-03-31T21:32:30.825063Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2026-03-31T21:32:30.825156Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: large_files/large_document.txt [INFO] [stdout] 2026-03-31T21:32:30.825637Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.825685Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' [INFO] [stdout] 2026-03-31T21:32:30.825699Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.825718Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2026-03-31T21:32:30.825733Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2026-03-31T21:32:30.825753Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2026-03-31T21:32:30.825766Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.825787Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' [INFO] [stdout] 2026-03-31T21:32:30.825798Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.825811Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2026-03-31T21:32:30.825826Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.825837Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' [INFO] [stdout] 2026-03-31T21:32:30.825858Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2026-03-31T21:32:30.825872Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2026-03-31T21:32:30.825914Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.825935Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' [INFO] [stdout] 2026-03-31T21:32:30.825948Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.825968Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2026-03-31T21:32:30.825982Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2026-03-31T21:32:30.835539Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'text_files/sample.txt': dispatch failure [INFO] [stdout] 2026-03-31T21:32:30.835722Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2026-03-31T21:32:30.835875Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2026-03-31T21:32:30.835926Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'text_files/sample.txt' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.835961Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2026-03-31T21:32:30.836013Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2026-03-31T21:32:30.836059Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2026-03-31T21:32:30.836087Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2026-03-31T21:32:30.836191Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'text_files/sample.txt': Failed to get metadata for text_files/sample.txt [INFO] [stdout] 2026-03-31T21:32:30.836323Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2026-03-31T21:32:30.836366Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/sample.txt [INFO] [stdout] 2026-03-31T21:32:30.836400Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2026-03-31T21:32:30.836427Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2026-03-31T21:32:30.836486Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2026-03-31T21:32:30.836574Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: text_files/sample.txt [INFO] [stdout] 2026-03-31T21:32:30.825287Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: large_files/large_document.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for large_files/large_document.txt [INFO] [stdout] 1: S3 HEAD request failed for 'large_files/large_document.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::get_file_metadata::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ::process_single_file::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ::run::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:57:43 [INFO] [stdout] 6: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:882:31 [INFO] [stdout] 7: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 9: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 10: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 11: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 12: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 13: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 14: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 15: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 16: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 18: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 19: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 21: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 22: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 23: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 24: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 25: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 26: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 27: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file [INFO] [stdout] at ./src/../modules/job_processor/core.rs:882:37 [INFO] [stdout] 29: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:854:50 [INFO] [stdout] 30: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 32: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 33: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 34: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 35: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 36: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 37: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 38: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 39: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 40: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 41: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 42: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 43: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 44: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 45: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 46: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 47: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 48: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 49: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 50: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 51: [INFO] [stdout] 52: clone [INFO] [stdout] 2026-03-31T21:32:30.839933Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2026-03-31T21:32:30.839961Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to get metadata for single file: large_files/large_document.txt [INFO] [stdout] 2026-03-31T21:32:30.836633Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: text_files/sample.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for text_files/sample.txt [INFO] [stdout] 1: S3 HEAD request failed for 'text_files/sample.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::get_file_metadata::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ::process_single_file::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1024:46 [INFO] [stdout] 6: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 18: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1017:31 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:980:52 [INFO] [stdout] 29: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 35: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 36: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 41: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 42: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 44: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 45: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 47: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 49: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2026-03-31T21:32:30.842098Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2026-03-31T21:32:30.842231Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to get metadata for single file: text_files/sample.txt [INFO] [stdout] 2026-03-31T21:32:30.801575Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2026-03-31T21:32:30.842635Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2026-03-31T21:32:30.842663Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.842687Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/' [INFO] [stdout] 2026-03-31T21:32:30.842697Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.842720Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2026-03-31T21:32:30.842735Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.842746Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'text_files/' [INFO] [stdout] 2026-03-31T21:32:30.842772Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2026-03-31T21:32:30.842785Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2026-03-31T21:32:30.842810Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] 2026-03-31T21:32:30.842326Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to get metadata for single file: text_files/sample.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for text_files/sample.txt [INFO] [stdout] 1: S3 HEAD request failed for 'text_files/sample.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::get_file_metadata::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ::process_single_file::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1024:46 [INFO] [stdout] 6: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 18: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1017:31 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:980:52 [INFO] [stdout] 29: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 35: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 36: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 41: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 42: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 44: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 45: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 47: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 49: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2026-03-31T21:32:30.847467Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2026-03-31T21:32:30.847574Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.847645Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/' [INFO] [stdout] 2026-03-31T21:32:30.847675Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.847704Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2026-03-31T21:32:30.847766Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2026-03-31T21:32:30.847796Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2026-03-31T21:32:30.847825Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.847882Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/' [INFO] [stdout] 2026-03-31T21:32:30.847910Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.847942Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2026-03-31T21:32:30.848007Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.848037Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'text_files/' [INFO] [stdout] 2026-03-31T21:32:30.848084Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2026-03-31T21:32:30.848129Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2026-03-31T21:32:30.848164Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] 2026-03-31T21:32:30.850133Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2026-03-31T21:32:30.852614Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2026-03-31T21:32:30.852755Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2026-03-31T21:32:30.852846Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'text_files/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.853167Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2026-03-31T21:32:30.853283Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2026-03-31T21:32:30.853529Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2026-03-31T21:32:30.853802Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2026-03-31T21:32:30.854670Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2026-03-31T21:32:30.855461Z  INFO ai_workbench_lib::modules::job_processor::core: Starting job execution for job_id: test-folder-67bf2751-830d-4b92-a27a-85bfbe17b3ff [INFO] [stdout] 2026-03-31T21:32:30.857625Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.857826Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/' [INFO] [stdout] 2026-03-31T21:32:30.789295Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/script.py [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/script.py [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/script.py': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::get_file_metadata::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ::process_single_file::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 17: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 34: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 35: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 40: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 41: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 43: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 44: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 46: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 48: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2026-03-31T21:32:30.858535Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2026-03-31T21:32:30.839978Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to get metadata for single file: large_files/large_document.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for large_files/large_document.txt [INFO] [stdout] 1: S3 HEAD request failed for 'large_files/large_document.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::get_file_metadata::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ::process_single_file::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ::run::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:57:43 [INFO] [stdout] 6: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:882:31 [INFO] [stdout] 7: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 9: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 10: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 11: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 12: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 13: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 14: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 15: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 16: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 18: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 19: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 20: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 21: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 22: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 23: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 24: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 25: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 26: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 27: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file [INFO] [stdout] at ./src/../modules/job_processor/core.rs:882:37 [INFO] [stdout] 29: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:854:50 [INFO] [stdout] 30: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 32: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 33: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 34: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 35: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 36: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 37: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 38: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 39: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 40: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 41: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 42: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 43: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 44: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 45: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 46: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 47: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 48: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 49: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 50: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 51: [INFO] [stdout] 52: clone [INFO] [stdout] 2026-03-31T21:32:30.858815Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2026-03-31T21:32:30.862445Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt': dispatch failure [INFO] [stdout] 2026-03-31T21:32:30.854861Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::process_folder::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1024:46 [INFO] [stdout] 5: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 17: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1017:31 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:980:52 [INFO] [stdout] 28: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 34: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 35: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 40: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 41: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 43: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 44: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 46: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 48: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2026-03-31T21:32:30.869668Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2026-03-31T21:32:30.869627Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2026-03-31T21:32:30.869758Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2026-03-31T21:32:30.869806Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.869851Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2026-03-31T21:32:30.869877Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2026-03-31T21:32:30.869902Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2026-03-31T21:32:30.869951Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2026-03-31T21:32:30.870055Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt': Failed to get metadata for text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2026-03-31T21:32:30.870101Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2026-03-31T21:32:30.870124Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2026-03-31T21:32:30.870151Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2026-03-31T21:32:30.870199Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2026-03-31T21:32:30.870232Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2026-03-31T21:32:30.870284Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2026-03-31T21:32:30.869687Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to list objects in S3 [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file ... ok [INFO] [stdout] 2026-03-31T21:32:30.877091Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2026-03-31T21:32:30.877123Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2026-03-31T21:32:30.877150Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2026-03-31T21:32:30.877160Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'text_files/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.877203Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2026-03-31T21:32:30.877231Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2026-03-31T21:32:30.877260Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2026-03-31T21:32:30.877287Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2026-03-31T21:32:30.877367Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2026-03-31T21:32:30.858609Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.877869Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2026-03-31T21:32:30.865902Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.877954Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/code.rs' [INFO] [stdout] 2026-03-31T21:32:30.877997Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.877981Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2026-03-31T21:32:30.878062Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2026-03-31T21:32:30.878110Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.878048Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2026-03-31T21:32:30.878164Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.878188Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/code.rs' [INFO] [stdout] 2026-03-31T21:32:30.878240Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/code.rs [INFO] [stdout] 2026-03-31T21:32:30.878254Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2026-03-31T21:32:30.878265Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.878274Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/code.rs' [INFO] [stdout] 2026-03-31T21:32:30.878284Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.878294Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/code.rs [INFO] [stdout] 2026-03-31T21:32:30.878305Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2026-03-31T21:32:30.878225Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/' [INFO] [stdout] 2026-03-31T21:32:30.878662Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.878715Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2026-03-31T21:32:30.878763Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.878801Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'text_files/' [INFO] [stdout] 2026-03-31T21:32:30.878829Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2026-03-31T21:32:30.878872Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2026-03-31T21:32:30.878903Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] 2026-03-31T21:32:30.870507Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::process_folder::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1024:46 [INFO] [stdout] 5: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 17: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1017:31 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:980:52 [INFO] [stdout] 28: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 34: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 35: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 40: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 41: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 43: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 44: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 46: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 48: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2026-03-31T21:32:30.879727Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2026-03-31T21:32:30.884467Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/code.rs': dispatch failure [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_config_validation ... ok [INFO] [stdout] 2026-03-31T21:32:30.870331Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 1: S3 HEAD request failed for 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::get_file_metadata::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ::process_single_file::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:673:55 [INFO] [stdout] 6: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 18: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end [INFO] [stdout] at ./src/../modules/job_processor/core.rs:710:37 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:638:57 [INFO] [stdout] 29: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 35: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 36: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 41: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 42: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 44: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 45: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 47: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 49: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2026-03-31T21:32:30.877417Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::process_folder::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:955:42 [INFO] [stdout] 5: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 17: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery [INFO] [stdout] at ./src/../modules/job_processor/core.rs:955:48 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:930:49 [INFO] [stdout] 28: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 34: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 35: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 40: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 41: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 43: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 44: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 46: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 48: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2026-03-31T21:32:30.901638Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2026-03-31T21:32:30.901947Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2026-03-31T21:32:30.901965Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/code.rs' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.901980Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2026-03-31T21:32:30.902016Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2026-03-31T21:32:30.902027Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2026-03-31T21:32:30.902037Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2026-03-31T21:32:30.902131Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/code.rs': Failed to get metadata for test-data/code.rs [INFO] [stdout] 2026-03-31T21:32:30.902150Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2026-03-31T21:32:30.902160Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/code.rs [INFO] [stdout] 2026-03-31T21:32:30.902174Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2026-03-31T21:32:30.902184Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2026-03-31T21:32:30.902193Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2026-03-31T21:32:30.902211Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/code.rs [INFO] [stdout] 2026-03-31T21:32:30.902224Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/code.rs [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/code.rs [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/code.rs': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::get_file_metadata::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ::process_single_file::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 17: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 34: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 35: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 40: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 41: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 43: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 44: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 46: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 48: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2026-03-31T21:32:30.903421Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2026-03-31T21:32:30.903462Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.903475Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/styles.css' [INFO] [stdout] 2026-03-31T21:32:30.903486Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.903498Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2026-03-31T21:32:30.903512Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.903523Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/styles.css' [INFO] [stdout] 2026-03-31T21:32:30.903533Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/styles.css [INFO] [stdout] 2026-03-31T21:32:30.903546Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2026-03-31T21:32:30.903558Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.903568Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/styles.css' [INFO] [stdout] 2026-03-31T21:32:30.903595Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.903606Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/styles.css [INFO] [stdout] 2026-03-31T21:32:30.903617Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2026-03-31T21:32:30.905565Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2026-03-31T21:32:30.901900Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2026-03-31T21:32:30.908631Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2026-03-31T21:32:30.927723Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to list objects in S3 [INFO] [stdout] 2026-03-31T21:32:30.924738Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2026-03-31T21:32:30.928630Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2026-03-31T21:32:30.928645Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'text_files/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.928659Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2026-03-31T21:32:30.928669Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2026-03-31T21:32:30.928679Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2026-03-31T21:32:30.928690Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2026-03-31T21:32:30.927750Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::process_folder::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:955:42 [INFO] [stdout] 5: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 17: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery [INFO] [stdout] at ./src/../modules/job_processor/core.rs:955:48 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:930:49 [INFO] [stdout] 28: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 34: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 35: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 40: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 41: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 43: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 44: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 46: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 48: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2026-03-31T21:32:30.928718Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2026-03-31T21:32:30.908813Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/styles.css': dispatch failure [INFO] [stdout] 2026-03-31T21:32:30.928753Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2026-03-31T21:32:30.928770Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2026-03-31T21:32:30.928778Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2026-03-31T21:32:30.928787Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/styles.css' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.928799Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2026-03-31T21:32:30.928809Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2026-03-31T21:32:30.928818Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2026-03-31T21:32:30.928827Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2026-03-31T21:32:30.928855Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to get metadata for single file: text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2026-03-31T21:32:30.929978Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/styles.css': Failed to get metadata for test-data/styles.css [INFO] [stdout] 2026-03-31T21:32:30.928784Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::process_folder::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ::run::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:57:43 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:787:31 [INFO] [stdout] 6: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 18: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end [INFO] [stdout] at ./src/../modules/job_processor/core.rs:787:37 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:759:52 [INFO] [stdout] 29: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 35: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 36: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 41: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 42: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 44: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 45: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 47: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 49: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2026-03-31T21:32:30.930006Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2026-03-31T21:32:30.930020Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/styles.css [INFO] [stdout] 2026-03-31T21:32:30.930021Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2026-03-31T21:32:30.930032Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2026-03-31T21:32:30.930041Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2026-03-31T21:32:30.930038Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to list objects in S3 [INFO] [stdout] 2026-03-31T21:32:30.930049Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2026-03-31T21:32:30.930059Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/styles.css [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery ... ok [INFO] [stdout] 2026-03-31T21:32:30.930070Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/styles.css [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/styles.css [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/styles.css': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::get_file_metadata::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ::process_single_file::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 17: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 34: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 35: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 40: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 41: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 43: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 44: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 46: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 48: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2026-03-31T21:32:30.931142Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2026-03-31T21:32:30.931180Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.931191Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/markup.html' [INFO] [stdout] 2026-03-31T21:32:30.931210Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.931223Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2026-03-31T21:32:30.931236Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.931246Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/markup.html' [INFO] [stdout] 2026-03-31T21:32:30.931256Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/markup.html [INFO] [stdout] 2026-03-31T21:32:30.931267Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2026-03-31T21:32:30.931278Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2026-03-31T21:32:30.931287Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/markup.html' [INFO] [stdout] 2026-03-31T21:32:30.931297Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.931308Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/markup.html [INFO] [stdout] 2026-03-31T21:32:30.931321Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2026-03-31T21:32:30.928871Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to get metadata for single file: text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 1: S3 HEAD request failed for 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::get_file_metadata::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ::process_single_file::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:673:55 [INFO] [stdout] 6: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 18: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end [INFO] [stdout] at ./src/../modules/job_processor/core.rs:710:37 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:638:57 [INFO] [stdout] 29: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 35: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 36: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 41: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 42: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 44: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 45: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 47: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 49: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2026-03-31T21:32:30.932484Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end ... ok [INFO] [stdout] 2026-03-31T21:32:30.935636Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/markup.html': dispatch failure [INFO] [stdout] 2026-03-31T21:32:30.935696Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2026-03-31T21:32:30.935741Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2026-03-31T21:32:30.935801Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/markup.html' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2026-03-31T21:32:30.935834Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2026-03-31T21:32:30.935881Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2026-03-31T21:32:30.935927Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2026-03-31T21:32:30.935958Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2026-03-31T21:32:30.936051Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/markup.html': Failed to get metadata for test-data/markup.html [INFO] [stdout] 2026-03-31T21:32:30.936106Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2026-03-31T21:32:30.936137Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/markup.html [INFO] [stdout] 2026-03-31T21:32:30.936193Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2026-03-31T21:32:30.936246Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2026-03-31T21:32:30.936279Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2026-03-31T21:32:30.936335Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/markup.html [INFO] [stdout] 2026-03-31T21:32:30.936384Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/markup.html [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/markup.html [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/markup.html': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::get_file_metadata::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ::process_single_file::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 17: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 34: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 35: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 40: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 41: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 43: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 44: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 46: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 48: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2026-03-31T21:32:30.937624Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_file_type_detection ... ok [INFO] [stdout] 2026-03-31T21:32:30.930051Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: ::msg:: [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ::process_folder::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ::discover_files::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ::run::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:57:43 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:787:31 [INFO] [stdout] 6: > as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: >> as core::future::future::Future>::poll [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: ::block_on::>>>::{closure#0}::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget::, ::block_on>>>::{closure#0}::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: ::block_on::>>>::{closure#0}::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: ::enter::, ::block_on>>>::{closure#0}::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: ::block_on::>>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: >::set::<::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: >::try_with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:462:12 [INFO] [stdout] 18: >::with::, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}>::{closure#0}, (alloc::boxed::Box, core::option::Option<()>)> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/local.rs:426:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler::<(alloc::boxed::Box, core::option::Option<()>), ::enter<::block_on>>>::{closure#0}, core::option::Option<()>>::{closure#0}> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: ::enter::<::block_on>>>::{closure#0}, core::option::Option<()>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: ::block_on::>>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: ::block_on::>>::{closure#0} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime::<::block_on>>::{closure#0}, ()> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: ::block_on_inner::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: ::block_on::>> [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end [INFO] [stdout] at ./src/../modules/job_processor/core.rs:787:37 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end::{closure#0} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:759:52 [INFO] [stdout] 29: >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::result::Result<(), alloc::string::String> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace::, fn() -> core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 35: std::panicking::catch_unwind::, core::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 36: std::panic::catch_unwind::, core::result::Result<(), alloc::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 41: std::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 42: ::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 44: std::panicking::catch_unwind::<(), core::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 45: std::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 47: ::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: + core::marker::Send> as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 49: ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2026-03-31T21:32:30.942038Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end ... ok [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_performance_characteristics ... FAILED [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter::integration_tests::test_edge_cases stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter::integration_tests::test_edge_cases' (24) panicked at src/../modules/file_splitter/splitter.rs:263:9: [INFO] [stdout] assertion failed: chunks[0].metadata.has_headers [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::libunwind::trace [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::trace_unsynchronized:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62606ba5f58a - std[e72de78501789eb0]::sys::backtrace::_print_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:74:9 [INFO] [stdout] 3: 0x62606ba5f58a - <::print::DisplayBacktrace as core[6aaeda2c7ad07a48]::fmt::Display>::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:44:26 [INFO] [stdout] 4: 0x62606ba788aa - ::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/fmt/rt.rs:152:76 [INFO] [stdout] 5: 0x62606ba788aa - core[6aaeda2c7ad07a48]::fmt::write [INFO] [stdout] 6: 0x62606ba64fc2 - std[e72de78501789eb0]::io::default_write_fmt::> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62606ba64fc2 - as std[e72de78501789eb0]::io::Write>::write_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:1994:13 [INFO] [stdout] 8: 0x62606ba3c67f - ::print [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:47:9 [INFO] [stdout] 9: 0x62606ba3c67f - std[e72de78501789eb0]::panicking::default_hook::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:292:27 [INFO] [stdout] 10: 0x62606ba56c39 - std[e72de78501789eb0]::panicking::default_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:316:9 [INFO] [stdout] 11: 0x62606a79fa8c - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 12: 0x62606a79fa8c - test[4f12b380f1fb68c5]::test_main_with_exit_callback::::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62606ba56eb2 - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 14: 0x62606ba56eb2 - std[e72de78501789eb0]::panicking::panic_with_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:833:13 [INFO] [stdout] 15: 0x62606ba3c76a - std[e72de78501789eb0]::panicking::panic_handler::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:691:13 [INFO] [stdout] 16: 0x62606ba338b9 - std[e72de78501789eb0]::sys::backtrace::__rust_end_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:182:18 [INFO] [stdout] 17: 0x62606ba3d6fd - __rustc[78a4cf188af856fc]::rust_begin_unwind [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:689:5 [INFO] [stdout] 18: 0x62606ba7909c - core[6aaeda2c7ad07a48]::panicking::panic_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62606ba79062 - core[6aaeda2c7ad07a48]::panicking::panic [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:150:5 [INFO] [stdout] 20: 0x62606a74b15e - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter::integration_tests::test_edge_cases [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:263:9 [INFO] [stdout] 21: 0x62606a7461f7 - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter::integration_tests::test_edge_cases::{closure#0} [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:251:25 [INFO] [stdout] 22: 0x62606a6f9d26 - >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 23: 0x62606a793b6b - core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62606a793b6b - test[4f12b380f1fb68c5]::__rust_begin_short_backtrace::, fn() -> core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 25: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 26: 0x62606a7a055b - as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 27: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::do_call::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 28: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::, core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 29: 0x62606a7a055b - std[e72de78501789eb0]::panic::catch_unwind::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 30: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 31: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 32: 0x62606a79bc74 - test[4f12b380f1fb68c5]::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 33: 0x62606a79bc74 - std[e72de78501789eb0]::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 34: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 35: 0x62606a7a3162 - ::{closure#1}::{closure#0}> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 36: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 37: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::<(), core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 38: 0x62606a7a3162 - std[e72de78501789eb0]::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 39: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 40: 0x62606a7a3162 - ::{closure#1} as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 41: 0x62606ba5e63f - + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 42: 0x62606ba5e63f - ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 43: 0x77550bae5aa4 - [INFO] [stdout] 44: 0x77550bb72a64 - clone [INFO] [stdout] 45: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter::integration_tests::test_mixed_file_types stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter::integration_tests::test_mixed_file_types' (27) panicked at src/../modules/file_splitter/splitter.rs:245:13: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: Text [INFO] [stdout] right: Xml [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::libunwind::trace [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::trace_unsynchronized:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62606ba5f58a - std[e72de78501789eb0]::sys::backtrace::_print_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:74:9 [INFO] [stdout] 3: 0x62606ba5f58a - <::print::DisplayBacktrace as core[6aaeda2c7ad07a48]::fmt::Display>::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:44:26 [INFO] [stdout] 4: 0x62606ba788aa - ::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/fmt/rt.rs:152:76 [INFO] [stdout] 5: 0x62606ba788aa - core[6aaeda2c7ad07a48]::fmt::write [INFO] [stdout] 6: 0x62606ba64fc2 - std[e72de78501789eb0]::io::default_write_fmt::> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62606ba64fc2 - as std[e72de78501789eb0]::io::Write>::write_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:1994:13 [INFO] [stdout] 8: 0x62606ba3c67f - ::print [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:47:9 [INFO] [stdout] 9: 0x62606ba3c67f - std[e72de78501789eb0]::panicking::default_hook::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:292:27 [INFO] [stdout] 10: 0x62606ba56c39 - std[e72de78501789eb0]::panicking::default_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:316:9 [INFO] [stdout] 11: 0x62606a79fa8c - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 12: 0x62606a79fa8c - test[4f12b380f1fb68c5]::test_main_with_exit_callback::::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62606ba56eb2 - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 14: 0x62606ba56eb2 - std[e72de78501789eb0]::panicking::panic_with_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:833:13 [INFO] [stdout] 15: 0x62606ba3c738 - std[e72de78501789eb0]::panicking::panic_handler::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:698:13 [INFO] [stdout] 16: 0x62606ba338b9 - std[e72de78501789eb0]::sys::backtrace::__rust_end_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:182:18 [INFO] [stdout] 17: 0x62606ba3d6fd - __rustc[78a4cf188af856fc]::rust_begin_unwind [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:689:5 [INFO] [stdout] 18: 0x62606ba7909c - core[6aaeda2c7ad07a48]::panicking::panic_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62606ba78f23 - core[6aaeda2c7ad07a48]::panicking::assert_failed_inner [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:439:17 [INFO] [stdout] 20: 0x62606a73942b - core[6aaeda2c7ad07a48]::panicking::assert_failed:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:394:5 [INFO] [stdout] 21: 0x62606a74bb16 - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter::integration_tests::test_mixed_file_types [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:245:13 [INFO] [stdout] 22: 0x62606a746227 - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter::integration_tests::test_mixed_file_types::{closure#0} [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:231:31 [INFO] [stdout] 23: 0x62606a6f9d66 - >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62606a793b6b - core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x62606a793b6b - test[4f12b380f1fb68c5]::__rust_begin_short_backtrace::, fn() -> core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 26: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 27: 0x62606a7a055b - as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 28: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::do_call::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 29: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::, core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 30: 0x62606a7a055b - std[e72de78501789eb0]::panic::catch_unwind::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 31: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 32: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 33: 0x62606a79bc74 - test[4f12b380f1fb68c5]::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 34: 0x62606a79bc74 - std[e72de78501789eb0]::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 35: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 36: 0x62606a7a3162 - ::{closure#1}::{closure#0}> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 37: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 38: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::<(), core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 39: 0x62606a7a3162 - std[e72de78501789eb0]::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 40: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 41: 0x62606a7a3162 - ::{closure#1} as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 42: 0x62606ba5e63f - + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 43: 0x62606ba5e63f - ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 44: 0x77550bae5aa4 - [INFO] [stdout] 45: 0x77550bb72a64 - clone [INFO] [stdout] 46: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter::tests::test_binary_file_rejection stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter::tests::test_binary_file_rejection' (29) panicked at src/../modules/file_splitter/splitter.rs:137:9: [INFO] [stdout] assertion failed: result.is_err() [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::libunwind::trace [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::trace_unsynchronized:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62606ba5f58a - std[e72de78501789eb0]::sys::backtrace::_print_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:74:9 [INFO] [stdout] 3: 0x62606ba5f58a - <::print::DisplayBacktrace as core[6aaeda2c7ad07a48]::fmt::Display>::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:44:26 [INFO] [stdout] 4: 0x62606ba788aa - ::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/fmt/rt.rs:152:76 [INFO] [stdout] 5: 0x62606ba788aa - core[6aaeda2c7ad07a48]::fmt::write [INFO] [stdout] 6: 0x62606ba64fc2 - std[e72de78501789eb0]::io::default_write_fmt::> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62606ba64fc2 - as std[e72de78501789eb0]::io::Write>::write_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:1994:13 [INFO] [stdout] 8: 0x62606ba3c67f - ::print [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:47:9 [INFO] [stdout] 9: 0x62606ba3c67f - std[e72de78501789eb0]::panicking::default_hook::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:292:27 [INFO] [stdout] 10: 0x62606ba56c39 - std[e72de78501789eb0]::panicking::default_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:316:9 [INFO] [stdout] 11: 0x62606a79fa8c - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 12: 0x62606a79fa8c - test[4f12b380f1fb68c5]::test_main_with_exit_callback::::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62606ba56eb2 - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 14: 0x62606ba56eb2 - std[e72de78501789eb0]::panicking::panic_with_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:833:13 [INFO] [stdout] 15: 0x62606ba3c76a - std[e72de78501789eb0]::panicking::panic_handler::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:691:13 [INFO] [stdout] 16: 0x62606ba338b9 - std[e72de78501789eb0]::sys::backtrace::__rust_end_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:182:18 [INFO] [stdout] 17: 0x62606ba3d6fd - __rustc[78a4cf188af856fc]::rust_begin_unwind [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:689:5 [INFO] [stdout] 18: 0x62606ba7909c - core[6aaeda2c7ad07a48]::panicking::panic_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62606ba79062 - core[6aaeda2c7ad07a48]::panicking::panic [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:150:5 [INFO] [stdout] 20: 0x62606a74dbd0 - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter::tests::test_binary_file_rejection [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:137:9 [INFO] [stdout] 21: 0x62606a746517 - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter::tests::test_binary_file_rejection::{closure#0} [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:130:36 [INFO] [stdout] 22: 0x62606a6f9f26 - >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 23: 0x62606a793b6b - core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62606a793b6b - test[4f12b380f1fb68c5]::__rust_begin_short_backtrace::, fn() -> core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 25: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 26: 0x62606a7a055b - as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 27: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::do_call::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 28: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::, core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 29: 0x62606a7a055b - std[e72de78501789eb0]::panic::catch_unwind::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 30: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 31: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 32: 0x62606a79bc74 - test[4f12b380f1fb68c5]::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 33: 0x62606a79bc74 - std[e72de78501789eb0]::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 34: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 35: 0x62606a7a3162 - ::{closure#1}::{closure#0}> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 36: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 37: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::<(), core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 38: 0x62606a7a3162 - std[e72de78501789eb0]::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 39: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 40: 0x62606a7a3162 - ::{closure#1} as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 41: 0x62606ba5e63f - + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 42: 0x62606ba5e63f - ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 43: 0x77550bae5aa4 - [INFO] [stdout] 44: 0x77550bb72a64 - clone [INFO] [stdout] 45: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection' (42) panicked at src/../modules/file_splitter/splitter_types/json_splitter.rs:364:9: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: JsonLines [INFO] [stdout] right: Standard [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::libunwind::trace [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::trace_unsynchronized:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62606ba5f58a - std[e72de78501789eb0]::sys::backtrace::_print_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:74:9 [INFO] [stdout] 3: 0x62606ba5f58a - <::print::DisplayBacktrace as core[6aaeda2c7ad07a48]::fmt::Display>::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:44:26 [INFO] [stdout] 4: 0x62606ba788aa - ::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/fmt/rt.rs:152:76 [INFO] [stdout] 5: 0x62606ba788aa - core[6aaeda2c7ad07a48]::fmt::write [INFO] [stdout] 6: 0x62606ba64fc2 - std[e72de78501789eb0]::io::default_write_fmt::> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62606ba64fc2 - as std[e72de78501789eb0]::io::Write>::write_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:1994:13 [INFO] [stdout] 8: 0x62606ba3c67f - ::print [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:47:9 [INFO] [stdout] 9: 0x62606ba3c67f - std[e72de78501789eb0]::panicking::default_hook::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:292:27 [INFO] [stdout] 10: 0x62606ba56c39 - std[e72de78501789eb0]::panicking::default_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:316:9 [INFO] [stdout] 11: 0x62606a79fa8c - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 12: 0x62606a79fa8c - test[4f12b380f1fb68c5]::test_main_with_exit_callback::::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62606ba56eb2 - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 14: 0x62606ba56eb2 - std[e72de78501789eb0]::panicking::panic_with_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:833:13 [INFO] [stdout] 15: 0x62606ba3c738 - std[e72de78501789eb0]::panicking::panic_handler::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:698:13 [INFO] [stdout] 16: 0x62606ba338b9 - std[e72de78501789eb0]::sys::backtrace::__rust_end_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:182:18 [INFO] [stdout] 17: 0x62606ba3d6fd - __rustc[78a4cf188af856fc]::rust_begin_unwind [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:689:5 [INFO] [stdout] 18: 0x62606ba7909c - core[6aaeda2c7ad07a48]::panicking::panic_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62606ba78f23 - core[6aaeda2c7ad07a48]::panicking::assert_failed_inner [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:439:17 [INFO] [stdout] 20: 0x62606a73947c - core[6aaeda2c7ad07a48]::panicking::assert_failed:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:394:5 [INFO] [stdout] 21: 0x62606a74f924 - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:364:9 [INFO] [stdout] 22: 0x62606a746667 - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection::{closure#0} [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:358:35 [INFO] [stdout] 23: 0x62606a6fa2a6 - >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62606a793b6b - core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x62606a793b6b - test[4f12b380f1fb68c5]::__rust_begin_short_backtrace::, fn() -> core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 26: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 27: 0x62606a7a055b - as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 28: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::do_call::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 29: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::, core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 30: 0x62606a7a055b - std[e72de78501789eb0]::panic::catch_unwind::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 31: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 32: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 33: 0x62606a79bc74 - test[4f12b380f1fb68c5]::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 34: 0x62606a79bc74 - std[e72de78501789eb0]::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 35: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 36: 0x62606a7a3162 - ::{closure#1}::{closure#0}> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 37: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 38: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::<(), core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 39: 0x62606a7a3162 - std[e72de78501789eb0]::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 40: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 41: 0x62606a7a3162 - ::{closure#1} as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 42: 0x62606ba5e63f - + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 43: 0x62606ba5e63f - ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 44: 0x77550bae5aa4 - [INFO] [stdout] 45: 0x77550bb72a64 - clone [INFO] [stdout] 46: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection' (45) panicked at src/../modules/file_splitter/splitter_types/json_splitter.rs:354:9: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: JsonLines [INFO] [stdout] right: Standard [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::libunwind::trace [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::trace_unsynchronized:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62606ba5f58a - std[e72de78501789eb0]::sys::backtrace::_print_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:74:9 [INFO] [stdout] 3: 0x62606ba5f58a - <::print::DisplayBacktrace as core[6aaeda2c7ad07a48]::fmt::Display>::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:44:26 [INFO] [stdout] 4: 0x62606ba788aa - ::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/fmt/rt.rs:152:76 [INFO] [stdout] 5: 0x62606ba788aa - core[6aaeda2c7ad07a48]::fmt::write [INFO] [stdout] 6: 0x62606ba64fc2 - std[e72de78501789eb0]::io::default_write_fmt::> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62606ba64fc2 - as std[e72de78501789eb0]::io::Write>::write_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:1994:13 [INFO] [stdout] 8: 0x62606ba3c67f - ::print [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:47:9 [INFO] [stdout] 9: 0x62606ba3c67f - std[e72de78501789eb0]::panicking::default_hook::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:292:27 [INFO] [stdout] 10: 0x62606ba56c39 - std[e72de78501789eb0]::panicking::default_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:316:9 [INFO] [stdout] 11: 0x62606a79fa8c - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 12: 0x62606a79fa8c - test[4f12b380f1fb68c5]::test_main_with_exit_callback::::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62606ba56eb2 - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 14: 0x62606ba56eb2 - std[e72de78501789eb0]::panicking::panic_with_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:833:13 [INFO] [stdout] 15: 0x62606ba3c738 - std[e72de78501789eb0]::panicking::panic_handler::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:698:13 [INFO] [stdout] 16: 0x62606ba338b9 - std[e72de78501789eb0]::sys::backtrace::__rust_end_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:182:18 [INFO] [stdout] 17: 0x62606ba3d6fd - __rustc[78a4cf188af856fc]::rust_begin_unwind [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:689:5 [INFO] [stdout] 18: 0x62606ba7909c - core[6aaeda2c7ad07a48]::panicking::panic_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62606ba78f23 - core[6aaeda2c7ad07a48]::panicking::assert_failed_inner [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:439:17 [INFO] [stdout] 20: 0x62606a73947c - core[6aaeda2c7ad07a48]::panicking::assert_failed:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:394:5 [INFO] [stdout] 21: 0x62606a750324 - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:354:9 [INFO] [stdout] 22: 0x62606a746727 - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection::{closure#0} [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:348:38 [INFO] [stdout] 23: 0x62606a6fa3a6 - >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62606a793b6b - core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x62606a793b6b - test[4f12b380f1fb68c5]::__rust_begin_short_backtrace::, fn() -> core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 26: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 27: 0x62606a7a055b - as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 28: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::do_call::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 29: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::, core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 30: 0x62606a7a055b - std[e72de78501789eb0]::panic::catch_unwind::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 31: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 32: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 33: 0x62606a79bc74 - test[4f12b380f1fb68c5]::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 34: 0x62606a79bc74 - std[e72de78501789eb0]::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 35: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 36: 0x62606a7a3162 - ::{closure#1}::{closure#0}> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 37: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 38: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::<(), core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 39: 0x62606a7a3162 - std[e72de78501789eb0]::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 40: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 41: 0x62606a7a3162 - ::{closure#1} as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 42: 0x62606ba5e63f - + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 43: 0x62606ba5e63f - ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 44: 0x77550bae5aa4 - [INFO] [stdout] 45: 0x77550bb72a64 - clone [INFO] [stdout] 46: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection' (41) panicked at src/../modules/file_splitter/splitter_types/json_splitter.rs:400:9: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: JsonLines [INFO] [stdout] right: Invalid [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::libunwind::trace [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::trace_unsynchronized:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62606ba5f58a - std[e72de78501789eb0]::sys::backtrace::_print_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:74:9 [INFO] [stdout] 3: 0x62606ba5f58a - <::print::DisplayBacktrace as core[6aaeda2c7ad07a48]::fmt::Display>::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:44:26 [INFO] [stdout] 4: 0x62606ba788aa - ::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/fmt/rt.rs:152:76 [INFO] [stdout] 5: 0x62606ba788aa - core[6aaeda2c7ad07a48]::fmt::write [INFO] [stdout] 6: 0x62606ba64fc2 - std[e72de78501789eb0]::io::default_write_fmt::> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62606ba64fc2 - as std[e72de78501789eb0]::io::Write>::write_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:1994:13 [INFO] [stdout] 8: 0x62606ba3c67f - ::print [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:47:9 [INFO] [stdout] 9: 0x62606ba3c67f - std[e72de78501789eb0]::panicking::default_hook::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:292:27 [INFO] [stdout] 10: 0x62606ba56c39 - std[e72de78501789eb0]::panicking::default_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:316:9 [INFO] [stdout] 11: 0x62606a79fa8c - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 12: 0x62606a79fa8c - test[4f12b380f1fb68c5]::test_main_with_exit_callback::::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62606ba56eb2 - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 14: 0x62606ba56eb2 - std[e72de78501789eb0]::panicking::panic_with_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:833:13 [INFO] [stdout] 15: 0x62606ba3c738 - std[e72de78501789eb0]::panicking::panic_handler::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:698:13 [INFO] [stdout] 16: 0x62606ba338b9 - std[e72de78501789eb0]::sys::backtrace::__rust_end_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:182:18 [INFO] [stdout] 17: 0x62606ba3d6fd - __rustc[78a4cf188af856fc]::rust_begin_unwind [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:689:5 [INFO] [stdout] 18: 0x62606ba7909c - core[6aaeda2c7ad07a48]::panicking::panic_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62606ba78f23 - core[6aaeda2c7ad07a48]::panicking::assert_failed_inner [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:439:17 [INFO] [stdout] 20: 0x62606a73947c - core[6aaeda2c7ad07a48]::panicking::assert_failed:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:394:5 [INFO] [stdout] 21: 0x62606a750204 - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:400:9 [INFO] [stdout] 22: 0x62606a7466f7 - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection::{closure#0} [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:395:37 [INFO] [stdout] 23: 0x62606a6fa366 - >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62606a793b6b - core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x62606a793b6b - test[4f12b380f1fb68c5]::__rust_begin_short_backtrace::, fn() -> core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 26: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 27: 0x62606a7a055b - as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 28: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::do_call::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 29: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::, core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 30: 0x62606a7a055b - std[e72de78501789eb0]::panic::catch_unwind::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 31: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 32: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 33: 0x62606a79bc74 - test[4f12b380f1fb68c5]::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 34: 0x62606a79bc74 - std[e72de78501789eb0]::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 35: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 36: 0x62606a7a3162 - ::{closure#1}::{closure#0}> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 37: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 38: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::<(), core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 39: 0x62606a7a3162 - std[e72de78501789eb0]::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 40: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 41: 0x62606a7a3162 - ::{closure#1} as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 42: 0x62606ba5e63f - + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 43: 0x62606ba5e63f - ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 44: 0x77550bae5aa4 - [INFO] [stdout] 45: 0x77550bb72a64 - clone [INFO] [stdout] 46: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter::integration_tests::test_performance_characteristics stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter::integration_tests::test_performance_characteristics' (28) panicked at src/../modules/file_splitter/splitter.rs:288:9: [INFO] [stdout] assertion failed: chunks.len() > 1 [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::libunwind::trace [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62606ba5f58a - std[e72de78501789eb0]::backtrace_rs::backtrace::trace_unsynchronized:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62606ba5f58a - std[e72de78501789eb0]::sys::backtrace::_print_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:74:9 [INFO] [stdout] 3: 0x62606ba5f58a - <::print::DisplayBacktrace as core[6aaeda2c7ad07a48]::fmt::Display>::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:44:26 [INFO] [stdout] 4: 0x62606ba788aa - ::fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/fmt/rt.rs:152:76 [INFO] [stdout] 5: 0x62606ba788aa - core[6aaeda2c7ad07a48]::fmt::write [INFO] [stdout] 6: 0x62606ba64fc2 - std[e72de78501789eb0]::io::default_write_fmt::> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62606ba64fc2 - as std[e72de78501789eb0]::io::Write>::write_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/io/mod.rs:1994:13 [INFO] [stdout] 8: 0x62606ba3c67f - ::print [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:47:9 [INFO] [stdout] 9: 0x62606ba3c67f - std[e72de78501789eb0]::panicking::default_hook::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:292:27 [INFO] [stdout] 10: 0x62606ba56c39 - std[e72de78501789eb0]::panicking::default_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:316:9 [INFO] [stdout] 11: 0x62606a79fa8c - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 12: 0x62606a79fa8c - test[4f12b380f1fb68c5]::test_main_with_exit_callback::::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62606ba56eb2 - core[6aaeda2c7ad07a48]::ops::function::Fn<(&'a std[e72de78501789eb0]::panic::PanicHookInfo<'b>,), Output = ()> + core[6aaeda2c7ad07a48]::marker::Sync + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::Fn<(&std[e72de78501789eb0]::panic::PanicHookInfo,)>>::call [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2254:9 [INFO] [stdout] 14: 0x62606ba56eb2 - std[e72de78501789eb0]::panicking::panic_with_hook [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:833:13 [INFO] [stdout] 15: 0x62606ba3c76a - std[e72de78501789eb0]::panicking::panic_handler::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:691:13 [INFO] [stdout] 16: 0x62606ba338b9 - std[e72de78501789eb0]::sys::backtrace::__rust_end_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:182:18 [INFO] [stdout] 17: 0x62606ba3d6fd - __rustc[78a4cf188af856fc]::rust_begin_unwind [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:689:5 [INFO] [stdout] 18: 0x62606ba7909c - core[6aaeda2c7ad07a48]::panicking::panic_fmt [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62606ba79062 - core[6aaeda2c7ad07a48]::panicking::panic [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panicking.rs:150:5 [INFO] [stdout] 20: 0x62606a74d20e - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter::integration_tests::test_performance_characteristics [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:288:9 [INFO] [stdout] 21: 0x62606a7462b7 - ai_workbench_lib[a7b34f76a47abd37]::modules::file_splitter::splitter::integration_tests::test_performance_characteristics::{closure#0} [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:273:42 [INFO] [stdout] 22: 0x62606a6f9e26 - >::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 23: 0x62606a793b6b - core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62606a793b6b - test[4f12b380f1fb68c5]::__rust_begin_short_backtrace::, fn() -> core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:663:18 [INFO] [stdout] 25: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:74 [INFO] [stdout] 26: 0x62606a7a055b - as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 27: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::do_call::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 28: 0x62606a7a055b - std[e72de78501789eb0]::panicking::catch_unwind::, core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 29: 0x62606a7a055b - std[e72de78501789eb0]::panic::catch_unwind::, core[6aaeda2c7ad07a48]::result::Result<(), alloc[28d2dab30c1dc666]::string::String>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 30: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test_in_process [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:686:27 [INFO] [stdout] 31: 0x62606a7a055b - test[4f12b380f1fb68c5]::run_test::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:607:43 [INFO] [stdout] 32: 0x62606a79bc74 - test[4f12b380f1fb68c5]::run_test::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/test/src/lib.rs:637:41 [INFO] [stdout] 33: 0x62606a79bc74 - std[e72de78501789eb0]::sys::backtrace::__rust_begin_short_backtrace:: [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/backtrace.rs:166:18 [INFO] [stdout] 34: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1}::{closure#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:91:13 [INFO] [stdout] 35: 0x62606a7a3162 - ::{closure#1}::{closure#0}> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/panic/unwind_safe.rs:275:9 [INFO] [stdout] 36: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::do_call::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:581:40 [INFO] [stdout] 37: 0x62606a7a3162 - std[e72de78501789eb0]::panicking::catch_unwind::<(), core[6aaeda2c7ad07a48]::panic::unwind_safe::AssertUnwindSafe::{closure#1}::{closure#0}>> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panicking.rs:544:19 [INFO] [stdout] 38: 0x62606a7a3162 - std[e72de78501789eb0]::panic::catch_unwind::::{closure#1}::{closure#0}>, ()> [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/panic.rs:359:14 [INFO] [stdout] 39: 0x62606a7a3162 - std[e72de78501789eb0]::thread::lifecycle::spawn_unchecked::::{closure#1} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/thread/lifecycle.rs:89:26 [INFO] [stdout] 40: 0x62606a7a3162 - ::{closure#1} as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once::{shim:vtable#0} [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 41: 0x62606ba5e63f - + core[6aaeda2c7ad07a48]::marker::Send> as core[6aaeda2c7ad07a48]::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/alloc/src/boxed.rs:2240:9 [INFO] [stdout] 42: 0x62606ba5e63f - ::new::thread_start [INFO] [stdout] at /rustc/dec9417b8611e34e787a3e4c37686b5131f9e5c5/library/std/src/sys/thread/unix.rs:118:17 [INFO] [stdout] 43: 0x77550bae5aa4 - [INFO] [stdout] 44: 0x77550bb72a64 - clone [INFO] [stdout] 45: 0x0 - [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] modules::file_splitter::splitter::integration_tests::test_edge_cases [INFO] [stdout] modules::file_splitter::splitter::integration_tests::test_mixed_file_types [INFO] [stdout] modules::file_splitter::splitter::integration_tests::test_performance_characteristics [INFO] [stdout] modules::file_splitter::splitter::tests::test_binary_file_rejection [INFO] [stdout] modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection [INFO] [stdout] modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection [INFO] [stdout] modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection [INFO] [stdout] [INFO] [stdout] test result: FAILED. 30 passed; 7 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.69s [INFO] [stdout] [INFO] [stderr] error: test failed, to rerun pass `--lib` [INFO] running `Command { std: "docker" "inspect" "80dbf007c83d42125fcce01c6b49d9fa3702a2c36ed41331a256f5346949dc74", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "80dbf007c83d42125fcce01c6b49d9fa3702a2c36ed41331a256f5346949dc74", kill_on_drop: false }` [INFO] [stdout] 80dbf007c83d42125fcce01c6b49d9fa3702a2c36ed41331a256f5346949dc74