[INFO] fetching crate ai-workbench-lib 0.4.0... [INFO] testing ai-workbench-lib-0.4.0 against 1.91.0 for beta-1.92-2 [INFO] extracting crate ai-workbench-lib 0.4.0 into /workspace/builds/worker-6-tc1/source [INFO] started tweaking crates.io crate ai-workbench-lib 0.4.0 [INFO] finished tweaking crates.io crate ai-workbench-lib 0.4.0 [INFO] tweaked toml for crates.io crate ai-workbench-lib 0.4.0 written to /workspace/builds/worker-6-tc1/source/Cargo.toml [INFO] validating manifest of crates.io crate ai-workbench-lib 0.4.0 on toolchain 1.91.0 [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+1.91.0" "metadata" "--manifest-path" "Cargo.toml" "--no-deps", kill_on_drop: false }` [INFO] crate crates.io crate ai-workbench-lib 0.4.0 already has a lockfile, it will not be regenerated [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+1.91.0" "fetch" "--manifest-path" "Cargo.toml", kill_on_drop: false }` [INFO] [stderr] Blocking waiting for file lock on package cache [INFO] [stderr] Updating crates.io index [INFO] [stderr] Downloading crates ... [INFO] [stderr] Downloaded time-core v0.1.5 [INFO] [stderr] Downloaded aws-smithy-checksums v0.63.8 [INFO] [stderr] Downloaded aws-smithy-http v0.62.3 [INFO] [stderr] Downloaded aws-runtime v1.5.10 [INFO] [stderr] Downloaded aws-smithy-eventstream v0.60.11 [INFO] [stderr] Downloaded aws-sdk-sso v1.83.0 [INFO] [stderr] Downloaded aws-sdk-ssooidc v1.84.0 [INFO] [stderr] Downloaded aws-smithy-runtime v1.9.1 [INFO] [stderr] Downloaded aws-sdk-sts v1.85.0 [INFO] [stderr] Downloaded markdown v1.0.0 [INFO] [stderr] Downloaded aws-sdk-bedrockruntime v1.105.0 [INFO] [stderr] Downloaded wasi v0.14.3+wasi-0.2.4 [INFO] [stderr] Downloaded aws-sigv4 v1.3.4 [INFO] [stderr] Downloaded aws-smithy-runtime-api v1.9.0 [INFO] [stderr] Downloaded time v0.3.42 [INFO] [stderr] Downloaded wit-bindgen v0.45.0 [INFO] [stderr] Downloaded crc-fast v1.3.0 [INFO] [stderr] Downloaded security-framework v3.3.0 [INFO] [stderr] Downloaded cc v1.2.35 [INFO] [stderr] Downloaded aws-credential-types v1.2.6 [INFO] [stderr] Downloaded time-macros v0.2.23 [INFO] [stderr] Downloaded aws-sdk-s3 v1.104.0 [INFO] [stderr] Downloaded aws-smithy-json v0.61.5 [INFO] [stderr] Downloaded aws-config v1.8.6 [INFO] [stderr] Downloaded aws-smithy-http-client v1.1.1 [INFO] [stderr] Downloaded unicode-id v0.3.5 [INFO] [stderr] Downloaded aws-types v1.3.8 [INFO] [stderr] Downloaded indexmap v2.11.0 [INFO] [stderr] Downloaded find-msvc-tools v0.1.0 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:4848fb76d95f26979359cc7e45710b1dbc8f3acb7aeedee7c460d7702230f228" "/opt/rustwide/cargo-home/bin/cargo" "+1.91.0" "metadata" "--no-deps" "--format-version=1", kill_on_drop: false }` [INFO] [stdout] 5fcb46fe23f161fd3456ca571cbe264b104339095d5a58ecaac0408d8549e2b0 [INFO] running `Command { std: "docker" "start" "-a" "5fcb46fe23f161fd3456ca571cbe264b104339095d5a58ecaac0408d8549e2b0", kill_on_drop: false }` [INFO] running `Command { std: "docker" "inspect" "5fcb46fe23f161fd3456ca571cbe264b104339095d5a58ecaac0408d8549e2b0", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "5fcb46fe23f161fd3456ca571cbe264b104339095d5a58ecaac0408d8549e2b0", kill_on_drop: false }` [INFO] [stdout] 5fcb46fe23f161fd3456ca571cbe264b104339095d5a58ecaac0408d8549e2b0 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=warn" "-e" "RUSTDOCFLAGS=--cap-lints=warn" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:4848fb76d95f26979359cc7e45710b1dbc8f3acb7aeedee7c460d7702230f228" "/opt/rustwide/cargo-home/bin/cargo" "+1.91.0" "build" "--frozen" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] b61300162a96a834cac61948e290b79d7c5bb730f870d0cc4a5a082704bc585a [INFO] running `Command { std: "docker" "start" "-a" "b61300162a96a834cac61948e290b79d7c5bb730f870d0cc4a5a082704bc585a", kill_on_drop: false }` [INFO] [stderr] Compiling libc v0.2.175 [INFO] [stderr] Compiling proc-macro2 v1.0.101 [INFO] [stderr] Compiling cfg-if v1.0.3 [INFO] [stderr] Compiling find-msvc-tools v0.1.0 [INFO] [stderr] Compiling subtle v2.6.1 [INFO] [stderr] Compiling deranged v0.5.3 [INFO] [stderr] Compiling time-core v0.1.5 [INFO] [stderr] Compiling foldhash v0.1.5 [INFO] [stderr] Compiling aws-lc-rs v1.13.3 [INFO] [stderr] Compiling const-oid v0.9.6 [INFO] [stderr] Compiling base64ct v1.8.0 [INFO] [stderr] Compiling rustls v0.23.31 [INFO] [stderr] Compiling base16ct v0.1.1 [INFO] [stderr] Compiling aws-types v1.3.8 [INFO] [stderr] Compiling crc32fast v1.5.0 [INFO] [stderr] Compiling serde v1.0.219 [INFO] [stderr] Compiling regex-lite v0.1.7 [INFO] [stderr] Compiling regex-syntax v0.8.6 [INFO] [stderr] Compiling serde_json v1.0.143 [INFO] [stderr] Compiling hashbrown v0.15.5 [INFO] [stderr] Compiling der v0.6.1 [INFO] [stderr] Compiling digest v0.10.7 [INFO] [stderr] Compiling anyhow v1.0.99 [INFO] [stderr] Compiling sharded-slab v0.1.7 [INFO] [stderr] Compiling thread_local v1.1.9 [INFO] [stderr] Compiling tracing-log v0.2.0 [INFO] [stderr] Compiling csv-core v0.1.12 [INFO] [stderr] Compiling unicode-id v0.3.5 [INFO] [stderr] Compiling nu-ansi-term v0.50.1 [INFO] [stderr] Compiling hmac v0.12.1 [INFO] [stderr] Compiling sha2 v0.10.9 [INFO] [stderr] Compiling sha1 v0.10.6 [INFO] [stderr] Compiling md-5 v0.10.6 [INFO] [stderr] Compiling markdown v1.0.0 [INFO] [stderr] Compiling indexmap v2.11.0 [INFO] [stderr] Compiling lru v0.12.5 [INFO] [stderr] Compiling tracing-subscriber v0.3.20 [INFO] [stderr] Compiling spki v0.6.0 [INFO] [stderr] Compiling quote v1.0.40 [INFO] [stderr] Compiling pkcs8 v0.9.0 [INFO] [stderr] Compiling time v0.3.42 [INFO] [stderr] Compiling syn v2.0.106 [INFO] [stderr] Compiling parking_lot_core v0.9.11 [INFO] [stderr] Compiling signal-hook-registry v1.4.6 [INFO] [stderr] Compiling mio v1.0.4 [INFO] [stderr] Compiling socket2 v0.6.0 [INFO] [stderr] Compiling jobserver v0.1.34 [INFO] [stderr] Compiling getrandom v0.2.16 [INFO] [stderr] Compiling getrandom v0.3.3 [INFO] [stderr] Compiling socket2 v0.5.10 [INFO] [stderr] Compiling rand_core v0.6.4 [INFO] [stderr] Compiling sec1 v0.3.0 [INFO] [stderr] Compiling parking_lot v0.12.4 [INFO] [stderr] Compiling cc v1.2.35 [INFO] [stderr] Compiling ff v0.12.1 [INFO] [stderr] Compiling crypto-bigint v0.4.9 [INFO] [stderr] Compiling signature v1.6.4 [INFO] [stderr] Compiling group v0.12.1 [INFO] [stderr] Compiling crypto-bigint v0.5.5 [INFO] [stderr] Compiling uuid v1.18.1 [INFO] [stderr] Compiling rand_core v0.9.3 [INFO] [stderr] Compiling rand_chacha v0.9.0 [INFO] [stderr] Compiling rand v0.9.2 [INFO] [stderr] Compiling regex-automata v0.4.10 [INFO] [stderr] Compiling rfc6979 v0.3.1 [INFO] [stderr] Compiling elliptic-curve v0.12.3 [INFO] [stderr] Compiling cmake v0.1.54 [INFO] [stderr] Compiling ecdsa v0.14.8 [INFO] [stderr] Compiling ring v0.17.14 [INFO] [stderr] Compiling aws-lc-sys v0.30.0 [INFO] [stderr] Compiling p256 v0.11.1 [INFO] [stderr] Compiling rustls v0.21.12 [INFO] [stderr] Compiling regex v1.11.2 [INFO] [stderr] Compiling rustls-webpki v0.101.7 [INFO] [stderr] Compiling sct v0.7.1 [INFO] [stderr] Compiling crc-fast v1.3.0 [INFO] [stderr] Compiling synstructure v0.13.2 [INFO] [stderr] Compiling tokio-macros v2.5.0 [INFO] [stderr] Compiling futures-macro v0.3.31 [INFO] [stderr] Compiling tracing-attributes v0.1.30 [INFO] [stderr] Compiling zerofrom-derive v0.1.6 [INFO] [stderr] Compiling yoke-derive v0.8.0 [INFO] [stderr] Compiling zerovec-derive v0.11.1 [INFO] [stderr] Compiling displaydoc v0.2.5 [INFO] [stderr] Compiling serde_derive v1.0.219 [INFO] [stderr] Compiling tokio v1.47.1 [INFO] [stderr] Compiling futures-util v0.3.31 [INFO] [stderr] Compiling tracing v0.1.41 [INFO] [stderr] Compiling zerofrom v0.1.6 [INFO] [stderr] Compiling yoke v0.8.0 [INFO] [stderr] Compiling zerovec v0.11.4 [INFO] [stderr] Compiling zerotrie v0.2.2 [INFO] [stderr] Compiling tinystr v0.8.1 [INFO] [stderr] Compiling potential_utf v0.1.3 [INFO] [stderr] Compiling icu_collections v2.0.0 [INFO] [stderr] Compiling icu_locale_core v2.0.0 [INFO] [stderr] Compiling icu_provider v2.0.0 [INFO] [stderr] Compiling icu_properties v2.0.1 [INFO] [stderr] Compiling icu_normalizer v2.0.0 [INFO] [stderr] Compiling idna_adapter v1.2.1 [INFO] [stderr] Compiling futures-executor v0.3.31 [INFO] [stderr] Compiling idna v1.1.0 [INFO] [stderr] Compiling futures v0.3.31 [INFO] [stderr] Compiling url v2.5.7 [INFO] [stderr] Compiling csv v1.3.1 [INFO] [stderr] Compiling rustls-webpki v0.103.4 [INFO] [stderr] Compiling tokio-util v0.7.16 [INFO] [stderr] Compiling aws-smithy-async v1.2.5 [INFO] [stderr] Compiling tokio-rustls v0.24.1 [INFO] [stderr] Compiling aws-smithy-types v1.3.2 [INFO] [stderr] Compiling h2 v0.4.12 [INFO] [stderr] Compiling h2 v0.3.27 [INFO] [stderr] Compiling aws-smithy-runtime-api v1.9.0 [INFO] [stderr] Compiling aws-smithy-eventstream v0.60.11 [INFO] [stderr] Compiling aws-smithy-json v0.61.5 [INFO] [stderr] Compiling aws-smithy-query v0.60.7 [INFO] [stderr] Compiling aws-smithy-http v0.62.3 [INFO] [stderr] Compiling aws-credential-types v1.2.6 [INFO] [stderr] Compiling aws-smithy-observability v0.1.3 [INFO] [stderr] Compiling aws-sigv4 v1.3.4 [INFO] [stderr] Compiling aws-smithy-checksums v0.63.8 [INFO] [stderr] Compiling hyper v1.7.0 [INFO] [stderr] Compiling tokio-rustls v0.26.2 [INFO] [stderr] Compiling hyper v0.14.32 [INFO] [stderr] Compiling hyper-util v0.1.16 [INFO] [stderr] Compiling hyper-rustls v0.27.7 [INFO] [stderr] Compiling hyper-rustls v0.24.2 [INFO] [stderr] Compiling aws-smithy-http-client v1.1.1 [INFO] [stderr] Compiling aws-smithy-runtime v1.9.1 [INFO] [stderr] Compiling aws-runtime v1.5.10 [INFO] [stderr] Compiling aws-sdk-sts v1.85.0 [INFO] [stderr] Compiling aws-sdk-sso v1.83.0 [INFO] [stderr] Compiling aws-sdk-ssooidc v1.84.0 [INFO] [stderr] Compiling aws-sdk-s3 v1.104.0 [INFO] [stderr] Compiling aws-sdk-bedrockruntime v1.105.0 [INFO] [stderr] Compiling aws-config v1.8.6 [INFO] [stderr] Compiling ai-workbench-lib v0.4.0 (/opt/rustwide/workdir) [INFO] [stderr] Finished `dev` profile [unoptimized + debuginfo] target(s) in 3m 42s [INFO] running `Command { std: "docker" "inspect" "b61300162a96a834cac61948e290b79d7c5bb730f870d0cc4a5a082704bc585a", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "b61300162a96a834cac61948e290b79d7c5bb730f870d0cc4a5a082704bc585a", kill_on_drop: false }` [INFO] [stdout] b61300162a96a834cac61948e290b79d7c5bb730f870d0cc4a5a082704bc585a [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=warn" "-e" "RUSTDOCFLAGS=--cap-lints=warn" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:4848fb76d95f26979359cc7e45710b1dbc8f3acb7aeedee7c460d7702230f228" "/opt/rustwide/cargo-home/bin/cargo" "+1.91.0" "test" "--frozen" "--no-run" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] 6f46842196b04d6b27c6c89d0ba628c420695ba9df04c4cabc5d3da70334620c [INFO] running `Command { std: "docker" "start" "-a" "6f46842196b04d6b27c6c89d0ba628c420695ba9df04c4cabc5d3da70334620c", kill_on_drop: false }` [INFO] [stderr] Compiling ai-workbench-lib v0.4.0 (/opt/rustwide/workdir) [INFO] [stderr] Finished `test` profile [unoptimized + debuginfo] target(s) in 10.89s [INFO] running `Command { std: "docker" "inspect" "6f46842196b04d6b27c6c89d0ba628c420695ba9df04c4cabc5d3da70334620c", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "6f46842196b04d6b27c6c89d0ba628c420695ba9df04c4cabc5d3da70334620c", kill_on_drop: false }` [INFO] [stdout] 6f46842196b04d6b27c6c89d0ba628c420695ba9df04c4cabc5d3da70334620c [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=warn" "-e" "RUSTDOCFLAGS=--cap-lints=warn" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:4848fb76d95f26979359cc7e45710b1dbc8f3acb7aeedee7c460d7702230f228" "/opt/rustwide/cargo-home/bin/cargo" "+1.91.0" "test" "--frozen", kill_on_drop: false }` [INFO] [stdout] eb8ca44228271c5b137cd8cd8ee8cfa277ec52bb452b766ce6f6f29d44ff2902 [INFO] running `Command { std: "docker" "start" "-a" "eb8ca44228271c5b137cd8cd8ee8cfa277ec52bb452b766ce6f6f29d44ff2902", kill_on_drop: false }` [INFO] [stderr] Finished `test` profile [unoptimized + debuginfo] target(s) in 0.46s [INFO] [stderr] Running unittests src/lib.rs (/opt/rustwide/target/debug/deps/ai_workbench_lib-36878ac039c7d20e) [INFO] [stdout] [INFO] [stdout] running 37 tests [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_file_info_structure ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_json_file_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_empty_file ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_file_type_detection_by_extension ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_invalid_csv_fallback ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::code_splitter::tests::test_simple_code_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_csv_format_validation ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_csv_single_chunk ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_csv_multiple_chunks_with_headers ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_record_boundary_preservation ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_tsv_format ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_json_lines_detection ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_json_lines_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_large_csv_file_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::code_splitter::tests::test_large_file_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::text_splitter::tests::test_large_file_multiple_chunks ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::text_splitter::tests::test_line_boundary_preservation ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::text_splitter::tests::test_small_file_single_chunk ... ok [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_large_folder_discovery ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_binary_file_rejection ... FAILED [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_mixed_file_types ... FAILED [INFO] [stdout] 2025-11-05T02:31:46.600976Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.601020Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/' [INFO] [stdout] 2025-11-05T02:31:46.601031Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.601042Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:31:46.601055Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2025-11-05T02:31:46.601064Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2025-11-05T02:31:46.601078Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.601090Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/' [INFO] [stdout] 2025-11-05T02:31:46.601098Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.601111Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2025-11-05T02:31:46.601125Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.601141Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'text_files/' [INFO] [stdout] 2025-11-05T02:31:46.601150Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:31:46.601163Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2025-11-05T02:31:46.601177Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection ... FAILED [INFO] [stdout] 2025-11-05T02:31:46.606849Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/data.csv': dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.606904Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.606933Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.606942Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/data.csv' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.606956Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:31:46.606965Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.606975Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.606986Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection ... FAILED [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_performance_characteristics ... FAILED [INFO] [stdout] 2025-11-05T02:31:46.608059Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:31:46.608655Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.608690Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.608717Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.608726Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'text_files/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.608764Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2025-11-05T02:31:46.608774Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.608783Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2025-11-05T02:31:46.608795Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.609381Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.609404Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.609443Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.609452Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'text_files/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.609465Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2025-11-05T02:31:46.609473Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.609481Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2025-11-05T02:31:46.609492Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.609906Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.609939Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' [INFO] [stdout] 2025-11-05T02:31:46.609952Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.609962Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:31:46.609977Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2025-11-05T02:31:46.609966Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/sample.txt': dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.609986Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2025-11-05T02:31:46.609998Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.609991Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.610007Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' [INFO] [stdout] 2025-11-05T02:31:46.610016Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.610017Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.610023Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/sample.txt' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.610031Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:31:46.610037Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:31:46.610044Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.610046Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.610054Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.610054Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' [INFO] [stdout] 2025-11-05T02:31:46.610062Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:31:46.610064Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:31:46.610077Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:31:46.610112Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.610130Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' [INFO] [stdout] 2025-11-05T02:31:46.610141Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.610151Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:31:46.610164Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_edge_cases ... FAILED [INFO] [stdout] 2025-11-05T02:31:46.615177Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt': dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.615241Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.615272Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.615281Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.615296Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:31:46.615305Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.615314Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.615322Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:31:46.615424Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt': Failed to get metadata for text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:31:46.615447Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:31:46.615456Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:31:46.615471Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:31:46.615479Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:31:46.615487Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:31:46.615500Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:31:46.608098Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'test-data/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_processing_summary::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:621:60 [INFO] [stdout] 4: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 7: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 8: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 9: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 13: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 14: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 15: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 16: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 18: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 21: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 23: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 25: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_processing_summary [INFO] [stdout] at ./src/../modules/file_discovery.rs:623:22 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_processing_summary::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:614:39 [INFO] [stdout] 27: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 30: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 31: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 32: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 33: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 34: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 35: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 36: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 38: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 39: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 40: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 41: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 42: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 43: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 44: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 45: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 46: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 47: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 48: [INFO] [stdout] 49: clone [INFO] [stdout] 2025-11-05T02:31:46.615683Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_processing_summary ... ok [INFO] [stdout] 2025-11-05T02:31:46.619662Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/data.csv': Failed to get metadata for test-data/data.csv [INFO] [stdout] 2025-11-05T02:31:46.619726Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:31:46.619753Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/data.csv [INFO] [stdout] 2025-11-05T02:31:46.619770Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:31:46.619780Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:31:46.619789Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:31:46.619802Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/data.csv [INFO] [stdout] 2025-11-05T02:31:46.615520Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 1: S3 HEAD request failed for 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:673:55 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 18: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end [INFO] [stdout] at ./src/../modules/job_processor/core.rs:710:37 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:638:57 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 35: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 36: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 41: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 42: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 44: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 45: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 47: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 49: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2025-11-05T02:31:46.620094Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.620122Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to get metadata for single file: text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:31:46.619818Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/data.csv [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/data.csv [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/data.csv': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:31:46.620577Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.620617Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.620627Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/config.json' [INFO] [stdout] 2025-11-05T02:31:46.620636Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.620650Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:31:46.620666Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.620141Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to get metadata for single file: text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 1: S3 HEAD request failed for 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:673:55 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 18: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end [INFO] [stdout] at ./src/../modules/job_processor/core.rs:710:37 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:638:57 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 35: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 36: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 41: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 42: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 44: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 45: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 47: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 49: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2025-11-05T02:31:46.620677Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/config.json' [INFO] [stdout] 2025-11-05T02:31:46.620686Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/config.json [INFO] [stdout] 2025-11-05T02:31:46.620698Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:31:46.620693Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2025-11-05T02:31:46.620710Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.620719Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/config.json' [INFO] [stdout] 2025-11-05T02:31:46.620728Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.620759Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/config.json [INFO] [stdout] 2025-11-05T02:31:46.620770Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:31:46.624395Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/config.json': dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.624537Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.624575Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.624584Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/config.json' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.624614Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:31:46.624624Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.624632Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.624640Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:31:46.632986Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:31:46.633143Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:955:42 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery [INFO] [stdout] at ./src/../modules/job_processor/core.rs:955:48 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:930:49 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:31:46.634308Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.634377Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:31:46.634435Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:955:42 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery [INFO] [stdout] at ./src/../modules/job_processor/core.rs:955:48 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:930:49 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:31:46.635061Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2025-11-05T02:31:46.635919Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:31:46.635964Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'mixed-files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_filtering::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:588:61 [INFO] [stdout] 4: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 7: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 8: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 9: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 13: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 14: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 15: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 16: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 18: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 21: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 23: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 25: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_filtering [INFO] [stdout] at ./src/../modules/file_discovery.rs:590:22 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_filtering::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:580:35 [INFO] [stdout] 27: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 30: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 31: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 32: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 33: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 34: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 35: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 36: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 38: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 39: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 40: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 41: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 42: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 43: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 44: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 45: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 46: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 47: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 48: [INFO] [stdout] 49: clone [INFO] [stdout] 2025-11-05T02:31:46.636813Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.637171Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.637210Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/sample.txt' [INFO] [stdout] 2025-11-05T02:31:46.637222Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.637234Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/sample.txt [INFO] [stdout] 2025-11-05T02:31:46.637252Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2025-11-05T02:31:46.637272Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2025-11-05T02:31:46.637288Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.637322Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/sample.txt' [INFO] [stdout] 2025-11-05T02:31:46.637333Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.637373Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:31:46.637390Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.637402Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'text_files/sample.txt' [INFO] [stdout] 2025-11-05T02:31:46.637413Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/sample.txt [INFO] [stdout] 2025-11-05T02:31:46.637426Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:31:46.637466Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.637482Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'text_files/sample.txt' [INFO] [stdout] 2025-11-05T02:31:46.637492Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.637503Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/sample.txt [INFO] [stdout] 2025-11-05T02:31:46.637519Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:31:46.637869Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:31:46.639839Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/config.json': Failed to get metadata for test-data/config.json [INFO] [stdout] 2025-11-05T02:31:46.639914Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:31:46.639940Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/config.json [INFO] [stdout] 2025-11-05T02:31:46.639972Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:31:46.639996Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:31:46.640020Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:31:46.640049Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/config.json [INFO] [stdout] 2025-11-05T02:31:46.637898Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_folder_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:470:60 [INFO] [stdout] 4: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 7: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 8: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 9: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 13: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 14: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 15: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 16: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 18: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 21: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 23: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 25: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_folder_discovery [INFO] [stdout] at ./src/../modules/file_discovery.rs:472:22 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_folder_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:462:37 [INFO] [stdout] 27: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 30: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 31: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 32: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 33: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 34: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 35: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 36: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 38: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 39: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 40: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 41: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 42: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 43: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 44: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 45: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 46: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 47: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 48: [INFO] [stdout] 49: clone [INFO] [stdout] 2025-11-05T02:31:46.640474Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery ... ok [INFO] [stdout] 2025-11-05T02:31:46.640080Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/config.json [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/config.json [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/config.json': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_file_filtering ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection ... FAILED [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end ... ok [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_folder_discovery ... ok [INFO] [stdout] 2025-11-05T02:31:46.643721Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/sample.txt': Failed to get metadata for test-data/sample.txt [INFO] [stdout] 2025-11-05T02:31:46.646805Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:31:46.646846Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/sample.txt [INFO] [stdout] 2025-11-05T02:31:46.646863Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:31:46.646873Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:31:46.646882Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:31:46.646905Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/sample.txt [INFO] [stdout] 2025-11-05T02:31:46.646925Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/sample.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/sample.txt [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/sample.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_single_file_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:441:62 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_single_file_discovery [INFO] [stdout] at ./src/../modules/file_discovery.rs:443:22 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_single_file_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:433:42 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:31:46.647845Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.648861Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.648958Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.649003Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/document.md' [INFO] [stdout] 2025-11-05T02:31:46.649032Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.649065Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:31:46.649099Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.649130Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/document.md' [INFO] [stdout] 2025-11-05T02:31:46.649163Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/document.md [INFO] [stdout] 2025-11-05T02:31:46.649194Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:31:46.649224Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.649252Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/document.md' [INFO] [stdout] 2025-11-05T02:31:46.649279Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.649315Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/document.md [INFO] [stdout] 2025-11-05T02:31:46.649346Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_large_text_file_splitting ... ok [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_single_file_discovery ... ok [INFO] [stdout] 2025-11-05T02:31:46.657115Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/document.md': dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.664863Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.664940Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.664952Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/document.md' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.664970Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:31:46.664980Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.664989Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.664999Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:31:46.665118Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/document.md': Failed to get metadata for test-data/document.md [INFO] [stdout] 2025-11-05T02:31:46.665144Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:31:46.665154Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/document.md [INFO] [stdout] 2025-11-05T02:31:46.665168Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:31:46.665177Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:31:46.665186Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:31:46.665199Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/document.md [INFO] [stdout] 2025-11-05T02:31:46.665214Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/document.md [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/document.md [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/document.md': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:31:46.666069Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.666109Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.666122Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/script.py' [INFO] [stdout] 2025-11-05T02:31:46.666132Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.666147Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:31:46.666161Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.666171Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/script.py' [INFO] [stdout] 2025-11-05T02:31:46.666181Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/script.py [INFO] [stdout] 2025-11-05T02:31:46.666193Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:31:46.666206Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.666215Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/script.py' [INFO] [stdout] 2025-11-05T02:31:46.666225Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.666236Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/script.py [INFO] [stdout] 2025-11-05T02:31:46.666249Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:31:46.676323Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/script.py': dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.679807Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.679949Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.679983Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/script.py' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.680016Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:31:46.680051Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.680077Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.680101Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:31:46.680243Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/script.py': Failed to get metadata for test-data/script.py [INFO] [stdout] 2025-11-05T02:31:46.680302Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:31:46.680331Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/script.py [INFO] [stdout] 2025-11-05T02:31:46.680364Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:31:46.680400Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:31:46.680439Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:31:46.680474Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/script.py [INFO] [stdout] 2025-11-05T02:31:46.680525Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/script.py [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/script.py [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/script.py': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:31:46.683799Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.683934Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.683974Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/code.rs' [INFO] [stdout] 2025-11-05T02:31:46.684006Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.684045Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:31:46.684088Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.684119Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/code.rs' [INFO] [stdout] 2025-11-05T02:31:46.684145Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/code.rs [INFO] [stdout] 2025-11-05T02:31:46.684184Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:31:46.684219Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.684256Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/code.rs' [INFO] [stdout] 2025-11-05T02:31:46.684289Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.684324Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/code.rs [INFO] [stdout] 2025-11-05T02:31:46.684365Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:31:46.681448Z  INFO ai_workbench_lib::modules::job_processor::core: Starting job execution for job_id: test-large-d66f6e54-ad3b-459d-9167-e4915e6e1494 [INFO] [stdout] 2025-11-05T02:31:46.687834Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.687874Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'large_files/large_document.txt' [INFO] [stdout] 2025-11-05T02:31:46.687887Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.687899Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:31:46.687914Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2025-11-05T02:31:46.687923Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2025-11-05T02:31:46.687970Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.687987Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'large_files/large_document.txt' [INFO] [stdout] 2025-11-05T02:31:46.687997Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.688010Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:31:46.688044Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.688058Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'large_files/large_document.txt' [INFO] [stdout] 2025-11-05T02:31:46.688068Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:31:46.688081Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:31:46.688119Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.688133Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'large_files/large_document.txt' [INFO] [stdout] 2025-11-05T02:31:46.688143Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.688154Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:31:46.688167Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:31:46.689468Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'text_files/sample.txt': dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.689503Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.689536Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.689551Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'text_files/sample.txt' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.689563Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:31:46.689572Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.689581Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.689590Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:31:46.692050Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'text_files/sample.txt': Failed to get metadata for text_files/sample.txt [INFO] [stdout] 2025-11-05T02:31:46.692825Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:31:46.693085Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/sample.txt [INFO] [stdout] 2025-11-05T02:31:46.693315Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:31:46.693489Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:31:46.693618Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:31:46.693793Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: text_files/sample.txt [INFO] [stdout] 2025-11-05T02:31:46.694072Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: text_files/sample.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for text_files/sample.txt [INFO] [stdout] 1: S3 HEAD request failed for 'text_files/sample.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1024:46 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 18: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1017:31 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:980:52 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 35: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 36: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 41: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 42: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 44: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 45: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 47: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 49: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2025-11-05T02:31:46.697503Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.697578Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to get metadata for single file: text_files/sample.txt [INFO] [stdout] 2025-11-05T02:31:46.698401Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/code.rs': dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.701785Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.701907Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.701944Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/code.rs' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.701982Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:31:46.702012Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.702048Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.702073Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:31:46.702207Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/code.rs': Failed to get metadata for test-data/code.rs [INFO] [stdout] 2025-11-05T02:31:46.702261Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:31:46.702290Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/code.rs [INFO] [stdout] 2025-11-05T02:31:46.702333Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:31:46.702360Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:31:46.702384Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:31:46.702431Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/code.rs [INFO] [stdout] 2025-11-05T02:31:46.702468Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/code.rs [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/code.rs [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/code.rs': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:31:46.701602Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'large_files/large_document.txt': dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.697780Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to get metadata for single file: text_files/sample.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for text_files/sample.txt [INFO] [stdout] 1: S3 HEAD request failed for 'text_files/sample.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1024:46 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 18: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1017:31 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:980:52 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 35: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 36: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 41: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 42: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 44: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 45: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 47: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 49: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2025-11-05T02:31:46.704898Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2025-11-05T02:31:46.705163Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.696697Z  INFO ai_workbench_lib::modules::job_processor::core: Starting job execution for job_id: test-folder-378fae45-bbdf-4f52-a667-84e40eb67251 [INFO] [stdout] 2025-11-05T02:31:46.705272Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.704796Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.705400Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.705451Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/styles.css' [INFO] [stdout] 2025-11-05T02:31:46.705482Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.705516Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:31:46.704962Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.705589Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.705315Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/' [INFO] [stdout] 2025-11-05T02:31:46.705677Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.705689Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:31:46.705704Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2025-11-05T02:31:46.705713Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2025-11-05T02:31:46.705725Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.705753Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/' [INFO] [stdout] 2025-11-05T02:31:46.705763Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.705774Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2025-11-05T02:31:46.705789Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.705800Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'text_files/' [INFO] [stdout] 2025-11-05T02:31:46.705810Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:31:46.705823Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2025-11-05T02:31:46.705844Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] 2025-11-05T02:31:46.705630Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.708858Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/styles.css' [INFO] [stdout] 2025-11-05T02:31:46.708891Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/styles.css [INFO] [stdout] 2025-11-05T02:31:46.708906Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:31:46.708920Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.708930Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/styles.css' [INFO] [stdout] 2025-11-05T02:31:46.708940Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.708951Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/styles.css [INFO] [stdout] 2025-11-05T02:31:46.708963Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:31:46.710829Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'large_files/large_document.txt' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.710899Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:31:46.710933Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.710963Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.710992Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:31:46.711137Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'large_files/large_document.txt': Failed to get metadata for large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:31:46.711197Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:31:46.711228Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:31:46.711262Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:31:46.711310Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:31:46.711345Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:31:46.711380Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:31:46.711435Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: large_files/large_document.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for large_files/large_document.txt [INFO] [stdout] 1: S3 HEAD request failed for 'large_files/large_document.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::JobProcessor::run::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:57:43 [INFO] [stdout] 6: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:882:31 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 9: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 10: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 11: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 15: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 16: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 18: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 19: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 20: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 24: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 25: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 27: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file [INFO] [stdout] at ./src/../modules/job_processor/core.rs:882:37 [INFO] [stdout] 29: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:854:50 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 32: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 33: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 34: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 35: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 36: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 37: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 38: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 40: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 41: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 42: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 43: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 44: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 45: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 46: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 47: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 48: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 49: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 50: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 51: [INFO] [stdout] 52: clone [INFO] [stdout] 2025-11-05T02:31:46.712534Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.712577Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to get metadata for single file: large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:31:46.712625Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to get metadata for single file: large_files/large_document.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for large_files/large_document.txt [INFO] [stdout] 1: S3 HEAD request failed for 'large_files/large_document.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::JobProcessor::run::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:57:43 [INFO] [stdout] 6: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:882:31 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 9: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 10: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 11: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 15: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 16: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 18: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 19: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 20: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 24: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 25: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 27: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file [INFO] [stdout] at ./src/../modules/job_processor/core.rs:882:37 [INFO] [stdout] 29: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:854:50 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 32: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 33: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 34: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 35: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 36: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 37: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 38: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 40: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 41: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 42: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 43: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 44: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 45: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 46: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 47: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 48: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 49: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 50: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 51: [INFO] [stdout] 52: clone [INFO] [stdout] 2025-11-05T02:31:46.714789Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2025-11-05T02:31:46.705648Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/' [INFO] [stdout] 2025-11-05T02:31:46.715828Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.715889Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:31:46.715928Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2025-11-05T02:31:46.715972Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2025-11-05T02:31:46.716037Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.716148Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/' [INFO] [stdout] 2025-11-05T02:31:46.716166Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.716181Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2025-11-05T02:31:46.716236Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.716251Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'text_files/' [INFO] [stdout] 2025-11-05T02:31:46.716260Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:31:46.716272Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2025-11-05T02:31:46.716294Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] 2025-11-05T02:31:46.721282Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file ... ok [INFO] [stdout] 2025-11-05T02:31:46.723239Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/styles.css': dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.723404Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.723455Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.723465Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/styles.css' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.723482Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:31:46.723492Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.723502Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.723511Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:31:46.723642Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/styles.css': Failed to get metadata for test-data/styles.css [INFO] [stdout] 2025-11-05T02:31:46.723673Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:31:46.723683Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/styles.css [INFO] [stdout] 2025-11-05T02:31:46.723696Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:31:46.723705Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:31:46.723713Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:31:46.723727Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/styles.css [INFO] [stdout] 2025-11-05T02:31:46.723765Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/styles.css [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/styles.css [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/styles.css': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:31:46.724577Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.724627Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.724639Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/markup.html' [INFO] [stdout] 2025-11-05T02:31:46.724648Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.724662Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:31:46.724677Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.724687Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/markup.html' [INFO] [stdout] 2025-11-05T02:31:46.724697Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/markup.html [INFO] [stdout] 2025-11-05T02:31:46.724710Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:31:46.724723Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:31:46.724733Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/markup.html' [INFO] [stdout] 2025-11-05T02:31:46.724760Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.724772Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/markup.html [INFO] [stdout] 2025-11-05T02:31:46.724785Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:31:46.726857Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.726911Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.726921Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'text_files/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.726937Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2025-11-05T02:31:46.726947Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.726956Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2025-11-05T02:31:46.726968Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.727095Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:31:46.727115Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1024:46 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1017:31 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:980:52 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:31:46.727959Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.727980Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:31:46.727997Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1024:46 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1017:31 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:980:52 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:31:46.728563Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2025-11-05T02:31:46.732286Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.732349Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.732392Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.732405Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'text_files/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.732436Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2025-11-05T02:31:46.732447Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.732456Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2025-11-05T02:31:46.732468Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.732594Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_config_validation ... ok [INFO] [stdout] 2025-11-05T02:31:46.736492Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/markup.html': dispatch failure [INFO] [stdout] 2025-11-05T02:31:46.736541Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:31:46.736574Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:31:46.736584Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/markup.html' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:31:46.736599Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:31:46.736608Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:31:46.736618Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:31:46.736626Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:31:46.732617Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::run::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:57:43 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:787:31 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 18: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end [INFO] [stdout] at ./src/../modules/job_processor/core.rs:787:37 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:759:52 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 35: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 36: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 41: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 42: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 44: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 45: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 47: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 49: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2025-11-05T02:31:46.737586Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:31:46.737607Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:31:46.737907Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/markup.html': Failed to get metadata for test-data/markup.html [INFO] [stdout] 2025-11-05T02:31:46.737938Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:31:46.737948Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/markup.html [INFO] [stdout] 2025-11-05T02:31:46.737961Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:31:46.737970Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:31:46.737979Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:31:46.737991Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/markup.html [INFO] [stdout] 2025-11-05T02:31:46.737626Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::run::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:57:43 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:787:31 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 18: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end [INFO] [stdout] at ./src/../modules/job_processor/core.rs:787:37 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:759:52 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 35: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 36: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 41: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 42: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 44: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 45: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 47: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 49: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2025-11-05T02:31:46.739248Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2025-11-05T02:31:46.738004Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/markup.html [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/markup.html [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/markup.html': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:315:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/local.rs:279:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:31:46.740370Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end ... ok [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_file_type_detection ... ok [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter::tests::test_binary_file_rejection stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter::tests::test_binary_file_rejection' (37) panicked at src/../modules/file_splitter/splitter.rs:137:9: [INFO] [stdout] assertion failed: result.is_err() [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::libunwind::trace::h62f3c6bea0fedab3 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::trace_unsynchronized::hc41aaca6c0af0bde [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x5e9258dbdf82 - std::sys::backtrace::_print_fmt::h33ac2b97007106cc [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x5e9258dbdf82 - ::fmt::h4e0a3aeea0f9c085 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x5e9258dd1a8f - core::fmt::rt::Argument::fmt::h1edd6a3e00b22f10 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x5e9258dd1a8f - core::fmt::write::hecf68a131630c74d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/mod.rs:1468:25 [INFO] [stdout] 6: 0x5e9258d87e91 - std::io::default_write_fmt::ha6b238eff7f0ef8a [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x5e9258d87e91 - std::io::Write::write_fmt::h9846fe3d2a36c1ea [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x5e9258d94cf2 - std::sys::backtrace::BacktraceLock::print::h75160192768e5621 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x5e9258d9a8af - std::panicking::default_hook::{{closure}}::h14d82797cfb1ddcb [INFO] [stderr] error: test failed, to rerun pass `--lib` [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x5e9258d9a741 - std::panicking::default_hook::h63f9bf8161c5d325 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x5e9257adecce - as core::ops::function::Fn>::call::hb8126f8384f98101 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 12: 0x5e9257adecce - test::test_main_with_exit_callback::{{closure}}::hae96ae0cd2f2ce70 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x5e9258d9b01f - as core::ops::function::Fn>::call::ha703f6686c81d0c6 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 14: 0x5e9258d9b01f - std::panicking::panic_with_hook::h3173740e06bd0752 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x5e9258d9ad96 - std::panicking::panic_handler::{{closure}}::hbac492c61eb56a87 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:700:13 [INFO] [stdout] 16: 0x5e9258d94e39 - std::sys::backtrace::__rust_end_short_backtrace::haa3eac3df9535320 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x5e9258d7b58d - __rustc[de0091b922c53d7e]::rust_begin_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x5e9258ddc970 - core::panicking::panic_fmt::h5138da2ef87be35b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:75:14 [INFO] [stdout] 19: 0x5e9258ddc94c - core::panicking::panic::hb515f22c0e98143a [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:145:5 [INFO] [stdout] 20: 0x5e92579fc560 - ai_workbench_lib::modules::file_splitter::splitter::tests::test_binary_file_rejection::h685d33ef407ca2a9 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:137:9 [INFO] [stdout] 21: 0x5e92579fc5a7 - ai_workbench_lib::modules::file_splitter::splitter::tests::test_binary_file_rejection::{{closure}}::ha08781190560f944 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:130:36 [INFO] [stdout] 22: 0x5e92579eb966 - core::ops::function::FnOnce::call_once::hb68fafa478b24e65 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 23: 0x5e9257adeaab - core::ops::function::FnOnce::call_once::h8f50ae93d93b62b4 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x5e9257adeaab - test::__rust_begin_short_backtrace::h447963718d05a644 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 25: 0x5e9257af46e5 - test::run_test_in_process::{{closure}}::hf890ba4755ca86c0 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 26: 0x5e9257af46e5 - as core::ops::function::FnOnce<()>>::call_once::h31e4b2860e583faa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 27: 0x5e9257af46e5 - std::panicking::catch_unwind::do_call::hec7cdbbe797e2cab [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 28: 0x5e9257af46e5 - std::panicking::catch_unwind::h24d3b93458b5ba67 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 29: 0x5e9257af46e5 - std::panic::catch_unwind::h46eeec7cf9c9336c [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 30: 0x5e9257af46e5 - test::run_test_in_process::h290de6e559006104 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 31: 0x5e9257af46e5 - test::run_test::{{closure}}::h26eeb2c5f7e9a995 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 32: 0x5e9257acaee4 - test::run_test::{{closure}}::h85af00882daabeaa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 33: 0x5e9257acaee4 - std::sys::backtrace::__rust_begin_short_backtrace::h9d9305dda51fa5f2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 34: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hc6aada8b43f0527b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 35: 0x5e9257ace88a - as core::ops::function::FnOnce<()>>::call_once::ha89fe5505c1206f5 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 36: 0x5e9257ace88a - std::panicking::catch_unwind::do_call::he3e0bef721d99583 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 37: 0x5e9257ace88a - std::panicking::catch_unwind::ha8241964d3d773cd [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 38: 0x5e9257ace88a - std::panic::catch_unwind::ha8554bc7696cae72 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 39: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::he06f1f62b4c90f8d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 40: 0x5e9257ace88a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha7fd20ecae19b9b9 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 41: 0x5e9258d8ff3f - as core::ops::function::FnOnce>::call_once::h43642ed9c40e0ab2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 42: 0x5e9258d8ff3f - std::sys::thread::unix::Thread::new::thread_start::h38da0f633f090ce2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 43: 0x7a4ec0b57aa4 - [INFO] [stdout] 44: 0x7a4ec0be4a64 - clone [INFO] [stdout] 45: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter::integration_tests::test_mixed_file_types stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter::integration_tests::test_mixed_file_types' (35) panicked at src/../modules/file_splitter/splitter.rs:245:13: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: Text [INFO] [stdout] right: Xml [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::libunwind::trace::h62f3c6bea0fedab3 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::trace_unsynchronized::hc41aaca6c0af0bde [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x5e9258dbdf82 - std::sys::backtrace::_print_fmt::h33ac2b97007106cc [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x5e9258dbdf82 - ::fmt::h4e0a3aeea0f9c085 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x5e9258dd1a8f - core::fmt::rt::Argument::fmt::h1edd6a3e00b22f10 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x5e9258dd1a8f - core::fmt::write::hecf68a131630c74d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/mod.rs:1468:25 [INFO] [stdout] 6: 0x5e9258d87e91 - std::io::default_write_fmt::ha6b238eff7f0ef8a [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x5e9258d87e91 - std::io::Write::write_fmt::h9846fe3d2a36c1ea [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x5e9258d94cf2 - std::sys::backtrace::BacktraceLock::print::h75160192768e5621 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x5e9258d9a8af - std::panicking::default_hook::{{closure}}::h14d82797cfb1ddcb [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x5e9258d9a741 - std::panicking::default_hook::h63f9bf8161c5d325 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x5e9257adecce - as core::ops::function::Fn>::call::hb8126f8384f98101 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 12: 0x5e9257adecce - test::test_main_with_exit_callback::{{closure}}::hae96ae0cd2f2ce70 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x5e9258d9b01f - as core::ops::function::Fn>::call::ha703f6686c81d0c6 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 14: 0x5e9258d9b01f - std::panicking::panic_with_hook::h3173740e06bd0752 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x5e9258d9adca - std::panicking::panic_handler::{{closure}}::hbac492c61eb56a87 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:707:13 [INFO] [stdout] 16: 0x5e9258d94e39 - std::sys::backtrace::__rust_end_short_backtrace::haa3eac3df9535320 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x5e9258d7b58d - __rustc[de0091b922c53d7e]::rust_begin_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x5e9258ddc970 - core::panicking::panic_fmt::h5138da2ef87be35b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:75:14 [INFO] [stdout] 19: 0x5e9258ddc723 - core::panicking::assert_failed_inner::h1c85740028aaa416 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:439:17 [INFO] [stdout] 20: 0x5e9257a51556 - core::panicking::assert_failed::h94d5d582705c1e9a [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:394:5 [INFO] [stdout] 21: 0x5e92579fa15c - ai_workbench_lib::modules::file_splitter::splitter::integration_tests::test_mixed_file_types::h8dfc56c05b9d2f4b [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:245:13 [INFO] [stdout] 22: 0x5e92579fa237 - ai_workbench_lib::modules::file_splitter::splitter::integration_tests::test_mixed_file_types::{{closure}}::h818f7524ccc64a62 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:231:31 [INFO] [stdout] 23: 0x5e92579eaf56 - core::ops::function::FnOnce::call_once::h335a49dfddee4069 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x5e9257adeaab - core::ops::function::FnOnce::call_once::h8f50ae93d93b62b4 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x5e9257adeaab - test::__rust_begin_short_backtrace::h447963718d05a644 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 26: 0x5e9257af46e5 - test::run_test_in_process::{{closure}}::hf890ba4755ca86c0 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 27: 0x5e9257af46e5 - as core::ops::function::FnOnce<()>>::call_once::h31e4b2860e583faa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 28: 0x5e9257af46e5 - std::panicking::catch_unwind::do_call::hec7cdbbe797e2cab [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 29: 0x5e9257af46e5 - std::panicking::catch_unwind::h24d3b93458b5ba67 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 30: 0x5e9257af46e5 - std::panic::catch_unwind::h46eeec7cf9c9336c [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 31: 0x5e9257af46e5 - test::run_test_in_process::h290de6e559006104 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 32: 0x5e9257af46e5 - test::run_test::{{closure}}::h26eeb2c5f7e9a995 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 33: 0x5e9257acaee4 - test::run_test::{{closure}}::h85af00882daabeaa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 34: 0x5e9257acaee4 - std::sys::backtrace::__rust_begin_short_backtrace::h9d9305dda51fa5f2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 35: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hc6aada8b43f0527b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 36: 0x5e9257ace88a - as core::ops::function::FnOnce<()>>::call_once::ha89fe5505c1206f5 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 37: 0x5e9257ace88a - std::panicking::catch_unwind::do_call::he3e0bef721d99583 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 38: 0x5e9257ace88a - std::panicking::catch_unwind::ha8241964d3d773cd [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 39: 0x5e9257ace88a - std::panic::catch_unwind::ha8554bc7696cae72 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 40: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::he06f1f62b4c90f8d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 41: 0x5e9257ace88a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha7fd20ecae19b9b9 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 42: 0x5e9258d8ff3f - as core::ops::function::FnOnce>::call_once::h43642ed9c40e0ab2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 43: 0x5e9258d8ff3f - std::sys::thread::unix::Thread::new::thread_start::h38da0f633f090ce2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 44: 0x7a4ec0b57aa4 - [INFO] [stdout] 45: 0x7a4ec0be4a64 - clone [INFO] [stdout] 46: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection' (50) panicked at src/../modules/file_splitter/splitter_types/json_splitter.rs:364:9: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: JsonLines [INFO] [stdout] right: Standard [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::libunwind::trace::h62f3c6bea0fedab3 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::trace_unsynchronized::hc41aaca6c0af0bde [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x5e9258dbdf82 - std::sys::backtrace::_print_fmt::h33ac2b97007106cc [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x5e9258dbdf82 - ::fmt::h4e0a3aeea0f9c085 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x5e9258dd1a8f - core::fmt::rt::Argument::fmt::h1edd6a3e00b22f10 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x5e9258dd1a8f - core::fmt::write::hecf68a131630c74d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/mod.rs:1468:25 [INFO] [stdout] 6: 0x5e9258d87e91 - std::io::default_write_fmt::ha6b238eff7f0ef8a [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x5e9258d87e91 - std::io::Write::write_fmt::h9846fe3d2a36c1ea [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x5e9258d94cf2 - std::sys::backtrace::BacktraceLock::print::h75160192768e5621 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x5e9258d9a8af - std::panicking::default_hook::{{closure}}::h14d82797cfb1ddcb [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x5e9258d9a741 - std::panicking::default_hook::h63f9bf8161c5d325 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x5e9257adecce - as core::ops::function::Fn>::call::hb8126f8384f98101 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 12: 0x5e9257adecce - test::test_main_with_exit_callback::{{closure}}::hae96ae0cd2f2ce70 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x5e9258d9b01f - as core::ops::function::Fn>::call::ha703f6686c81d0c6 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 14: 0x5e9258d9b01f - std::panicking::panic_with_hook::h3173740e06bd0752 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x5e9258d9adca - std::panicking::panic_handler::{{closure}}::hbac492c61eb56a87 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:707:13 [INFO] [stdout] 16: 0x5e9258d94e39 - std::sys::backtrace::__rust_end_short_backtrace::haa3eac3df9535320 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x5e9258d7b58d - __rustc[de0091b922c53d7e]::rust_begin_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x5e9258ddc970 - core::panicking::panic_fmt::h5138da2ef87be35b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:75:14 [INFO] [stdout] 19: 0x5e9258ddc723 - core::panicking::assert_failed_inner::h1c85740028aaa416 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:439:17 [INFO] [stdout] 20: 0x5e9257a51517 - core::panicking::assert_failed::h12c8df9a288deb34 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:394:5 [INFO] [stdout] 21: 0x5e92579f7889 - ai_workbench_lib::modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection::h2cf49804dd994548 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:364:9 [INFO] [stdout] 22: 0x5e92579f78b7 - ai_workbench_lib::modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection::{{closure}}::h4b96d12041b91781 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:358:35 [INFO] [stdout] 23: 0x5e92579eaed6 - core::ops::function::FnOnce::call_once::h28e6ed1ed1570233 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x5e9257adeaab - core::ops::function::FnOnce::call_once::h8f50ae93d93b62b4 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x5e9257adeaab - test::__rust_begin_short_backtrace::h447963718d05a644 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 26: 0x5e9257af46e5 - test::run_test_in_process::{{closure}}::hf890ba4755ca86c0 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 27: 0x5e9257af46e5 - as core::ops::function::FnOnce<()>>::call_once::h31e4b2860e583faa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 28: 0x5e9257af46e5 - std::panicking::catch_unwind::do_call::hec7cdbbe797e2cab [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 29: 0x5e9257af46e5 - std::panicking::catch_unwind::h24d3b93458b5ba67 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 30: 0x5e9257af46e5 - std::panic::catch_unwind::h46eeec7cf9c9336c [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 31: 0x5e9257af46e5 - test::run_test_in_process::h290de6e559006104 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 32: 0x5e9257af46e5 - test::run_test::{{closure}}::h26eeb2c5f7e9a995 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 33: 0x5e9257acaee4 - test::run_test::{{closure}}::h85af00882daabeaa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 34: 0x5e9257acaee4 - std::sys::backtrace::__rust_begin_short_backtrace::h9d9305dda51fa5f2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 35: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hc6aada8b43f0527b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 36: 0x5e9257ace88a - as core::ops::function::FnOnce<()>>::call_once::ha89fe5505c1206f5 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 37: 0x5e9257ace88a - std::panicking::catch_unwind::do_call::he3e0bef721d99583 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 38: 0x5e9257ace88a - std::panicking::catch_unwind::ha8241964d3d773cd [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 39: 0x5e9257ace88a - std::panic::catch_unwind::ha8554bc7696cae72 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 40: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::he06f1f62b4c90f8d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 41: 0x5e9257ace88a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha7fd20ecae19b9b9 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 42: 0x5e9258d8ff3f - as core::ops::function::FnOnce>::call_once::h43642ed9c40e0ab2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 43: 0x5e9258d8ff3f - std::sys::thread::unix::Thread::new::thread_start::h38da0f633f090ce2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 44: 0x7a4ec0b57aa4 - [INFO] [stdout] 45: 0x7a4ec0be4a64 - clone [INFO] [stdout] 46: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection' (53) panicked at src/../modules/file_splitter/splitter_types/json_splitter.rs:354:9: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: JsonLines [INFO] [stdout] right: Standard [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::libunwind::trace::h62f3c6bea0fedab3 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::trace_unsynchronized::hc41aaca6c0af0bde [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x5e9258dbdf82 - std::sys::backtrace::_print_fmt::h33ac2b97007106cc [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x5e9258dbdf82 - ::fmt::h4e0a3aeea0f9c085 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x5e9258dd1a8f - core::fmt::rt::Argument::fmt::h1edd6a3e00b22f10 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x5e9258dd1a8f - core::fmt::write::hecf68a131630c74d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/mod.rs:1468:25 [INFO] [stdout] 6: 0x5e9258d87e91 - std::io::default_write_fmt::ha6b238eff7f0ef8a [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x5e9258d87e91 - std::io::Write::write_fmt::h9846fe3d2a36c1ea [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x5e9258d94cf2 - std::sys::backtrace::BacktraceLock::print::h75160192768e5621 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x5e9258d9a8af - std::panicking::default_hook::{{closure}}::h14d82797cfb1ddcb [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x5e9258d9a741 - std::panicking::default_hook::h63f9bf8161c5d325 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x5e9257adecce - as core::ops::function::Fn>::call::hb8126f8384f98101 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 12: 0x5e9257adecce - test::test_main_with_exit_callback::{{closure}}::hae96ae0cd2f2ce70 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x5e9258d9b01f - as core::ops::function::Fn>::call::ha703f6686c81d0c6 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 14: 0x5e9258d9b01f - std::panicking::panic_with_hook::h3173740e06bd0752 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x5e9258d9adca - std::panicking::panic_handler::{{closure}}::hbac492c61eb56a87 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:707:13 [INFO] [stdout] 16: 0x5e9258d94e39 - std::sys::backtrace::__rust_end_short_backtrace::haa3eac3df9535320 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x5e9258d7b58d - __rustc[de0091b922c53d7e]::rust_begin_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x5e9258ddc970 - core::panicking::panic_fmt::h5138da2ef87be35b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:75:14 [INFO] [stdout] 19: 0x5e9258ddc723 - core::panicking::assert_failed_inner::h1c85740028aaa416 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:439:17 [INFO] [stdout] 20: 0x5e9257a51517 - core::panicking::assert_failed::h12c8df9a288deb34 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:394:5 [INFO] [stdout] 21: 0x5e92579f83d9 - ai_workbench_lib::modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection::h494e08821af32359 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:354:9 [INFO] [stdout] 22: 0x5e92579f8407 - ai_workbench_lib::modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection::{{closure}}::h140fce5e797708da [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:348:38 [INFO] [stdout] 23: 0x5e92579eba06 - core::ops::function::FnOnce::call_once::hbb13919b22ce9661 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x5e9257adeaab - core::ops::function::FnOnce::call_once::h8f50ae93d93b62b4 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x5e9257adeaab - test::__rust_begin_short_backtrace::h447963718d05a644 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 26: 0x5e9257af46e5 - test::run_test_in_process::{{closure}}::hf890ba4755ca86c0 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 27: 0x5e9257af46e5 - as core::ops::function::FnOnce<()>>::call_once::h31e4b2860e583faa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 28: 0x5e9257af46e5 - std::panicking::catch_unwind::do_call::hec7cdbbe797e2cab [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 29: 0x5e9257af46e5 - std::panicking::catch_unwind::h24d3b93458b5ba67 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 30: 0x5e9257af46e5 - std::panic::catch_unwind::h46eeec7cf9c9336c [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 31: 0x5e9257af46e5 - test::run_test_in_process::h290de6e559006104 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 32: 0x5e9257af46e5 - test::run_test::{{closure}}::h26eeb2c5f7e9a995 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 33: 0x5e9257acaee4 - test::run_test::{{closure}}::h85af00882daabeaa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 34: 0x5e9257acaee4 - std::sys::backtrace::__rust_begin_short_backtrace::h9d9305dda51fa5f2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 35: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hc6aada8b43f0527b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 36: 0x5e9257ace88a - as core::ops::function::FnOnce<()>>::call_once::ha89fe5505c1206f5 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 37: 0x5e9257ace88a - std::panicking::catch_unwind::do_call::he3e0bef721d99583 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 38: 0x5e9257ace88a - std::panicking::catch_unwind::ha8241964d3d773cd [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 39: 0x5e9257ace88a - std::panic::catch_unwind::ha8554bc7696cae72 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 40: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::he06f1f62b4c90f8d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 41: 0x5e9257ace88a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha7fd20ecae19b9b9 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 42: 0x5e9258d8ff3f - as core::ops::function::FnOnce>::call_once::h43642ed9c40e0ab2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 43: 0x5e9258d8ff3f - std::sys::thread::unix::Thread::new::thread_start::h38da0f633f090ce2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 44: 0x7a4ec0b57aa4 - [INFO] [stdout] 45: 0x7a4ec0be4a64 - clone [INFO] [stdout] 46: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter::integration_tests::test_performance_characteristics stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter::integration_tests::test_performance_characteristics' (36) panicked at src/../modules/file_splitter/splitter.rs:288:9: [INFO] [stdout] assertion failed: chunks.len() > 1 [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::libunwind::trace::h62f3c6bea0fedab3 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::trace_unsynchronized::hc41aaca6c0af0bde [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x5e9258dbdf82 - std::sys::backtrace::_print_fmt::h33ac2b97007106cc [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x5e9258dbdf82 - ::fmt::h4e0a3aeea0f9c085 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x5e9258dd1a8f - core::fmt::rt::Argument::fmt::h1edd6a3e00b22f10 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x5e9258dd1a8f - core::fmt::write::hecf68a131630c74d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/mod.rs:1468:25 [INFO] [stdout] 6: 0x5e9258d87e91 - std::io::default_write_fmt::ha6b238eff7f0ef8a [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x5e9258d87e91 - std::io::Write::write_fmt::h9846fe3d2a36c1ea [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x5e9258d94cf2 - std::sys::backtrace::BacktraceLock::print::h75160192768e5621 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x5e9258d9a8af - std::panicking::default_hook::{{closure}}::h14d82797cfb1ddcb [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x5e9258d9a741 - std::panicking::default_hook::h63f9bf8161c5d325 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x5e9257adecce - as core::ops::function::Fn>::call::hb8126f8384f98101 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 12: 0x5e9257adecce - test::test_main_with_exit_callback::{{closure}}::hae96ae0cd2f2ce70 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x5e9258d9b01f - as core::ops::function::Fn>::call::ha703f6686c81d0c6 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 14: 0x5e9258d9b01f - std::panicking::panic_with_hook::h3173740e06bd0752 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x5e9258d9ad96 - std::panicking::panic_handler::{{closure}}::hbac492c61eb56a87 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:700:13 [INFO] [stdout] 16: 0x5e9258d94e39 - std::sys::backtrace::__rust_end_short_backtrace::haa3eac3df9535320 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x5e9258d7b58d - __rustc[de0091b922c53d7e]::rust_begin_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x5e9258ddc970 - core::panicking::panic_fmt::h5138da2ef87be35b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:75:14 [INFO] [stdout] 19: 0x5e9258ddc94c - core::panicking::panic::hb515f22c0e98143a [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:145:5 [INFO] [stdout] 20: 0x5e92579fb9ce - ai_workbench_lib::modules::file_splitter::splitter::integration_tests::test_performance_characteristics::h84f1837170165ff4 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:288:9 [INFO] [stdout] 21: 0x5e92579fbb57 - ai_workbench_lib::modules::file_splitter::splitter::integration_tests::test_performance_characteristics::{{closure}}::h0b2439c75ebb67e6 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:273:42 [INFO] [stdout] 22: 0x5e92579eb386 - core::ops::function::FnOnce::call_once::h534cc342357b0a06 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 23: 0x5e9257adeaab - core::ops::function::FnOnce::call_once::h8f50ae93d93b62b4 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x5e9257adeaab - test::__rust_begin_short_backtrace::h447963718d05a644 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 25: 0x5e9257af46e5 - test::run_test_in_process::{{closure}}::hf890ba4755ca86c0 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 26: 0x5e9257af46e5 - as core::ops::function::FnOnce<()>>::call_once::h31e4b2860e583faa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 27: 0x5e9257af46e5 - std::panicking::catch_unwind::do_call::hec7cdbbe797e2cab [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 28: 0x5e9257af46e5 - std::panicking::catch_unwind::h24d3b93458b5ba67 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 29: 0x5e9257af46e5 - std::panic::catch_unwind::h46eeec7cf9c9336c [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 30: 0x5e9257af46e5 - test::run_test_in_process::h290de6e559006104 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 31: 0x5e9257af46e5 - test::run_test::{{closure}}::h26eeb2c5f7e9a995 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 32: 0x5e9257acaee4 - test::run_test::{{closure}}::h85af00882daabeaa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 33: 0x5e9257acaee4 - std::sys::backtrace::__rust_begin_short_backtrace::h9d9305dda51fa5f2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 34: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hc6aada8b43f0527b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 35: 0x5e9257ace88a - as core::ops::function::FnOnce<()>>::call_once::ha89fe5505c1206f5 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 36: 0x5e9257ace88a - std::panicking::catch_unwind::do_call::he3e0bef721d99583 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 37: 0x5e9257ace88a - std::panicking::catch_unwind::ha8241964d3d773cd [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 38: 0x5e9257ace88a - std::panic::catch_unwind::ha8554bc7696cae72 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 39: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::he06f1f62b4c90f8d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 40: 0x5e9257ace88a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha7fd20ecae19b9b9 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 41: 0x5e9258d8ff3f - as core::ops::function::FnOnce>::call_once::h43642ed9c40e0ab2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 42: 0x5e9258d8ff3f - std::sys::thread::unix::Thread::new::thread_start::h38da0f633f090ce2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 43: 0x7a4ec0b57aa4 - [INFO] [stdout] 44: 0x7a4ec0be4a64 - clone [INFO] [stdout] 45: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter::integration_tests::test_edge_cases stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter::integration_tests::test_edge_cases' (32) panicked at src/../modules/file_splitter/splitter.rs:263:9: [INFO] [stdout] assertion failed: chunks[0].metadata.has_headers [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::libunwind::trace::h62f3c6bea0fedab3 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::trace_unsynchronized::hc41aaca6c0af0bde [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x5e9258dbdf82 - std::sys::backtrace::_print_fmt::h33ac2b97007106cc [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x5e9258dbdf82 - ::fmt::h4e0a3aeea0f9c085 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x5e9258dd1a8f - core::fmt::rt::Argument::fmt::h1edd6a3e00b22f10 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x5e9258dd1a8f - core::fmt::write::hecf68a131630c74d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/mod.rs:1468:25 [INFO] [stdout] 6: 0x5e9258d87e91 - std::io::default_write_fmt::ha6b238eff7f0ef8a [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x5e9258d87e91 - std::io::Write::write_fmt::h9846fe3d2a36c1ea [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x5e9258d94cf2 - std::sys::backtrace::BacktraceLock::print::h75160192768e5621 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x5e9258d9a8af - std::panicking::default_hook::{{closure}}::h14d82797cfb1ddcb [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x5e9258d9a741 - std::panicking::default_hook::h63f9bf8161c5d325 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x5e9257adecce - as core::ops::function::Fn>::call::hb8126f8384f98101 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 12: 0x5e9257adecce - test::test_main_with_exit_callback::{{closure}}::hae96ae0cd2f2ce70 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x5e9258d9b01f - as core::ops::function::Fn>::call::ha703f6686c81d0c6 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 14: 0x5e9258d9b01f - std::panicking::panic_with_hook::h3173740e06bd0752 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x5e9258d9ad96 - std::panicking::panic_handler::{{closure}}::hbac492c61eb56a87 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:700:13 [INFO] [stdout] 16: 0x5e9258d94e39 - std::sys::backtrace::__rust_end_short_backtrace::haa3eac3df9535320 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x5e9258d7b58d - __rustc[de0091b922c53d7e]::rust_begin_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x5e9258ddc970 - core::panicking::panic_fmt::h5138da2ef87be35b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:75:14 [INFO] [stdout] 19: 0x5e9258ddc94c - core::panicking::panic::hb515f22c0e98143a [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:145:5 [INFO] [stdout] 20: 0x5e92579f9812 - ai_workbench_lib::modules::file_splitter::splitter::integration_tests::test_edge_cases::hcfbf2f07caffd97c [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:263:9 [INFO] [stdout] 21: 0x5e92579f9b17 - ai_workbench_lib::modules::file_splitter::splitter::integration_tests::test_edge_cases::{{closure}}::h161136a9260a6376 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:251:25 [INFO] [stdout] 22: 0x5e92579eb7c6 - core::ops::function::FnOnce::call_once::h89e397df8a9fd238 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 23: 0x5e9257adeaab - core::ops::function::FnOnce::call_once::h8f50ae93d93b62b4 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x5e9257adeaab - test::__rust_begin_short_backtrace::h447963718d05a644 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 25: 0x5e9257af46e5 - test::run_test_in_process::{{closure}}::hf890ba4755ca86c0 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 26: 0x5e9257af46e5 - as core::ops::function::FnOnce<()>>::call_once::h31e4b2860e583faa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 27: 0x5e9257af46e5 - std::panicking::catch_unwind::do_call::hec7cdbbe797e2cab [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 28: 0x5e9257af46e5 - std::panicking::catch_unwind::h24d3b93458b5ba67 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 29: 0x5e9257af46e5 - std::panic::catch_unwind::h46eeec7cf9c9336c [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 30: 0x5e9257af46e5 - test::run_test_in_process::h290de6e559006104 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 31: 0x5e9257af46e5 - test::run_test::{{closure}}::h26eeb2c5f7e9a995 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 32: 0x5e9257acaee4 - test::run_test::{{closure}}::h85af00882daabeaa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 33: 0x5e9257acaee4 - std::sys::backtrace::__rust_begin_short_backtrace::h9d9305dda51fa5f2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 34: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hc6aada8b43f0527b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 35: 0x5e9257ace88a - as core::ops::function::FnOnce<()>>::call_once::ha89fe5505c1206f5 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 36: 0x5e9257ace88a - std::panicking::catch_unwind::do_call::he3e0bef721d99583 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 37: 0x5e9257ace88a - std::panicking::catch_unwind::ha8241964d3d773cd [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 38: 0x5e9257ace88a - std::panic::catch_unwind::ha8554bc7696cae72 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 39: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::he06f1f62b4c90f8d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 40: 0x5e9257ace88a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha7fd20ecae19b9b9 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 41: 0x5e9258d8ff3f - as core::ops::function::FnOnce>::call_once::h43642ed9c40e0ab2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 42: 0x5e9258d8ff3f - std::sys::thread::unix::Thread::new::thread_start::h38da0f633f090ce2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 43: 0x7a4ec0b57aa4 - [INFO] [stdout] 44: 0x7a4ec0be4a64 - clone [INFO] [stdout] 45: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection' (49) panicked at src/../modules/file_splitter/splitter_types/json_splitter.rs:400:9: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: JsonLines [INFO] [stdout] right: Invalid [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::libunwind::trace::h62f3c6bea0fedab3 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x5e9258dbdf82 - std::backtrace_rs::backtrace::trace_unsynchronized::hc41aaca6c0af0bde [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x5e9258dbdf82 - std::sys::backtrace::_print_fmt::h33ac2b97007106cc [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x5e9258dbdf82 - ::fmt::h4e0a3aeea0f9c085 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x5e9258dd1a8f - core::fmt::rt::Argument::fmt::h1edd6a3e00b22f10 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x5e9258dd1a8f - core::fmt::write::hecf68a131630c74d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/fmt/mod.rs:1468:25 [INFO] [stdout] 6: 0x5e9258d87e91 - std::io::default_write_fmt::ha6b238eff7f0ef8a [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x5e9258d87e91 - std::io::Write::write_fmt::h9846fe3d2a36c1ea [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x5e9258d94cf2 - std::sys::backtrace::BacktraceLock::print::h75160192768e5621 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x5e9258d9a8af - std::panicking::default_hook::{{closure}}::h14d82797cfb1ddcb [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x5e9258d9a741 - std::panicking::default_hook::h63f9bf8161c5d325 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x5e9257adecce - as core::ops::function::Fn>::call::hb8126f8384f98101 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 12: 0x5e9257adecce - test::test_main_with_exit_callback::{{closure}}::hae96ae0cd2f2ce70 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x5e9258d9b01f - as core::ops::function::Fn>::call::ha703f6686c81d0c6 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1999:9 [INFO] [stdout] 14: 0x5e9258d9b01f - std::panicking::panic_with_hook::h3173740e06bd0752 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x5e9258d9adca - std::panicking::panic_handler::{{closure}}::hbac492c61eb56a87 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:707:13 [INFO] [stdout] 16: 0x5e9258d94e39 - std::sys::backtrace::__rust_end_short_backtrace::haa3eac3df9535320 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x5e9258d7b58d - __rustc[de0091b922c53d7e]::rust_begin_unwind [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x5e9258ddc970 - core::panicking::panic_fmt::h5138da2ef87be35b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:75:14 [INFO] [stdout] 19: 0x5e9258ddc723 - core::panicking::assert_failed_inner::h1c85740028aaa416 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:439:17 [INFO] [stdout] 20: 0x5e9257a51517 - core::panicking::assert_failed::h12c8df9a288deb34 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panicking.rs:394:5 [INFO] [stdout] 21: 0x5e92579f8259 - ai_workbench_lib::modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection::h747599e4b8a9bac7 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:400:9 [INFO] [stdout] 22: 0x5e92579f8287 - ai_workbench_lib::modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection::{{closure}}::he74c736939379660 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:395:37 [INFO] [stdout] 23: 0x5e92579eaf96 - core::ops::function::FnOnce::call_once::h38f5cb51435cf10b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x5e9257adeaab - core::ops::function::FnOnce::call_once::h8f50ae93d93b62b4 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x5e9257adeaab - test::__rust_begin_short_backtrace::h447963718d05a644 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:663:18 [INFO] [stdout] 26: 0x5e9257af46e5 - test::run_test_in_process::{{closure}}::hf890ba4755ca86c0 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:74 [INFO] [stdout] 27: 0x5e9257af46e5 - as core::ops::function::FnOnce<()>>::call_once::h31e4b2860e583faa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 28: 0x5e9257af46e5 - std::panicking::catch_unwind::do_call::hec7cdbbe797e2cab [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 29: 0x5e9257af46e5 - std::panicking::catch_unwind::h24d3b93458b5ba67 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 30: 0x5e9257af46e5 - std::panic::catch_unwind::h46eeec7cf9c9336c [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 31: 0x5e9257af46e5 - test::run_test_in_process::h290de6e559006104 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:686:27 [INFO] [stdout] 32: 0x5e9257af46e5 - test::run_test::{{closure}}::h26eeb2c5f7e9a995 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:607:43 [INFO] [stdout] 33: 0x5e9257acaee4 - test::run_test::{{closure}}::h85af00882daabeaa [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/test/src/lib.rs:637:41 [INFO] [stdout] 34: 0x5e9257acaee4 - std::sys::backtrace::__rust_begin_short_backtrace::h9d9305dda51fa5f2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 35: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hc6aada8b43f0527b [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:559:17 [INFO] [stdout] 36: 0x5e9257ace88a - as core::ops::function::FnOnce<()>>::call_once::ha89fe5505c1206f5 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 37: 0x5e9257ace88a - std::panicking::catch_unwind::do_call::he3e0bef721d99583 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:590:40 [INFO] [stdout] 38: 0x5e9257ace88a - std::panicking::catch_unwind::ha8241964d3d773cd [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panicking.rs:553:19 [INFO] [stdout] 39: 0x5e9257ace88a - std::panic::catch_unwind::ha8554bc7696cae72 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/panic.rs:359:14 [INFO] [stdout] 40: 0x5e9257ace88a - std::thread::Builder::spawn_unchecked_::{{closure}}::he06f1f62b4c90f8d [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/thread/mod.rs:557:30 [INFO] [stdout] 41: 0x5e9257ace88a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha7fd20ecae19b9b9 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 42: 0x5e9258d8ff3f - as core::ops::function::FnOnce>::call_once::h43642ed9c40e0ab2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/alloc/src/boxed.rs:1985:9 [INFO] [stdout] 43: 0x5e9258d8ff3f - std::sys::thread::unix::Thread::new::thread_start::h38da0f633f090ce2 [INFO] [stdout] at /rustc/f8297e351a40c1439a467bbbb6879088047f50b3/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 44: 0x7a4ec0b57aa4 - [INFO] [stdout] 45: 0x7a4ec0be4a64 - clone [INFO] [stdout] 46: 0x0 - [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] modules::file_splitter::splitter::integration_tests::test_edge_cases [INFO] [stdout] modules::file_splitter::splitter::integration_tests::test_mixed_file_types [INFO] [stdout] modules::file_splitter::splitter::integration_tests::test_performance_characteristics [INFO] [stdout] modules::file_splitter::splitter::tests::test_binary_file_rejection [INFO] [stdout] modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection [INFO] [stdout] modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection [INFO] [stdout] modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection [INFO] [stdout] [INFO] [stdout] test result: FAILED. 30 passed; 7 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.42s [INFO] [stdout] [INFO] running `Command { std: "docker" "inspect" "eb8ca44228271c5b137cd8cd8ee8cfa277ec52bb452b766ce6f6f29d44ff2902", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "eb8ca44228271c5b137cd8cd8ee8cfa277ec52bb452b766ce6f6f29d44ff2902", kill_on_drop: false }` [INFO] [stdout] eb8ca44228271c5b137cd8cd8ee8cfa277ec52bb452b766ce6f6f29d44ff2902