[INFO] fetching crate ai-workbench-lib 0.4.0... [INFO] testing ai-workbench-lib-0.4.0 against beta-2025-10-28 for beta-1.92-2 [INFO] extracting crate ai-workbench-lib 0.4.0 into /workspace/builds/worker-6-tc2/source [INFO] started tweaking crates.io crate ai-workbench-lib 0.4.0 [INFO] finished tweaking crates.io crate ai-workbench-lib 0.4.0 [INFO] tweaked toml for crates.io crate ai-workbench-lib 0.4.0 written to /workspace/builds/worker-6-tc2/source/Cargo.toml [INFO] validating manifest of crates.io crate ai-workbench-lib 0.4.0 on toolchain beta-2025-10-28 [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+beta-2025-10-28" "metadata" "--manifest-path" "Cargo.toml" "--no-deps", kill_on_drop: false }` [INFO] crate crates.io crate ai-workbench-lib 0.4.0 already has a lockfile, it will not be regenerated [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+beta-2025-10-28" "fetch" "--manifest-path" "Cargo.toml", kill_on_drop: false }` [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:4848fb76d95f26979359cc7e45710b1dbc8f3acb7aeedee7c460d7702230f228" "/opt/rustwide/cargo-home/bin/cargo" "+beta-2025-10-28" "metadata" "--no-deps" "--format-version=1", kill_on_drop: false }` [INFO] [stdout] 0b4d2c27a8a8413b877f3a9c3670d61f107d594c0866ed90b5a89af1f240734e [INFO] running `Command { std: "docker" "start" "-a" "0b4d2c27a8a8413b877f3a9c3670d61f107d594c0866ed90b5a89af1f240734e", kill_on_drop: false }` [INFO] running `Command { std: "docker" "inspect" "0b4d2c27a8a8413b877f3a9c3670d61f107d594c0866ed90b5a89af1f240734e", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "0b4d2c27a8a8413b877f3a9c3670d61f107d594c0866ed90b5a89af1f240734e", kill_on_drop: false }` [INFO] [stdout] 0b4d2c27a8a8413b877f3a9c3670d61f107d594c0866ed90b5a89af1f240734e [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=warn" "-e" "RUSTDOCFLAGS=--cap-lints=warn" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:4848fb76d95f26979359cc7e45710b1dbc8f3acb7aeedee7c460d7702230f228" "/opt/rustwide/cargo-home/bin/cargo" "+beta-2025-10-28" "build" "--frozen" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] 4758e8b54339ff9d94ab161192dbbbb647d2b1bea2d6c518e3095fca37121a05 [INFO] running `Command { std: "docker" "start" "-a" "4758e8b54339ff9d94ab161192dbbbb647d2b1bea2d6c518e3095fca37121a05", kill_on_drop: false }` [INFO] [stderr] Compiling libc v0.2.175 [INFO] [stderr] Compiling proc-macro2 v1.0.101 [INFO] [stderr] Compiling cfg-if v1.0.3 [INFO] [stderr] Compiling find-msvc-tools v0.1.0 [INFO] [stderr] Compiling subtle v2.6.1 [INFO] [stderr] Compiling deranged v0.5.3 [INFO] [stderr] Compiling time-core v0.1.5 [INFO] [stderr] Compiling foldhash v0.1.5 [INFO] [stderr] Compiling aws-lc-rs v1.13.3 [INFO] [stderr] Compiling const-oid v0.9.6 [INFO] [stderr] Compiling base64ct v1.8.0 [INFO] [stderr] Compiling rustls v0.23.31 [INFO] [stderr] Compiling base16ct v0.1.1 [INFO] [stderr] Compiling aws-types v1.3.8 [INFO] [stderr] Compiling serde v1.0.219 [INFO] [stderr] Compiling regex-lite v0.1.7 [INFO] [stderr] Compiling crc32fast v1.5.0 [INFO] [stderr] Compiling regex-syntax v0.8.6 [INFO] [stderr] Compiling hashbrown v0.15.5 [INFO] [stderr] Compiling serde_json v1.0.143 [INFO] [stderr] Compiling anyhow v1.0.99 [INFO] [stderr] Compiling sharded-slab v0.1.7 [INFO] [stderr] Compiling thread_local v1.1.9 [INFO] [stderr] Compiling tracing-log v0.2.0 [INFO] [stderr] Compiling csv-core v0.1.12 [INFO] [stderr] Compiling der v0.6.1 [INFO] [stderr] Compiling digest v0.10.7 [INFO] [stderr] Compiling unicode-id v0.3.5 [INFO] [stderr] Compiling nu-ansi-term v0.50.1 [INFO] [stderr] Compiling hmac v0.12.1 [INFO] [stderr] Compiling sha2 v0.10.9 [INFO] [stderr] Compiling sha1 v0.10.6 [INFO] [stderr] Compiling md-5 v0.10.6 [INFO] [stderr] Compiling markdown v1.0.0 [INFO] [stderr] Compiling quote v1.0.40 [INFO] [stderr] Compiling indexmap v2.11.0 [INFO] [stderr] Compiling lru v0.12.5 [INFO] [stderr] Compiling syn v2.0.106 [INFO] [stderr] Compiling tracing-subscriber v0.3.20 [INFO] [stderr] Compiling spki v0.6.0 [INFO] [stderr] Compiling parking_lot_core v0.9.11 [INFO] [stderr] Compiling signal-hook-registry v1.4.6 [INFO] [stderr] Compiling socket2 v0.6.0 [INFO] [stderr] Compiling mio v1.0.4 [INFO] [stderr] Compiling jobserver v0.1.34 [INFO] [stderr] Compiling getrandom v0.2.16 [INFO] [stderr] Compiling rand_core v0.6.4 [INFO] [stderr] Compiling getrandom v0.3.3 [INFO] [stderr] Compiling pkcs8 v0.9.0 [INFO] [stderr] Compiling socket2 v0.5.10 [INFO] [stderr] Compiling parking_lot v0.12.4 [INFO] [stderr] Compiling ff v0.12.1 [INFO] [stderr] Compiling crypto-bigint v0.4.9 [INFO] [stderr] Compiling group v0.12.1 [INFO] [stderr] Compiling time v0.3.42 [INFO] [stderr] Compiling cc v1.2.35 [INFO] [stderr] Compiling sec1 v0.3.0 [INFO] [stderr] Compiling signature v1.6.4 [INFO] [stderr] Compiling crypto-bigint v0.5.5 [INFO] [stderr] Compiling uuid v1.18.1 [INFO] [stderr] Compiling rand_core v0.9.3 [INFO] [stderr] Compiling rand_chacha v0.9.0 [INFO] [stderr] Compiling regex-automata v0.4.10 [INFO] [stderr] Compiling rand v0.9.2 [INFO] [stderr] Compiling rfc6979 v0.3.1 [INFO] [stderr] Compiling elliptic-curve v0.12.3 [INFO] [stderr] Compiling ecdsa v0.14.8 [INFO] [stderr] Compiling cmake v0.1.54 [INFO] [stderr] Compiling aws-lc-sys v0.30.0 [INFO] [stderr] Compiling ring v0.17.14 [INFO] [stderr] Compiling p256 v0.11.1 [INFO] [stderr] Compiling rustls v0.21.12 [INFO] [stderr] Compiling regex v1.11.2 [INFO] [stderr] Compiling synstructure v0.13.2 [INFO] [stderr] Compiling rustls-webpki v0.101.7 [INFO] [stderr] Compiling sct v0.7.1 [INFO] [stderr] Compiling crc-fast v1.3.0 [INFO] [stderr] Compiling tokio-macros v2.5.0 [INFO] [stderr] Compiling futures-macro v0.3.31 [INFO] [stderr] Compiling tracing-attributes v0.1.30 [INFO] [stderr] Compiling zerofrom-derive v0.1.6 [INFO] [stderr] Compiling yoke-derive v0.8.0 [INFO] [stderr] Compiling zerovec-derive v0.11.1 [INFO] [stderr] Compiling displaydoc v0.2.5 [INFO] [stderr] Compiling serde_derive v1.0.219 [INFO] [stderr] Compiling tokio v1.47.1 [INFO] [stderr] Compiling zerofrom v0.1.6 [INFO] [stderr] Compiling futures-util v0.3.31 [INFO] [stderr] Compiling yoke v0.8.0 [INFO] [stderr] Compiling tracing v0.1.41 [INFO] [stderr] Compiling zerovec v0.11.4 [INFO] [stderr] Compiling zerotrie v0.2.2 [INFO] [stderr] Compiling tinystr v0.8.1 [INFO] [stderr] Compiling potential_utf v0.1.3 [INFO] [stderr] Compiling icu_collections v2.0.0 [INFO] [stderr] Compiling icu_locale_core v2.0.0 [INFO] [stderr] Compiling icu_provider v2.0.0 [INFO] [stderr] Compiling icu_properties v2.0.1 [INFO] [stderr] Compiling icu_normalizer v2.0.0 [INFO] [stderr] Compiling futures-executor v0.3.31 [INFO] [stderr] Compiling futures v0.3.31 [INFO] [stderr] Compiling idna_adapter v1.2.1 [INFO] [stderr] Compiling idna v1.1.0 [INFO] [stderr] Compiling rustls-webpki v0.103.4 [INFO] [stderr] Compiling url v2.5.7 [INFO] [stderr] Compiling csv v1.3.1 [INFO] [stderr] Compiling tokio-util v0.7.16 [INFO] [stderr] Compiling aws-smithy-async v1.2.5 [INFO] [stderr] Compiling tokio-rustls v0.24.1 [INFO] [stderr] Compiling aws-smithy-types v1.3.2 [INFO] [stderr] Compiling h2 v0.4.12 [INFO] [stderr] Compiling h2 v0.3.27 [INFO] [stderr] Compiling aws-smithy-runtime-api v1.9.0 [INFO] [stderr] Compiling aws-smithy-eventstream v0.60.11 [INFO] [stderr] Compiling aws-smithy-json v0.61.5 [INFO] [stderr] Compiling aws-smithy-query v0.60.7 [INFO] [stderr] Compiling aws-smithy-http v0.62.3 [INFO] [stderr] Compiling aws-credential-types v1.2.6 [INFO] [stderr] Compiling aws-smithy-observability v0.1.3 [INFO] [stderr] Compiling aws-smithy-checksums v0.63.8 [INFO] [stderr] Compiling aws-sigv4 v1.3.4 [INFO] [stderr] Compiling hyper v1.7.0 [INFO] [stderr] Compiling tokio-rustls v0.26.2 [INFO] [stderr] Compiling hyper v0.14.32 [INFO] [stderr] Compiling hyper-util v0.1.16 [INFO] [stderr] Compiling hyper-rustls v0.27.7 [INFO] [stderr] Compiling hyper-rustls v0.24.2 [INFO] [stderr] Compiling aws-smithy-http-client v1.1.1 [INFO] [stderr] Compiling aws-smithy-runtime v1.9.1 [INFO] [stderr] Compiling aws-runtime v1.5.10 [INFO] [stderr] Compiling aws-sdk-sso v1.83.0 [INFO] [stderr] Compiling aws-sdk-sts v1.85.0 [INFO] [stderr] Compiling aws-sdk-ssooidc v1.84.0 [INFO] [stderr] Compiling aws-sdk-bedrockruntime v1.105.0 [INFO] [stderr] Compiling aws-sdk-s3 v1.104.0 [INFO] [stderr] Compiling aws-config v1.8.6 [INFO] [stderr] Compiling ai-workbench-lib v0.4.0 (/opt/rustwide/workdir) [INFO] [stderr] Finished `dev` profile [unoptimized + debuginfo] target(s) in 3m 43s [INFO] running `Command { std: "docker" "inspect" "4758e8b54339ff9d94ab161192dbbbb647d2b1bea2d6c518e3095fca37121a05", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "4758e8b54339ff9d94ab161192dbbbb647d2b1bea2d6c518e3095fca37121a05", kill_on_drop: false }` [INFO] [stdout] 4758e8b54339ff9d94ab161192dbbbb647d2b1bea2d6c518e3095fca37121a05 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=warn" "-e" "RUSTDOCFLAGS=--cap-lints=warn" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:4848fb76d95f26979359cc7e45710b1dbc8f3acb7aeedee7c460d7702230f228" "/opt/rustwide/cargo-home/bin/cargo" "+beta-2025-10-28" "test" "--frozen" "--no-run" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] a4eb4b68fd6ae062ad6079899b844dc9431a4eef0230c733ad0d2aefa97d438d [INFO] running `Command { std: "docker" "start" "-a" "a4eb4b68fd6ae062ad6079899b844dc9431a4eef0230c733ad0d2aefa97d438d", kill_on_drop: false }` [INFO] [stderr] Compiling ai-workbench-lib v0.4.0 (/opt/rustwide/workdir) [INFO] [stderr] Finished `test` profile [unoptimized + debuginfo] target(s) in 10.11s [INFO] running `Command { std: "docker" "inspect" "a4eb4b68fd6ae062ad6079899b844dc9431a4eef0230c733ad0d2aefa97d438d", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "a4eb4b68fd6ae062ad6079899b844dc9431a4eef0230c733ad0d2aefa97d438d", kill_on_drop: false }` [INFO] [stdout] a4eb4b68fd6ae062ad6079899b844dc9431a4eef0230c733ad0d2aefa97d438d [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-6-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=warn" "-e" "RUSTDOCFLAGS=--cap-lints=warn" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:4848fb76d95f26979359cc7e45710b1dbc8f3acb7aeedee7c460d7702230f228" "/opt/rustwide/cargo-home/bin/cargo" "+beta-2025-10-28" "test" "--frozen", kill_on_drop: false }` [INFO] [stdout] 5c269d0fc29196fff7b39a0afc81a54ead67e7a95847c233b2892e3b44035595 [INFO] running `Command { std: "docker" "start" "-a" "5c269d0fc29196fff7b39a0afc81a54ead67e7a95847c233b2892e3b44035595", kill_on_drop: false }` [INFO] [stderr] Finished `test` profile [unoptimized + debuginfo] target(s) in 0.36s [INFO] [stderr] Running unittests src/lib.rs (/opt/rustwide/target/debug/deps/ai_workbench_lib-78309b9c82f83a0d) [INFO] [stdout] [INFO] [stdout] running 37 tests [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_file_info_structure ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_invalid_csv_fallback ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_file_type_detection_by_extension ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::code_splitter::tests::test_simple_code_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_json_file_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_csv_format_validation ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_csv_single_chunk ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_record_boundary_preservation ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_csv_multiple_chunks_with_headers ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::csv_splitter::tests::test_tsv_format ... ok [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_large_csv_file_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_json_lines_detection ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_json_lines_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::code_splitter::tests::test_large_file_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::text_splitter::tests::test_large_file_multiple_chunks ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::text_splitter::tests::test_line_boundary_preservation ... ok [INFO] [stdout] test modules::file_splitter::splitter_types::text_splitter::tests::test_small_file_single_chunk ... ok [INFO] [stdout] test modules::file_splitter::splitter::tests::test_empty_file ... ok [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_large_text_file_splitting ... ok [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_mixed_file_types ... FAILED [INFO] [stdout] test modules::file_splitter::splitter::tests::test_binary_file_rejection ... FAILED [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection ... FAILED [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection ... FAILED [INFO] [stdout] 2025-11-05T02:35:43.864161Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.864616Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'mixed-files/' [INFO] [stdout] 2025-11-05T02:35:43.864633Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.864648Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2025-11-05T02:35:43.864662Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.864674Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'mixed-files/' [INFO] [stdout] 2025-11-05T02:35:43.864684Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/mixed-files/ [INFO] [stdout] 2025-11-05T02:35:43.864697Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2025-11-05T02:35:43.864700Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.864713Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] test modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection ... FAILED [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_edge_cases ... FAILED [INFO] [stdout] 2025-11-05T02:35:43.868796Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/' [INFO] [stdout] 2025-11-05T02:35:43.868841Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.868856Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2025-11-05T02:35:43.868872Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.868883Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'text_files/' [INFO] [stdout] 2025-11-05T02:35:43.868892Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:35:43.868903Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2025-11-05T02:35:43.868917Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] 2025-11-05T02:35:43.870078Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.870113Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.870145Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.870153Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'mixed-files/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.870170Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2025-11-05T02:35:43.870179Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.870196Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2025-11-05T02:35:43.870208Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.870289Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:35:43.870890Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.870972Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'large-dataset/' [INFO] [stdout] 2025-11-05T02:35:43.870991Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.871003Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2025-11-05T02:35:43.871016Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.871042Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'large-dataset/' [INFO] [stdout] 2025-11-05T02:35:43.871052Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large-dataset/ [INFO] [stdout] 2025-11-05T02:35:43.871064Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2025-11-05T02:35:43.871076Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] 2025-11-05T02:35:43.873042Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.875777Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/sample.txt' [INFO] [stdout] 2025-11-05T02:35:43.875845Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.875882Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:35:43.875918Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.875947Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/sample.txt' [INFO] [stdout] 2025-11-05T02:35:43.875973Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.876005Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:35:43.876043Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.876087Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/sample.txt' [INFO] [stdout] 2025-11-05T02:35:43.876120Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.876161Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.876190Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:35:43.876267Z  INFO ai_workbench_lib::modules::job_processor::core: Starting job execution for job_id: test-folder-33a03ae5-89f9-41fa-bc18-f67363827011 [INFO] [stdout] 2025-11-05T02:35:43.876354Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.876400Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/' [INFO] [stdout] 2025-11-05T02:35:43.876449Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.876486Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:35:43.876519Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2025-11-05T02:35:43.876561Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2025-11-05T02:35:43.876618Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.876677Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/' [INFO] [stdout] 2025-11-05T02:35:43.876707Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.876754Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2025-11-05T02:35:43.876829Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.876863Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'text_files/' [INFO] [stdout] 2025-11-05T02:35:43.876899Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:35:43.876934Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2025-11-05T02:35:43.876967Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] 2025-11-05T02:35:43.870314Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'mixed-files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_filtering::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:588:61 [INFO] [stdout] 4: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 7: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 8: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 9: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 13: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 14: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 15: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 16: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 18: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 21: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 23: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 25: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_filtering [INFO] [stdout] at ./src/../modules/file_discovery.rs:590:22 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_filtering::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:580:35 [INFO] [stdout] 27: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 30: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 31: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 32: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 33: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 34: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 35: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 36: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 38: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 39: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 40: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 41: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 42: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 43: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 44: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 45: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 46: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 47: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 48: [INFO] [stdout] 49: clone [INFO] [stdout] 2025-11-05T02:35:43.877421Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.873721Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.875664Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.877785Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/' [INFO] [stdout] 2025-11-05T02:35:43.877802Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.877808Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' [INFO] [stdout] 2025-11-05T02:35:43.877822Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2025-11-05T02:35:43.877826Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.877839Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.876183Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.877860Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'test-data/' [INFO] [stdout] 2025-11-05T02:35:43.877879Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/ [INFO] [stdout] 2025-11-05T02:35:43.877891Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2025-11-05T02:35:43.877837Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:35:43.877918Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] 2025-11-05T02:35:43.877930Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2025-11-05T02:35:43.877942Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2025-11-05T02:35:43.877967Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.877978Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' [INFO] [stdout] 2025-11-05T02:35:43.877988Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.877999Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:35:43.878012Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.878022Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' [INFO] [stdout] 2025-11-05T02:35:43.878033Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:35:43.878046Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:35:43.878107Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.878120Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' [INFO] [stdout] 2025-11-05T02:35:43.878131Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.878141Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:35:43.878153Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_file_filtering ... ok [INFO] [stdout] 2025-11-05T02:35:43.878612Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.878637Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.878677Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.878686Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'text_files/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.878700Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2025-11-05T02:35:43.878709Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.878719Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2025-11-05T02:35:43.878731Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.878835Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:35:43.878855Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_folder_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:470:60 [INFO] [stdout] 4: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 7: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 8: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 9: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 13: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 14: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 15: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 16: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 18: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 21: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 23: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 25: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_folder_discovery [INFO] [stdout] at ./src/../modules/file_discovery.rs:472:22 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_folder_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:462:37 [INFO] [stdout] 27: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 30: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 31: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 32: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 33: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 34: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 35: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 36: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 38: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 39: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 40: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 41: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 42: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 43: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 44: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 45: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 46: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 47: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 48: [INFO] [stdout] 49: clone [INFO] [stdout] 2025-11-05T02:35:43.879646Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.874823Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.881788Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/sample.txt' [INFO] [stdout] 2025-11-05T02:35:43.881811Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.881823Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:35:43.881841Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.877866Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.881858Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/sample.txt' [INFO] [stdout] 2025-11-05T02:35:43.881868Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.881880Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:35:43.881872Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.881892Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'large-dataset/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.881900Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.881908Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2025-11-05T02:35:43.881912Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/sample.txt' [INFO] [stdout] 2025-11-05T02:35:43.881918Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.881920Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.881927Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2025-11-05T02:35:43.881930Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.881938Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.881942Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:35:43.882037Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_folder_discovery ... ok [INFO] [stdout] 2025-11-05T02:35:43.882053Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'large-dataset/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_large_folder_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:551:61 [INFO] [stdout] 4: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 7: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 8: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 9: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 13: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 14: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 15: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 16: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 18: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 21: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 23: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 25: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_large_folder_discovery [INFO] [stdout] at ./src/../modules/file_discovery.rs:553:22 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_large_folder_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:543:43 [INFO] [stdout] 27: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 30: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 31: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 32: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 33: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 34: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 35: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 36: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 38: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 39: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 40: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 41: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 42: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 43: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 44: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 45: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 46: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 47: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 48: [INFO] [stdout] 49: clone [INFO] [stdout] 2025-11-05T02:35:43.882900Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_large_folder_discovery ... ok [INFO] [stdout] 2025-11-05T02:35:43.884800Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.884804Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt': dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.884828Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.884833Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.884859Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.884860Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.884868Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'text_files/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.884870Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.884881Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2025-11-05T02:35:43.884885Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:35:43.884889Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.884894Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.884898Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2025-11-05T02:35:43.884904Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.884909Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.884913Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:35:43.884994Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:35:43.886937Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/sample.txt': dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.886980Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.887008Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.887017Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/sample.txt' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.887031Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:35:43.887040Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.887048Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.887063Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:35:43.887183Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt': Failed to get metadata for text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:35:43.887219Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:35:43.887229Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:35:43.887244Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:35:43.887252Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:35:43.887260Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:35:43.887277Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:35:43.887586Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.887625Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/sample.txt' [INFO] [stdout] 2025-11-05T02:35:43.887635Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.887646Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.887664Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2025-11-05T02:35:43.887674Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2025-11-05T02:35:43.887698Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.887717Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/sample.txt' [INFO] [stdout] 2025-11-05T02:35:43.887726Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.887757Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:35:43.887782Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.887798Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'text_files/sample.txt' [INFO] [stdout] 2025-11-05T02:35:43.887806Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.887823Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:35:43.887861Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.887880Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'text_files/sample.txt' [INFO] [stdout] 2025-11-05T02:35:43.887889Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.887903Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.887922Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:35:43.887302Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 1: S3 HEAD request failed for 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:673:55 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 18: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end [INFO] [stdout] at ./src/../modules/job_processor/core.rs:710:37 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:638:57 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 35: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 36: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 41: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 42: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 44: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 45: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 47: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 49: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2025-11-05T02:35:43.888449Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.888473Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to get metadata for single file: text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 2025-11-05T02:35:43.888496Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to get metadata for single file: text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt [INFO] [stdout] 1: S3 HEAD request failed for 'text_files/0c472776-7c22-464b-93bf-3714fd229b01.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:673:55 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 18: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end [INFO] [stdout] at ./src/../modules/job_processor/core.rs:710:37 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:638:57 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 35: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 36: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 41: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 42: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 44: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 45: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 47: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 49: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2025-11-05T02:35:43.889074Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2025-11-05T02:35:43.885014Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::run::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:57:43 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:787:31 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 18: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end [INFO] [stdout] at ./src/../modules/job_processor/core.rs:787:37 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:759:52 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 35: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 36: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 41: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 42: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 44: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 45: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 47: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 49: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2025-11-05T02:35:43.890310Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.890336Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:35:43.890351Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::run::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:57:43 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:787:31 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 18: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end [INFO] [stdout] at ./src/../modules/job_processor/core.rs:787:37 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:759:52 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 35: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 36: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 41: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 42: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 44: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 45: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 47: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 49: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2025-11-05T02:35:43.890927Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_single_file_end_to_end ... ok [INFO] [stdout] 2025-11-05T02:35:43.892324Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.892363Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/' [INFO] [stdout] 2025-11-05T02:35:43.892373Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.892396Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:35:43.892419Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2025-11-05T02:35:43.892428Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2025-11-05T02:35:43.892442Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.892465Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/' [INFO] [stdout] 2025-11-05T02:35:43.892473Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.892491Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2025-11-05T02:35:43.892504Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.892524Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'text_files/' [INFO] [stdout] 2025-11-05T02:35:43.892533Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:35:43.892553Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2025-11-05T02:35:43.892569Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_folder_end_to_end ... ok [INFO] [stdout] 2025-11-05T02:35:43.895349Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.895404Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.895448Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.895457Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'test-data/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.895471Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2025-11-05T02:35:43.895480Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.895489Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2025-11-05T02:35:43.895507Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.895585Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:35:43.895604Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'test-data/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_processing_summary::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:621:60 [INFO] [stdout] 4: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 7: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 8: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 9: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 13: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 14: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 15: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 16: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 18: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 21: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 23: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 25: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_processing_summary [INFO] [stdout] at ./src/../modules/file_discovery.rs:623:22 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_processing_summary::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:614:39 [INFO] [stdout] 27: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 30: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 31: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 32: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 33: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 34: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 35: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 36: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 38: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 39: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 40: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 41: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 42: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 43: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 44: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 45: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 46: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 47: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 48: [INFO] [stdout] 49: clone [INFO] [stdout] 2025-11-05T02:35:43.896463Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_processing_summary ... ok [INFO] [stdout] 2025-11-05T02:35:43.897949Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'text_files/sample.txt': dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.897984Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.898160Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.898182Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'text_files/sample.txt' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.898195Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:35:43.898204Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.898204Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/sample.txt': dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.898225Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.898251Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.898259Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/sample.txt' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.898282Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:35:43.898291Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.898300Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.898308Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:35:43.898381Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/sample.txt': Failed to get metadata for test-data/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.898398Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:35:43.898415Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.898428Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:35:43.898436Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:35:43.898444Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:35:43.898455Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.898764Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.898784Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:35:43.898850Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'text_files/sample.txt': Failed to get metadata for text_files/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.898865Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:35:43.898877Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.898889Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:35:43.898897Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:35:43.898904Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:35:43.898915Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: text_files/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.898474Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/sample.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/sample.txt [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/sample.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_single_file_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:441:62 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_single_file_discovery [INFO] [stdout] at ./src/../modules/file_discovery.rs:443:22 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_single_file_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:433:42 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.899284Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.898928Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: text_files/sample.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for text_files/sample.txt [INFO] [stdout] 1: S3 HEAD request failed for 'text_files/sample.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1024:46 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 18: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1017:31 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:980:52 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 35: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 36: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 41: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 42: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 44: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 45: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 47: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 49: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2025-11-05T02:35:43.899715Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.899734Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to get metadata for single file: text_files/sample.txt [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_single_file_discovery ... ok [INFO] [stdout] 2025-11-05T02:35:43.900114Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/sample.txt': Failed to get metadata for test-data/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.900137Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:35:43.900145Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.900158Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:35:43.900166Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:35:43.900173Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:35:43.900184Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/sample.txt [INFO] [stdout] 2025-11-05T02:35:43.899772Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to get metadata for single file: text_files/sample.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for text_files/sample.txt [INFO] [stdout] 1: S3 HEAD request failed for 'text_files/sample.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1024:46 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 9: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 10: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 15: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 16: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 17: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 18: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 19: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 23: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1017:31 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:980:52 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 32: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 33: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 34: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 35: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 36: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 37: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 40: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 41: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 42: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 43: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 44: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 45: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 46: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 47: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 48: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 49: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 50: [INFO] [stdout] 51: clone [INFO] [stdout] 2025-11-05T02:35:43.900330Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2025-11-05T02:35:43.900425Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.900439Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'text_files/' [INFO] [stdout] 2025-11-05T02:35:43.900449Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.900460Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:35:43.900474Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2025-11-05T02:35:43.900483Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2025-11-05T02:35:43.900496Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.900506Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'text_files/' [INFO] [stdout] 2025-11-05T02:35:43.900514Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.900528Z  INFO ai_workbench_lib::modules::file_discovery: Detected folder input (ends with '/') - processing as folder [INFO] [stdout] 2025-11-05T02:35:43.900543Z  INFO ai_workbench_lib::modules::file_discovery: === process_folder DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.900554Z  INFO ai_workbench_lib::modules::file_discovery: Processing folder: 'text_files/' [INFO] [stdout] 2025-11-05T02:35:43.900563Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/text_files/ [INFO] [stdout] 2025-11-05T02:35:43.900576Z  INFO ai_workbench_lib::modules::file_discovery: Making S3 ListObjects request #1 [INFO] [stdout] 2025-11-05T02:35:43.900593Z  INFO ai_workbench_lib::modules::file_discovery: Sending ListObjectsV2 request to S3... [INFO] [stdout] 2025-11-05T02:35:43.900196Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/sample.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/sample.txt [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/sample.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.900998Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.901032Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.901042Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/data.csv' [INFO] [stdout] 2025-11-05T02:35:43.901050Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.901062Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:35:43.901075Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.901084Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/data.csv' [INFO] [stdout] 2025-11-05T02:35:43.901093Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/data.csv [INFO] [stdout] 2025-11-05T02:35:43.901105Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:35:43.901118Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.901128Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/data.csv' [INFO] [stdout] 2025-11-05T02:35:43.901137Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.901146Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/data.csv [INFO] [stdout] 2025-11-05T02:35:43.901157Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:35:43.906875Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.906933Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.906964Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.906972Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'text_files/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.906985Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2025-11-05T02:35:43.906993Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.907002Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2025-11-05T02:35:43.907012Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.907097Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:35:43.907551Z ERROR ai_workbench_lib::modules::file_discovery: S3 ListObjectsV2 request failed: dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.907575Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.907603Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.907611Z ERROR ai_workbench_lib::modules::file_discovery: 1. Folder/prefix 'text_files/' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.907624Z ERROR ai_workbench_lib::modules::file_discovery: 2. No list permissions for this bucket/prefix [INFO] [stdout] 2025-11-05T02:35:43.907633Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.907642Z ERROR ai_workbench_lib::modules::file_discovery: 4. Bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' does not exist [INFO] [stdout] 2025-11-05T02:35:43.907653Z ERROR ai_workbench_lib::modules::file_discovery: 5. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.907718Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:35:43.907116Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1024:46 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1017:31 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:980:52 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.910243Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.910273Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:35:43.915663Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/data.csv': dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.915709Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.915759Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.915769Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/data.csv' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.915782Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:35:43.915791Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.915800Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.915809Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:35:43.915894Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/data.csv': Failed to get metadata for test-data/data.csv [INFO] [stdout] 2025-11-05T02:35:43.915912Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:35:43.915919Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/data.csv [INFO] [stdout] 2025-11-05T02:35:43.915931Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:35:43.915940Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:35:43.915947Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:35:43.915958Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/data.csv [INFO] [stdout] 2025-11-05T02:35:43.915971Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/data.csv [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/data.csv [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/data.csv': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.916797Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.916832Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.916843Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/config.json' [INFO] [stdout] 2025-11-05T02:35:43.916851Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.916864Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:35:43.916878Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.916887Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/config.json' [INFO] [stdout] 2025-11-05T02:35:43.916896Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/config.json [INFO] [stdout] 2025-11-05T02:35:43.916909Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:35:43.916920Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.916930Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/config.json' [INFO] [stdout] 2025-11-05T02:35:43.916940Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.916950Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/config.json [INFO] [stdout] 2025-11-05T02:35:43.916961Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:35:43.917269Z  INFO ai_workbench_lib::modules::job_processor::core: Starting job execution for job_id: test-large-f00e91f7-1c91-465c-a936-f406d42788de [INFO] [stdout] 2025-11-05T02:35:43.917384Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.917398Z  INFO ai_workbench_lib::modules::job_processor::core: Input specification: 'large_files/large_document.txt' [INFO] [stdout] 2025-11-05T02:35:43.917408Z  INFO ai_workbench_lib::modules::job_processor::core: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.917418Z  INFO ai_workbench_lib::modules::job_processor::core: Full S3 path would be: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:35:43.917434Z  INFO ai_workbench_lib::modules::job_processor::core: FileDiscovery instance created successfully [INFO] [stdout] 2025-11-05T02:35:43.917442Z  INFO ai_workbench_lib::modules::job_processor::core: Attempting to discover files... [INFO] [stdout] 2025-11-05T02:35:43.917520Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.917533Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'large_files/large_document.txt' [INFO] [stdout] 2025-11-05T02:35:43.917542Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.917556Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:35:43.917589Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.917598Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'large_files/large_document.txt' [INFO] [stdout] 2025-11-05T02:35:43.917607Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:35:43.917619Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:35:43.917656Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.917665Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'large_files/large_document.txt' [INFO] [stdout] 2025-11-05T02:35:43.917673Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.917683Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:35:43.917702Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:35:43.910290Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1024:46 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation [INFO] [stdout] at ./src/../modules/job_processor/core.rs:1017:31 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_config_validation::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:980:52 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.919209Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_config_validation ... ok [INFO] [stdout] 2025-11-05T02:35:43.907731Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:955:42 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery [INFO] [stdout] at ./src/../modules/job_processor/core.rs:955:48 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:930:49 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.923164Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.923197Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to list objects in S3 [INFO] [stdout] 2025-11-05T02:35:43.923210Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to list objects in S3 [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] S3 ListObjectsV2 failed for folder 'text_files/': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_folder::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:157:32 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:31:45 [INFO] [stdout] 3: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:955:42 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery [INFO] [stdout] at ./src/../modules/job_processor/core.rs:955:48 [INFO] [stdout] 27: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:930:49 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.923712Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_file_discovery ... ok [INFO] [stdout] 2025-11-05T02:35:43.927056Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/config.json': dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.927105Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.927136Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.927144Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/config.json' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.927157Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:35:43.927166Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.927175Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.927184Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:35:43.927270Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/config.json': Failed to get metadata for test-data/config.json [INFO] [stdout] 2025-11-05T02:35:43.927289Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:35:43.927297Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/config.json [INFO] [stdout] 2025-11-05T02:35:43.927310Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:35:43.927318Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:35:43.927326Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:35:43.927338Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/config.json [INFO] [stdout] 2025-11-05T02:35:43.927350Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/config.json [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/config.json [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/config.json': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.928197Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.928229Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.928239Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/document.md' [INFO] [stdout] 2025-11-05T02:35:43.928247Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.928260Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:35:43.928273Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.928282Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/document.md' [INFO] [stdout] 2025-11-05T02:35:43.928291Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/document.md [INFO] [stdout] 2025-11-05T02:35:43.928303Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:35:43.928315Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.928324Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/document.md' [INFO] [stdout] 2025-11-05T02:35:43.928333Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.928343Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/document.md [INFO] [stdout] 2025-11-05T02:35:43.928354Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:35:43.933249Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'large_files/large_document.txt': dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.934522Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/document.md': dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.935850Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.935896Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.935906Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/document.md' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.935936Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:35:43.935945Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.935955Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.935963Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:35:43.935932Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.936013Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.936023Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'large_files/large_document.txt' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.936038Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:35:43.936047Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.936056Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.936065Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:35:43.936064Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/document.md': Failed to get metadata for test-data/document.md [INFO] [stdout] 2025-11-05T02:35:43.936085Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:35:43.936093Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/document.md [INFO] [stdout] 2025-11-05T02:35:43.936106Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:35:43.936114Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:35:43.936122Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:35:43.936134Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/document.md [INFO] [stdout] 2025-11-05T02:35:43.936168Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'large_files/large_document.txt': Failed to get metadata for large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:35:43.936188Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:35:43.936195Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:35:43.936207Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:35:43.936219Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:35:43.936226Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:35:43.936236Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:35:43.936148Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/document.md [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/document.md [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/document.md': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.936996Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.937031Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.937042Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/script.py' [INFO] [stdout] 2025-11-05T02:35:43.937051Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.937064Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:35:43.937078Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.937088Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/script.py' [INFO] [stdout] 2025-11-05T02:35:43.937097Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/script.py [INFO] [stdout] 2025-11-05T02:35:43.937108Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:35:43.937120Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.937129Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/script.py' [INFO] [stdout] 2025-11-05T02:35:43.937138Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.937148Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/script.py [INFO] [stdout] 2025-11-05T02:35:43.937160Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:35:43.936249Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: large_files/large_document.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for large_files/large_document.txt [INFO] [stdout] 1: S3 HEAD request failed for 'large_files/large_document.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::JobProcessor::run::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:57:43 [INFO] [stdout] 6: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:882:31 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 9: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 10: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 11: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 15: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 16: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 18: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 19: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 20: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 24: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 25: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 27: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file [INFO] [stdout] at ./src/../modules/job_processor/core.rs:882:37 [INFO] [stdout] 29: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:854:50 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 32: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 33: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 34: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 35: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 36: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 37: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 38: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 40: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 41: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 42: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 43: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 44: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 45: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 46: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 47: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 48: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 49: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 50: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 51: [INFO] [stdout] 52: clone [INFO] [stdout] 2025-11-05T02:35:43.937712Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.940974Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/script.py': dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.941064Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.941116Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.941145Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/script.py' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.941177Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:35:43.941204Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.941230Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.941257Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:35:43.941793Z ERROR ai_workbench_lib::modules::job_processor::core: File discovery failed with error: Failed to get metadata for single file: large_files/large_document.txt [INFO] [stdout] 2025-11-05T02:35:43.943113Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/script.py': Failed to get metadata for test-data/script.py [INFO] [stdout] 2025-11-05T02:35:43.943163Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:35:43.943201Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/script.py [INFO] [stdout] 2025-11-05T02:35:43.943233Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:35:43.943258Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:35:43.943360Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:35:43.943497Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/script.py [INFO] [stdout] 2025-11-05T02:35:43.943077Z ERROR ai_workbench_lib::modules::job_processor::core: Error context: Failed to get metadata for single file: large_files/large_document.txt [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for large_files/large_document.txt [INFO] [stdout] 1: S3 HEAD request failed for 'large_files/large_document.txt': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::job_processor::core::JobProcessor::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:102:83 [INFO] [stdout] 5: ai_workbench_lib::modules::job_processor::core::JobProcessor::run::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:57:43 [INFO] [stdout] 6: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:882:31 [INFO] [stdout] 7: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 8: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 9: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 10: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 11: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 14: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 15: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 16: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 17: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 18: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 19: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 20: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 22: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 24: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 25: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 26: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 27: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 28: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file [INFO] [stdout] at ./src/../modules/job_processor/core.rs:882:37 [INFO] [stdout] 29: ai_workbench_lib::modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file::{{closure}} [INFO] [stdout] at ./src/../modules/job_processor/core.rs:854:50 [INFO] [stdout] 30: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 31: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 32: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 33: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 34: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 35: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 36: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 37: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 38: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 39: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 40: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 41: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 42: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 43: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 44: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 45: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 46: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 47: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 48: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 49: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 50: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 51: [INFO] [stdout] 52: clone [INFO] [stdout] 2025-11-05T02:35:43.943767Z  INFO ai_workbench_lib::modules::job_processor::core: === FILE DISCOVERY DEBUG END (FAILED) === [INFO] [stdout] 2025-11-05T02:35:43.943585Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/script.py [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/script.py [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/script.py': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.945612Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.945787Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.945935Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/code.rs' [INFO] [stdout] test modules::job_processor::core::job_processor_tests::test_job_processor_with_large_file ... 2025-11-05T02:35:43.946040Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] ok [INFO] [stdout] 2025-11-05T02:35:43.946063Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:35:43.946078Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.946089Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/code.rs' [INFO] [stdout] 2025-11-05T02:35:43.946098Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/code.rs [INFO] [stdout] 2025-11-05T02:35:43.946116Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:35:43.946128Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.946138Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/code.rs' [INFO] [stdout] 2025-11-05T02:35:43.946147Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.946157Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/code.rs [INFO] [stdout] 2025-11-05T02:35:43.946167Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:35:43.949796Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/code.rs': dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.949844Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.949874Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.949884Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/code.rs' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.949899Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:35:43.949908Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.949918Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.949927Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:35:43.950001Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/code.rs': Failed to get metadata for test-data/code.rs [INFO] [stdout] 2025-11-05T02:35:43.950019Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:35:43.950027Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/code.rs [INFO] [stdout] 2025-11-05T02:35:43.950039Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:35:43.950047Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:35:43.950055Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:35:43.950066Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/code.rs [INFO] [stdout] 2025-11-05T02:35:43.950078Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/code.rs [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/code.rs [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/code.rs': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.950934Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.950966Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.950977Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/styles.css' [INFO] [stdout] 2025-11-05T02:35:43.950984Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.950997Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:35:43.951010Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.951020Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/styles.css' [INFO] [stdout] 2025-11-05T02:35:43.951028Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/styles.css [INFO] [stdout] 2025-11-05T02:35:43.951040Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:35:43.951051Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.951060Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/styles.css' [INFO] [stdout] 2025-11-05T02:35:43.951069Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.951078Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/styles.css [INFO] [stdout] 2025-11-05T02:35:43.951094Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:35:43.954143Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/styles.css': dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.954192Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.954219Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.954227Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/styles.css' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.954240Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:35:43.954248Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.954264Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.954272Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:35:43.954345Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/styles.css': Failed to get metadata for test-data/styles.css [INFO] [stdout] 2025-11-05T02:35:43.954362Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:35:43.954369Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/styles.css [INFO] [stdout] 2025-11-05T02:35:43.954387Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:35:43.954395Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:35:43.954417Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:35:43.954430Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/styles.css [INFO] [stdout] test modules::file_splitter::splitter::integration_tests::test_performance_characteristics ... FAILED [INFO] [stdout] 2025-11-05T02:35:43.954446Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/styles.css [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/styles.css [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/styles.css': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.958316Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] 2025-11-05T02:35:43.958381Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.958391Z  INFO ai_workbench_lib::modules::file_discovery: Input spec: 'test-data/markup.html' [INFO] [stdout] 2025-11-05T02:35:43.958399Z  INFO ai_workbench_lib::modules::file_discovery: Workspace bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.958418Z  INFO ai_workbench_lib::modules::file_discovery: Detected single file input (no trailing '/') - processing as single file [INFO] [stdout] 2025-11-05T02:35:43.958433Z  INFO ai_workbench_lib::modules::file_discovery: === process_single_file DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.958445Z  INFO ai_workbench_lib::modules::file_discovery: Processing single file: 'test-data/markup.html' [INFO] [stdout] 2025-11-05T02:35:43.958454Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/markup.html [INFO] [stdout] 2025-11-05T02:35:43.958465Z  INFO ai_workbench_lib::modules::file_discovery: Attempting to get file metadata... [INFO] [stdout] 2025-11-05T02:35:43.958477Z  INFO ai_workbench_lib::modules::file_discovery: === get_file_metadata DEBUG START === [INFO] [stdout] 2025-11-05T02:35:43.958487Z  INFO ai_workbench_lib::modules::file_discovery: Getting metadata for file: 'test-data/markup.html' [INFO] [stdout] 2025-11-05T02:35:43.958496Z  INFO ai_workbench_lib::modules::file_discovery: Bucket: 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.958505Z  INFO ai_workbench_lib::modules::file_discovery: Full S3 path: s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/markup.html [INFO] [stdout] 2025-11-05T02:35:43.958516Z  INFO ai_workbench_lib::modules::file_discovery: Sending HEAD request to S3... [INFO] [stdout] 2025-11-05T02:35:43.961964Z ERROR ai_workbench_lib::modules::file_discovery: S3 HEAD request failed for 'test-data/markup.html': dispatch failure [INFO] [stdout] 2025-11-05T02:35:43.962007Z ERROR ai_workbench_lib::modules::file_discovery: Error details: DispatchFailure(DispatchFailure { source: ConnectorError { kind: Other(None), source: CredentialsNotLoaded(CredentialsNotLoaded { source: Some("no providers in chain provided credentials") }), connection: Unknown } }) [INFO] [stdout] 2025-11-05T02:35:43.962034Z ERROR ai_workbench_lib::modules::file_discovery: Possible causes: [INFO] [stdout] 2025-11-05T02:35:43.962041Z ERROR ai_workbench_lib::modules::file_discovery: 1. File 'test-data/markup.html' does not exist in bucket 'ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2' [INFO] [stdout] 2025-11-05T02:35:43.962053Z ERROR ai_workbench_lib::modules::file_discovery: 2. No read permissions for this file/bucket [INFO] [stdout] 2025-11-05T02:35:43.962061Z ERROR ai_workbench_lib::modules::file_discovery: 3. AWS credentials are invalid or expired [INFO] [stdout] 2025-11-05T02:35:43.962069Z ERROR ai_workbench_lib::modules::file_discovery: 4. Network connectivity issues [INFO] [stdout] 2025-11-05T02:35:43.962076Z ERROR ai_workbench_lib::modules::file_discovery: 5. S3 service is temporarily unavailable [INFO] [stdout] 2025-11-05T02:35:43.962155Z ERROR ai_workbench_lib::modules::file_discovery: Failed to get metadata for file 'test-data/markup.html': Failed to get metadata for test-data/markup.html [INFO] [stdout] 2025-11-05T02:35:43.962175Z ERROR ai_workbench_lib::modules::file_discovery: This usually means: [INFO] [stdout] 2025-11-05T02:35:43.962182Z ERROR ai_workbench_lib::modules::file_discovery: 1. File does not exist at s3://ai-workbench-6c9c43db-7fe6-42f1-8b11-8f82323f83f0-eu-west-2/test-data/markup.html [INFO] [stdout] 2025-11-05T02:35:43.962194Z ERROR ai_workbench_lib::modules::file_discovery: 2. No permissions to access the file [INFO] [stdout] 2025-11-05T02:35:43.962201Z ERROR ai_workbench_lib::modules::file_discovery: 3. S3 service is unavailable [INFO] [stdout] 2025-11-05T02:35:43.962208Z ERROR ai_workbench_lib::modules::file_discovery: 4. Incorrect bucket name or file path [INFO] [stdout] 2025-11-05T02:35:43.962219Z ERROR ai_workbench_lib::modules::file_discovery: discover_files failed: Failed to get metadata for single file: test-data/markup.html [INFO] [stdout] 2025-11-05T02:35:43.962230Z ERROR ai_workbench_lib::modules::file_discovery: Error details: Failed to get metadata for single file: test-data/markup.html [INFO] [stdout] [INFO] [stdout] Caused by: [INFO] [stdout] 0: Failed to get metadata for test-data/markup.html [INFO] [stdout] 1: S3 HEAD request failed for 'test-data/markup.html': dispatch failure [INFO] [stdout] [INFO] [stdout] Stack backtrace: [INFO] [stdout] 0: anyhow::error::::msg [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.99/src/backtrace.rs:27:14 [INFO] [stdout] 1: ai_workbench_lib::modules::file_discovery::FileDiscovery::get_file_metadata::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:267:28 [INFO] [stdout] 2: ai_workbench_lib::modules::file_discovery::FileDiscovery::process_single_file::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:59:64 [INFO] [stdout] 3: ai_workbench_lib::modules::file_discovery::FileDiscovery::discover_files::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:35:50 [INFO] [stdout] 4: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:516:61 [INFO] [stdout] 5: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 6: as core::future::future::Future>::poll [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/future/future.rs:133:9 [INFO] [stdout] 7: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:70 [INFO] [stdout] 8: tokio::task::coop::with_budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:167:5 [INFO] [stdout] 9: tokio::task::coop::budget [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/task/coop/mod.rs:133:5 [INFO] [stdout] 10: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:742:25 [INFO] [stdout] 11: tokio::runtime::scheduler::current_thread::Context::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:432:19 [INFO] [stdout] 12: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:741:44 [INFO] [stdout] 13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:68 [INFO] [stdout] 14: tokio::runtime::context::scoped::Scoped::set [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/scoped.rs:40:9 [INFO] [stdout] 15: tokio::runtime::context::set_scheduler::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:38 [INFO] [stdout] 16: std::thread::local::LocalKey::try_with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:508:12 [INFO] [stdout] 17: std::thread::local::LocalKey::with [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/local.rs:472:20 [INFO] [stdout] 18: tokio::runtime::context::set_scheduler [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context.rs:176:17 [INFO] [stdout] 19: tokio::runtime::scheduler::current_thread::CoreGuard::enter [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:829:27 [INFO] [stdout] 20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:729:24 [INFO] [stdout] 21: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}} [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:200:33 [INFO] [stdout] 22: tokio::runtime::context::runtime::enter_runtime [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/context/runtime.rs:65:16 [INFO] [stdout] 23: tokio::runtime::scheduler::current_thread::CurrentThread::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/scheduler/current_thread/mod.rs:188:9 [INFO] [stdout] 24: tokio::runtime::runtime::Runtime::block_on_inner [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:356:52 [INFO] [stdout] 25: tokio::runtime::runtime::Runtime::block_on [INFO] [stdout] at /opt/rustwide/cargo-home/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.47.1/src/runtime/runtime.rs:330:18 [INFO] [stdout] 26: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection [INFO] [stdout] at ./src/../modules/file_discovery.rs:515:36 [INFO] [stdout] 27: ai_workbench_lib::modules::file_discovery::file_discovery_tests::test_file_type_detection::{{closure}} [INFO] [stdout] at ./src/../modules/file_discovery.rs:498:40 [INFO] [stdout] 28: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 29: core::ops::function::FnOnce::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 30: test::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 31: test::run_test_in_process::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 32: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 33: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 34: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 35: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 36: test::run_test_in_process [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 37: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 38: test::run_test::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 39: std::sys::backtrace::__rust_begin_short_backtrace [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 41: as core::ops::function::FnOnce<()>>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 42: std::panicking::catch_unwind::do_call [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 43: std::panicking::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 44: std::panic::catch_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 45: std::thread::Builder::spawn_unchecked_::{{closure}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 46: core::ops::function::FnOnce::call_once{{vtable.shim}} [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 47: as core::ops::function::FnOnce>::call_once [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 48: std::sys::thread::unix::Thread::new::thread_start [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 49: [INFO] [stdout] 50: clone [INFO] [stdout] 2025-11-05T02:35:43.963032Z  INFO ai_workbench_lib::modules::file_discovery: === FileDiscovery::discover_files DEBUG END === [INFO] [stdout] test modules::file_discovery::file_discovery_tests::test_file_type_detection ... ok [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter::integration_tests::test_mixed_file_types stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter::integration_tests::test_mixed_file_types' (35) panicked at src/../modules/file_splitter/splitter.rs:245:13: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: Text [INFO] [stdout] right: Xml [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62f20d077c42 - std::backtrace_rs::backtrace::libunwind::trace::h16acf28769fcd909 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62f20d077c42 - std::backtrace_rs::backtrace::trace_unsynchronized::hb5541aa5d4f92513 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62f20d077c42 - std::sys::backtrace::_print_fmt::h8ac8268cb667a306 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x62f20d077c42 - ::fmt::h47627beda0767f4c [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x62f20d08b55f - core::fmt::rt::Argument::fmt::h686a01ad57c1c92a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x62f20d08b55f - core::fmt::write::h8a63064c67176fd8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/mod.rs:1469:25 [INFO] [stdout] 6: 0x62f20d041df3 - std::io::default_write_fmt::hd4797e693c0ec256 [INFO] [stderr] error: test failed, to rerun pass `--lib` [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62f20d041df3 - std::io::Write::write_fmt::h791e240ad2720c99 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x62f20d04ee22 - std::sys::backtrace::BacktraceLock::print::h4634d27c0afbaebb [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x62f20d05491f - std::panicking::default_hook::{{closure}}::h23e0299d4738b766 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x62f20d0547b1 - std::panicking::default_hook::hfeec6294e8751b62 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x62f20bde5bae - as core::ops::function::Fn>::call::h3f5a16438cee2b4e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 12: 0x62f20bde5bae - test::test_main_with_exit_callback::{{closure}}::h9214498cef9aafc5 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62f20d054fdf - as core::ops::function::Fn>::call::hdb18da3e68df7464 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 14: 0x62f20d054fdf - std::panicking::panic_with_hook::hd57cd463d793ed79 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x62f20d054d8a - std::panicking::panic_handler::{{closure}}::hddd6216c3fddee3b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:707:13 [INFO] [stdout] 16: 0x62f20d04ef59 - std::sys::backtrace::__rust_end_short_backtrace::h5e7a56f283378525 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x62f20d03589d - __rustc[256c90bb7e548b45]::rust_begin_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x62f20d0963f0 - core::panicking::panic_fmt::hf19c198748ef6a95 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62f20d0961a3 - core::panicking::assert_failed_inner::h709867d03648810d [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:444:17 [INFO] [stdout] 20: 0x62f20bd451f2 - core::panicking::assert_failed::hdc5c84637a6b97a2 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:399:5 [INFO] [stdout] 21: 0x62f20bd60c9c - ai_workbench_lib::modules::file_splitter::splitter::integration_tests::test_mixed_file_types::hee2d4b6ef2637273 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:245:13 [INFO] [stdout] 22: 0x62f20bd60d77 - ai_workbench_lib::modules::file_splitter::splitter::integration_tests::test_mixed_file_types::{{closure}}::hf1214c5842ce2027 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:231:31 [INFO] [stdout] 23: 0x62f20bd29116 - core::ops::function::FnOnce::call_once::h5ed4c0ae7350ee02 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62f20bde59fb - core::ops::function::FnOnce::call_once::h43f43c5185cf39c2 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x62f20bde59fb - test::__rust_begin_short_backtrace::hc4fa634eb2764bdc [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 26: 0x62f20bdfa018 - test::run_test_in_process::{{closure}}::h9f4b799a5211b72a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 27: 0x62f20bdfa018 - as core::ops::function::FnOnce<()>>::call_once::ha572bf7dc5230a8f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 28: 0x62f20bdfa018 - std::panicking::catch_unwind::do_call::hfd6ff51f25dd177b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 29: 0x62f20bdfa018 - std::panicking::catch_unwind::h8609b446221def5f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 30: 0x62f20bdfa018 - std::panic::catch_unwind::h362f73cc48531e26 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 31: 0x62f20bdfa018 - test::run_test_in_process::hf52fd21d696aae67 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 32: 0x62f20bdfa018 - test::run_test::{{closure}}::h7e780b8354e6d007 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 33: 0x62f20bdd2de4 - test::run_test::{{closure}}::hdc2695f2276aad3e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 34: 0x62f20bdd2de4 - std::sys::backtrace::__rust_begin_short_backtrace::h2555b2113ed90fc9 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 35: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::h99800d480987dfb6 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 36: 0x62f20bdd671a - as core::ops::function::FnOnce<()>>::call_once::hbbb69b5294b2221b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 37: 0x62f20bdd671a - std::panicking::catch_unwind::do_call::h495689350c27df20 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 38: 0x62f20bdd671a - std::panicking::catch_unwind::hf22437c51d1b3740 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 39: 0x62f20bdd671a - std::panic::catch_unwind::habda1a57de8231f8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 40: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::h5833d8dda8c7e80b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 41: 0x62f20bdd671a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha2910283a5058402 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 42: 0x62f20d04a16f - as core::ops::function::FnOnce>::call_once::he4d85cb537521972 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 43: 0x62f20d04a16f - std::sys::thread::unix::Thread::new::thread_start::hbaf586a9e50a98cd [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 44: 0x78fade4b5aa4 - [INFO] [stdout] 45: 0x78fade542a64 - clone [INFO] [stdout] 46: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter::tests::test_binary_file_rejection stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter::tests::test_binary_file_rejection' (37) panicked at src/../modules/file_splitter/splitter.rs:137:9: [INFO] [stdout] assertion failed: result.is_err() [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62f20d077c42 - std::backtrace_rs::backtrace::libunwind::trace::h16acf28769fcd909 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62f20d077c42 - std::backtrace_rs::backtrace::trace_unsynchronized::hb5541aa5d4f92513 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62f20d077c42 - std::sys::backtrace::_print_fmt::h8ac8268cb667a306 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x62f20d077c42 - ::fmt::h47627beda0767f4c [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x62f20d08b55f - core::fmt::rt::Argument::fmt::h686a01ad57c1c92a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x62f20d08b55f - core::fmt::write::h8a63064c67176fd8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/mod.rs:1469:25 [INFO] [stdout] 6: 0x62f20d041df3 - std::io::default_write_fmt::hd4797e693c0ec256 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62f20d041df3 - std::io::Write::write_fmt::h791e240ad2720c99 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x62f20d04ee22 - std::sys::backtrace::BacktraceLock::print::h4634d27c0afbaebb [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x62f20d05491f - std::panicking::default_hook::{{closure}}::h23e0299d4738b766 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x62f20d0547b1 - std::panicking::default_hook::hfeec6294e8751b62 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x62f20bde5bae - as core::ops::function::Fn>::call::h3f5a16438cee2b4e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 12: 0x62f20bde5bae - test::test_main_with_exit_callback::{{closure}}::h9214498cef9aafc5 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62f20d054fdf - as core::ops::function::Fn>::call::hdb18da3e68df7464 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 14: 0x62f20d054fdf - std::panicking::panic_with_hook::hd57cd463d793ed79 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x62f20d054d56 - std::panicking::panic_handler::{{closure}}::hddd6216c3fddee3b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:700:13 [INFO] [stdout] 16: 0x62f20d04ef59 - std::sys::backtrace::__rust_end_short_backtrace::h5e7a56f283378525 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x62f20d03589d - __rustc[256c90bb7e548b45]::rust_begin_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x62f20d0963f0 - core::panicking::panic_fmt::hf19c198748ef6a95 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62f20d0963cc - core::panicking::panic::h5387b4c27e81d0dd [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:150:5 [INFO] [stdout] 20: 0x62f20bd630c0 - ai_workbench_lib::modules::file_splitter::splitter::tests::test_binary_file_rejection::ha3971d529099df5c [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:137:9 [INFO] [stdout] 21: 0x62f20bd63107 - ai_workbench_lib::modules::file_splitter::splitter::tests::test_binary_file_rejection::{{closure}}::h97753eafea6f4047 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:130:36 [INFO] [stdout] 22: 0x62f20bd28e46 - core::ops::function::FnOnce::call_once::h3a89582a1f8fab58 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 23: 0x62f20bde59fb - core::ops::function::FnOnce::call_once::h43f43c5185cf39c2 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62f20bde59fb - test::__rust_begin_short_backtrace::hc4fa634eb2764bdc [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 25: 0x62f20bdfa018 - test::run_test_in_process::{{closure}}::h9f4b799a5211b72a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 26: 0x62f20bdfa018 - as core::ops::function::FnOnce<()>>::call_once::ha572bf7dc5230a8f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 27: 0x62f20bdfa018 - std::panicking::catch_unwind::do_call::hfd6ff51f25dd177b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 28: 0x62f20bdfa018 - std::panicking::catch_unwind::h8609b446221def5f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 29: 0x62f20bdfa018 - std::panic::catch_unwind::h362f73cc48531e26 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 30: 0x62f20bdfa018 - test::run_test_in_process::hf52fd21d696aae67 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 31: 0x62f20bdfa018 - test::run_test::{{closure}}::h7e780b8354e6d007 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 32: 0x62f20bdd2de4 - test::run_test::{{closure}}::hdc2695f2276aad3e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 33: 0x62f20bdd2de4 - std::sys::backtrace::__rust_begin_short_backtrace::h2555b2113ed90fc9 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 34: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::h99800d480987dfb6 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 35: 0x62f20bdd671a - as core::ops::function::FnOnce<()>>::call_once::hbbb69b5294b2221b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 36: 0x62f20bdd671a - std::panicking::catch_unwind::do_call::h495689350c27df20 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 37: 0x62f20bdd671a - std::panicking::catch_unwind::hf22437c51d1b3740 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 38: 0x62f20bdd671a - std::panic::catch_unwind::habda1a57de8231f8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 39: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::h5833d8dda8c7e80b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 40: 0x62f20bdd671a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha2910283a5058402 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 41: 0x62f20d04a16f - as core::ops::function::FnOnce>::call_once::he4d85cb537521972 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 42: 0x62f20d04a16f - std::sys::thread::unix::Thread::new::thread_start::hbaf586a9e50a98cd [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 43: 0x78fade4b5aa4 - [INFO] [stdout] 44: 0x78fade542a64 - clone [INFO] [stdout] 45: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection' (49) panicked at src/../modules/file_splitter/splitter_types/json_splitter.rs:400:9: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: JsonLines [INFO] [stdout] right: Invalid [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62f20d077c42 - std::backtrace_rs::backtrace::libunwind::trace::h16acf28769fcd909 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62f20d077c42 - std::backtrace_rs::backtrace::trace_unsynchronized::hb5541aa5d4f92513 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62f20d077c42 - std::sys::backtrace::_print_fmt::h8ac8268cb667a306 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x62f20d077c42 - ::fmt::h47627beda0767f4c [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x62f20d08b55f - core::fmt::rt::Argument::fmt::h686a01ad57c1c92a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x62f20d08b55f - core::fmt::write::h8a63064c67176fd8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/mod.rs:1469:25 [INFO] [stdout] 6: 0x62f20d041df3 - std::io::default_write_fmt::hd4797e693c0ec256 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62f20d041df3 - std::io::Write::write_fmt::h791e240ad2720c99 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x62f20d04ee22 - std::sys::backtrace::BacktraceLock::print::h4634d27c0afbaebb [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x62f20d05491f - std::panicking::default_hook::{{closure}}::h23e0299d4738b766 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x62f20d0547b1 - std::panicking::default_hook::hfeec6294e8751b62 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x62f20bde5bae - as core::ops::function::Fn>::call::h3f5a16438cee2b4e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 12: 0x62f20bde5bae - test::test_main_with_exit_callback::{{closure}}::h9214498cef9aafc5 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62f20d054fdf - as core::ops::function::Fn>::call::hdb18da3e68df7464 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 14: 0x62f20d054fdf - std::panicking::panic_with_hook::hd57cd463d793ed79 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x62f20d054d8a - std::panicking::panic_handler::{{closure}}::hddd6216c3fddee3b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:707:13 [INFO] [stdout] 16: 0x62f20d04ef59 - std::sys::backtrace::__rust_end_short_backtrace::h5e7a56f283378525 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x62f20d03589d - __rustc[256c90bb7e548b45]::rust_begin_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x62f20d0963f0 - core::panicking::panic_fmt::hf19c198748ef6a95 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62f20d0961a3 - core::panicking::assert_failed_inner::h709867d03648810d [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:444:17 [INFO] [stdout] 20: 0x62f20bd45174 - core::panicking::assert_failed::hb8855a1a1c65396b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:399:5 [INFO] [stdout] 21: 0x62f20bd5ed89 - ai_workbench_lib::modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection::hc786119d60659c2b [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:400:9 [INFO] [stdout] 22: 0x62f20bd5edb7 - ai_workbench_lib::modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection::{{closure}}::h51c04af5c494dbdb [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:395:37 [INFO] [stdout] 23: 0x62f20bd298d6 - core::ops::function::FnOnce::call_once::hccf0de0297c1aeb8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62f20bde59fb - core::ops::function::FnOnce::call_once::h43f43c5185cf39c2 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x62f20bde59fb - test::__rust_begin_short_backtrace::hc4fa634eb2764bdc [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 26: 0x62f20bdfa018 - test::run_test_in_process::{{closure}}::h9f4b799a5211b72a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 27: 0x62f20bdfa018 - as core::ops::function::FnOnce<()>>::call_once::ha572bf7dc5230a8f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 28: 0x62f20bdfa018 - std::panicking::catch_unwind::do_call::hfd6ff51f25dd177b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 29: 0x62f20bdfa018 - std::panicking::catch_unwind::h8609b446221def5f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 30: 0x62f20bdfa018 - std::panic::catch_unwind::h362f73cc48531e26 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 31: 0x62f20bdfa018 - test::run_test_in_process::hf52fd21d696aae67 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 32: 0x62f20bdfa018 - test::run_test::{{closure}}::h7e780b8354e6d007 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 33: 0x62f20bdd2de4 - test::run_test::{{closure}}::hdc2695f2276aad3e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 34: 0x62f20bdd2de4 - std::sys::backtrace::__rust_begin_short_backtrace::h2555b2113ed90fc9 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 35: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::h99800d480987dfb6 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 36: 0x62f20bdd671a - as core::ops::function::FnOnce<()>>::call_once::hbbb69b5294b2221b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 37: 0x62f20bdd671a - std::panicking::catch_unwind::do_call::h495689350c27df20 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 38: 0x62f20bdd671a - std::panicking::catch_unwind::hf22437c51d1b3740 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 39: 0x62f20bdd671a - std::panic::catch_unwind::habda1a57de8231f8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 40: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::h5833d8dda8c7e80b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 41: 0x62f20bdd671a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha2910283a5058402 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 42: 0x62f20d04a16f - as core::ops::function::FnOnce>::call_once::he4d85cb537521972 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 43: 0x62f20d04a16f - std::sys::thread::unix::Thread::new::thread_start::hbaf586a9e50a98cd [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 44: 0x78fade4b5aa4 - [INFO] [stdout] 45: 0x78fade542a64 - clone [INFO] [stdout] 46: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection' (53) panicked at src/../modules/file_splitter/splitter_types/json_splitter.rs:354:9: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: JsonLines [INFO] [stdout] right: Standard [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62f20d077c42 - std::backtrace_rs::backtrace::libunwind::trace::h16acf28769fcd909 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62f20d077c42 - std::backtrace_rs::backtrace::trace_unsynchronized::hb5541aa5d4f92513 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62f20d077c42 - std::sys::backtrace::_print_fmt::h8ac8268cb667a306 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x62f20d077c42 - ::fmt::h47627beda0767f4c [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x62f20d08b55f - core::fmt::rt::Argument::fmt::h686a01ad57c1c92a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x62f20d08b55f - core::fmt::write::h8a63064c67176fd8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/mod.rs:1469:25 [INFO] [stdout] 6: 0x62f20d041df3 - std::io::default_write_fmt::hd4797e693c0ec256 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62f20d041df3 - std::io::Write::write_fmt::h791e240ad2720c99 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x62f20d04ee22 - std::sys::backtrace::BacktraceLock::print::h4634d27c0afbaebb [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x62f20d05491f - std::panicking::default_hook::{{closure}}::h23e0299d4738b766 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x62f20d0547b1 - std::panicking::default_hook::hfeec6294e8751b62 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x62f20bde5bae - as core::ops::function::Fn>::call::h3f5a16438cee2b4e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 12: 0x62f20bde5bae - test::test_main_with_exit_callback::{{closure}}::h9214498cef9aafc5 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62f20d054fdf - as core::ops::function::Fn>::call::hdb18da3e68df7464 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 14: 0x62f20d054fdf - std::panicking::panic_with_hook::hd57cd463d793ed79 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x62f20d054d8a - std::panicking::panic_handler::{{closure}}::hddd6216c3fddee3b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:707:13 [INFO] [stdout] 16: 0x62f20d04ef59 - std::sys::backtrace::__rust_end_short_backtrace::h5e7a56f283378525 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x62f20d03589d - __rustc[256c90bb7e548b45]::rust_begin_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x62f20d0963f0 - core::panicking::panic_fmt::hf19c198748ef6a95 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62f20d0961a3 - core::panicking::assert_failed_inner::h709867d03648810d [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:444:17 [INFO] [stdout] 20: 0x62f20bd45174 - core::panicking::assert_failed::hb8855a1a1c65396b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:399:5 [INFO] [stdout] 21: 0x62f20bd5ef09 - ai_workbench_lib::modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection::hf7c820e51df7370a [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:354:9 [INFO] [stdout] 22: 0x62f20bd5ef37 - ai_workbench_lib::modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection::{{closure}}::hc27789dce7043da3 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:348:38 [INFO] [stdout] 23: 0x62f20bd28f16 - core::ops::function::FnOnce::call_once::h48f291f24a898a8f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62f20bde59fb - core::ops::function::FnOnce::call_once::h43f43c5185cf39c2 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x62f20bde59fb - test::__rust_begin_short_backtrace::hc4fa634eb2764bdc [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 26: 0x62f20bdfa018 - test::run_test_in_process::{{closure}}::h9f4b799a5211b72a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 27: 0x62f20bdfa018 - as core::ops::function::FnOnce<()>>::call_once::ha572bf7dc5230a8f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 28: 0x62f20bdfa018 - std::panicking::catch_unwind::do_call::hfd6ff51f25dd177b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 29: 0x62f20bdfa018 - std::panicking::catch_unwind::h8609b446221def5f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 30: 0x62f20bdfa018 - std::panic::catch_unwind::h362f73cc48531e26 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 31: 0x62f20bdfa018 - test::run_test_in_process::hf52fd21d696aae67 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 32: 0x62f20bdfa018 - test::run_test::{{closure}}::h7e780b8354e6d007 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 33: 0x62f20bdd2de4 - test::run_test::{{closure}}::hdc2695f2276aad3e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 34: 0x62f20bdd2de4 - std::sys::backtrace::__rust_begin_short_backtrace::h2555b2113ed90fc9 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 35: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::h99800d480987dfb6 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 36: 0x62f20bdd671a - as core::ops::function::FnOnce<()>>::call_once::hbbb69b5294b2221b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 37: 0x62f20bdd671a - std::panicking::catch_unwind::do_call::h495689350c27df20 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 38: 0x62f20bdd671a - std::panicking::catch_unwind::hf22437c51d1b3740 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 39: 0x62f20bdd671a - std::panic::catch_unwind::habda1a57de8231f8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 40: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::h5833d8dda8c7e80b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 41: 0x62f20bdd671a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha2910283a5058402 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 42: 0x62f20d04a16f - as core::ops::function::FnOnce>::call_once::he4d85cb537521972 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 43: 0x62f20d04a16f - std::sys::thread::unix::Thread::new::thread_start::hbaf586a9e50a98cd [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 44: 0x78fade4b5aa4 - [INFO] [stdout] 45: 0x78fade542a64 - clone [INFO] [stdout] 46: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection' (50) panicked at src/../modules/file_splitter/splitter_types/json_splitter.rs:364:9: [INFO] [stdout] assertion `left == right` failed [INFO] [stdout] left: JsonLines [INFO] [stdout] right: Standard [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62f20d077c42 - std::backtrace_rs::backtrace::libunwind::trace::h16acf28769fcd909 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62f20d077c42 - std::backtrace_rs::backtrace::trace_unsynchronized::hb5541aa5d4f92513 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62f20d077c42 - std::sys::backtrace::_print_fmt::h8ac8268cb667a306 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x62f20d077c42 - ::fmt::h47627beda0767f4c [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x62f20d08b55f - core::fmt::rt::Argument::fmt::h686a01ad57c1c92a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x62f20d08b55f - core::fmt::write::h8a63064c67176fd8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/mod.rs:1469:25 [INFO] [stdout] 6: 0x62f20d041df3 - std::io::default_write_fmt::hd4797e693c0ec256 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62f20d041df3 - std::io::Write::write_fmt::h791e240ad2720c99 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x62f20d04ee22 - std::sys::backtrace::BacktraceLock::print::h4634d27c0afbaebb [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x62f20d05491f - std::panicking::default_hook::{{closure}}::h23e0299d4738b766 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x62f20d0547b1 - std::panicking::default_hook::hfeec6294e8751b62 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x62f20bde5bae - as core::ops::function::Fn>::call::h3f5a16438cee2b4e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 12: 0x62f20bde5bae - test::test_main_with_exit_callback::{{closure}}::h9214498cef9aafc5 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62f20d054fdf - as core::ops::function::Fn>::call::hdb18da3e68df7464 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 14: 0x62f20d054fdf - std::panicking::panic_with_hook::hd57cd463d793ed79 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x62f20d054d8a - std::panicking::panic_handler::{{closure}}::hddd6216c3fddee3b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:707:13 [INFO] [stdout] 16: 0x62f20d04ef59 - std::sys::backtrace::__rust_end_short_backtrace::h5e7a56f283378525 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x62f20d03589d - __rustc[256c90bb7e548b45]::rust_begin_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x62f20d0963f0 - core::panicking::panic_fmt::hf19c198748ef6a95 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62f20d0961a3 - core::panicking::assert_failed_inner::h709867d03648810d [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:444:17 [INFO] [stdout] 20: 0x62f20bd45174 - core::panicking::assert_failed::hb8855a1a1c65396b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:399:5 [INFO] [stdout] 21: 0x62f20bd5e3b9 - ai_workbench_lib::modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection::h09155efc8960967f [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:364:9 [INFO] [stdout] 22: 0x62f20bd5e3e7 - ai_workbench_lib::modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection::{{closure}}::h22140ccc46481afe [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter_types/json_splitter.rs:358:35 [INFO] [stdout] 23: 0x62f20bd29476 - core::ops::function::FnOnce::call_once::h84756867d230bff0 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62f20bde59fb - core::ops::function::FnOnce::call_once::h43f43c5185cf39c2 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 25: 0x62f20bde59fb - test::__rust_begin_short_backtrace::hc4fa634eb2764bdc [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 26: 0x62f20bdfa018 - test::run_test_in_process::{{closure}}::h9f4b799a5211b72a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 27: 0x62f20bdfa018 - as core::ops::function::FnOnce<()>>::call_once::ha572bf7dc5230a8f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 28: 0x62f20bdfa018 - std::panicking::catch_unwind::do_call::hfd6ff51f25dd177b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 29: 0x62f20bdfa018 - std::panicking::catch_unwind::h8609b446221def5f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 30: 0x62f20bdfa018 - std::panic::catch_unwind::h362f73cc48531e26 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 31: 0x62f20bdfa018 - test::run_test_in_process::hf52fd21d696aae67 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 32: 0x62f20bdfa018 - test::run_test::{{closure}}::h7e780b8354e6d007 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 33: 0x62f20bdd2de4 - test::run_test::{{closure}}::hdc2695f2276aad3e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 34: 0x62f20bdd2de4 - std::sys::backtrace::__rust_begin_short_backtrace::h2555b2113ed90fc9 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 35: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::h99800d480987dfb6 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 36: 0x62f20bdd671a - as core::ops::function::FnOnce<()>>::call_once::hbbb69b5294b2221b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 37: 0x62f20bdd671a - std::panicking::catch_unwind::do_call::h495689350c27df20 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 38: 0x62f20bdd671a - std::panicking::catch_unwind::hf22437c51d1b3740 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 39: 0x62f20bdd671a - std::panic::catch_unwind::habda1a57de8231f8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 40: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::h5833d8dda8c7e80b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 41: 0x62f20bdd671a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha2910283a5058402 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 42: 0x62f20d04a16f - as core::ops::function::FnOnce>::call_once::he4d85cb537521972 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 43: 0x62f20d04a16f - std::sys::thread::unix::Thread::new::thread_start::hbaf586a9e50a98cd [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 44: 0x78fade4b5aa4 - [INFO] [stdout] 45: 0x78fade542a64 - clone [INFO] [stdout] 46: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter::integration_tests::test_edge_cases stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter::integration_tests::test_edge_cases' (32) panicked at src/../modules/file_splitter/splitter.rs:263:9: [INFO] [stdout] assertion failed: chunks[0].metadata.has_headers [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62f20d077c42 - std::backtrace_rs::backtrace::libunwind::trace::h16acf28769fcd909 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62f20d077c42 - std::backtrace_rs::backtrace::trace_unsynchronized::hb5541aa5d4f92513 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62f20d077c42 - std::sys::backtrace::_print_fmt::h8ac8268cb667a306 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x62f20d077c42 - ::fmt::h47627beda0767f4c [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x62f20d08b55f - core::fmt::rt::Argument::fmt::h686a01ad57c1c92a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x62f20d08b55f - core::fmt::write::h8a63064c67176fd8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/mod.rs:1469:25 [INFO] [stdout] 6: 0x62f20d041df3 - std::io::default_write_fmt::hd4797e693c0ec256 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62f20d041df3 - std::io::Write::write_fmt::h791e240ad2720c99 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x62f20d04ee22 - std::sys::backtrace::BacktraceLock::print::h4634d27c0afbaebb [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x62f20d05491f - std::panicking::default_hook::{{closure}}::h23e0299d4738b766 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x62f20d0547b1 - std::panicking::default_hook::hfeec6294e8751b62 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x62f20bde5bae - as core::ops::function::Fn>::call::h3f5a16438cee2b4e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 12: 0x62f20bde5bae - test::test_main_with_exit_callback::{{closure}}::h9214498cef9aafc5 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62f20d054fdf - as core::ops::function::Fn>::call::hdb18da3e68df7464 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 14: 0x62f20d054fdf - std::panicking::panic_with_hook::hd57cd463d793ed79 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x62f20d054d56 - std::panicking::panic_handler::{{closure}}::hddd6216c3fddee3b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:700:13 [INFO] [stdout] 16: 0x62f20d04ef59 - std::sys::backtrace::__rust_end_short_backtrace::h5e7a56f283378525 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x62f20d03589d - __rustc[256c90bb7e548b45]::rust_begin_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x62f20d0963f0 - core::panicking::panic_fmt::hf19c198748ef6a95 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62f20d0963cc - core::panicking::panic::h5387b4c27e81d0dd [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:150:5 [INFO] [stdout] 20: 0x62f20bd60352 - ai_workbench_lib::modules::file_splitter::splitter::integration_tests::test_edge_cases::h2249bc5dddf83e23 [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:263:9 [INFO] [stdout] 21: 0x62f20bd60657 - ai_workbench_lib::modules::file_splitter::splitter::integration_tests::test_edge_cases::{{closure}}::he9169380fcc86e7c [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:251:25 [INFO] [stdout] 22: 0x62f20bd297b6 - core::ops::function::FnOnce::call_once::hb1e68f9043acc238 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 23: 0x62f20bde59fb - core::ops::function::FnOnce::call_once::h43f43c5185cf39c2 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62f20bde59fb - test::__rust_begin_short_backtrace::hc4fa634eb2764bdc [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 25: 0x62f20bdfa018 - test::run_test_in_process::{{closure}}::h9f4b799a5211b72a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 26: 0x62f20bdfa018 - as core::ops::function::FnOnce<()>>::call_once::ha572bf7dc5230a8f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 27: 0x62f20bdfa018 - std::panicking::catch_unwind::do_call::hfd6ff51f25dd177b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 28: 0x62f20bdfa018 - std::panicking::catch_unwind::h8609b446221def5f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 29: 0x62f20bdfa018 - std::panic::catch_unwind::h362f73cc48531e26 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 30: 0x62f20bdfa018 - test::run_test_in_process::hf52fd21d696aae67 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 31: 0x62f20bdfa018 - test::run_test::{{closure}}::h7e780b8354e6d007 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 32: 0x62f20bdd2de4 - test::run_test::{{closure}}::hdc2695f2276aad3e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 33: 0x62f20bdd2de4 - std::sys::backtrace::__rust_begin_short_backtrace::h2555b2113ed90fc9 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 34: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::h99800d480987dfb6 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 35: 0x62f20bdd671a - as core::ops::function::FnOnce<()>>::call_once::hbbb69b5294b2221b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 36: 0x62f20bdd671a - std::panicking::catch_unwind::do_call::h495689350c27df20 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 37: 0x62f20bdd671a - std::panicking::catch_unwind::hf22437c51d1b3740 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 38: 0x62f20bdd671a - std::panic::catch_unwind::habda1a57de8231f8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 39: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::h5833d8dda8c7e80b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 40: 0x62f20bdd671a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha2910283a5058402 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 41: 0x62f20d04a16f - as core::ops::function::FnOnce>::call_once::he4d85cb537521972 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 42: 0x62f20d04a16f - std::sys::thread::unix::Thread::new::thread_start::hbaf586a9e50a98cd [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 43: 0x78fade4b5aa4 - [INFO] [stdout] 44: 0x78fade542a64 - clone [INFO] [stdout] 45: 0x0 - [INFO] [stdout] [INFO] [stdout] ---- modules::file_splitter::splitter::integration_tests::test_performance_characteristics stdout ---- [INFO] [stdout] [INFO] [stdout] thread 'modules::file_splitter::splitter::integration_tests::test_performance_characteristics' (36) panicked at src/../modules/file_splitter/splitter.rs:288:9: [INFO] [stdout] assertion failed: chunks.len() > 1 [INFO] [stdout] stack backtrace: [INFO] [stdout] 0: 0x62f20d077c42 - std::backtrace_rs::backtrace::libunwind::trace::h16acf28769fcd909 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/libunwind.rs:117:9 [INFO] [stdout] 1: 0x62f20d077c42 - std::backtrace_rs::backtrace::trace_unsynchronized::hb5541aa5d4f92513 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/../../backtrace/src/backtrace/mod.rs:66:14 [INFO] [stdout] 2: 0x62f20d077c42 - std::sys::backtrace::_print_fmt::h8ac8268cb667a306 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:66:9 [INFO] [stdout] 3: 0x62f20d077c42 - ::fmt::h47627beda0767f4c [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:39:26 [INFO] [stdout] 4: 0x62f20d08b55f - core::fmt::rt::Argument::fmt::h686a01ad57c1c92a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/rt.rs:173:76 [INFO] [stdout] 5: 0x62f20d08b55f - core::fmt::write::h8a63064c67176fd8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/fmt/mod.rs:1469:25 [INFO] [stdout] 6: 0x62f20d041df3 - std::io::default_write_fmt::hd4797e693c0ec256 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:639:11 [INFO] [stdout] 7: 0x62f20d041df3 - std::io::Write::write_fmt::h791e240ad2720c99 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/io/mod.rs:1954:13 [INFO] [stdout] 8: 0x62f20d04ee22 - std::sys::backtrace::BacktraceLock::print::h4634d27c0afbaebb [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:42:9 [INFO] [stdout] 9: 0x62f20d05491f - std::panicking::default_hook::{{closure}}::h23e0299d4738b766 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:301:27 [INFO] [stdout] 10: 0x62f20d0547b1 - std::panicking::default_hook::hfeec6294e8751b62 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:325:9 [INFO] [stdout] 11: 0x62f20bde5bae - as core::ops::function::Fn>::call::h3f5a16438cee2b4e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 12: 0x62f20bde5bae - test::test_main_with_exit_callback::{{closure}}::h9214498cef9aafc5 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:145:21 [INFO] [stdout] 13: 0x62f20d054fdf - as core::ops::function::Fn>::call::hdb18da3e68df7464 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2019:9 [INFO] [stdout] 14: 0x62f20d054fdf - std::panicking::panic_with_hook::hd57cd463d793ed79 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:842:13 [INFO] [stdout] 15: 0x62f20d054d56 - std::panicking::panic_handler::{{closure}}::hddd6216c3fddee3b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:700:13 [INFO] [stdout] 16: 0x62f20d04ef59 - std::sys::backtrace::__rust_end_short_backtrace::h5e7a56f283378525 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:174:18 [INFO] [stdout] 17: 0x62f20d03589d - __rustc[256c90bb7e548b45]::rust_begin_unwind [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:698:5 [INFO] [stdout] 18: 0x62f20d0963f0 - core::panicking::panic_fmt::hf19c198748ef6a95 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:80:14 [INFO] [stdout] 19: 0x62f20d0963cc - core::panicking::panic::h5387b4c27e81d0dd [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panicking.rs:150:5 [INFO] [stdout] 20: 0x62f20bd6251e - ai_workbench_lib::modules::file_splitter::splitter::integration_tests::test_performance_characteristics::h0386d97bd79d011e [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:288:9 [INFO] [stdout] 21: 0x62f20bd62757 - ai_workbench_lib::modules::file_splitter::splitter::integration_tests::test_performance_characteristics::{{closure}}::h762471e4047501cb [INFO] [stdout] at /opt/rustwide/workdir/src/../modules/file_splitter/splitter.rs:273:42 [INFO] [stdout] 22: 0x62f20bd29306 - core::ops::function::FnOnce::call_once::h7bf529cefc9592b4 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 23: 0x62f20bde59fb - core::ops::function::FnOnce::call_once::h43f43c5185cf39c2 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 24: 0x62f20bde59fb - test::__rust_begin_short_backtrace::hc4fa634eb2764bdc [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:663:18 [INFO] [stdout] 25: 0x62f20bdfa018 - test::run_test_in_process::{{closure}}::h9f4b799a5211b72a [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:74 [INFO] [stdout] 26: 0x62f20bdfa018 - as core::ops::function::FnOnce<()>>::call_once::ha572bf7dc5230a8f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 27: 0x62f20bdfa018 - std::panicking::catch_unwind::do_call::hfd6ff51f25dd177b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 28: 0x62f20bdfa018 - std::panicking::catch_unwind::h8609b446221def5f [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 29: 0x62f20bdfa018 - std::panic::catch_unwind::h362f73cc48531e26 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 30: 0x62f20bdfa018 - test::run_test_in_process::hf52fd21d696aae67 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:686:27 [INFO] [stdout] 31: 0x62f20bdfa018 - test::run_test::{{closure}}::h7e780b8354e6d007 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:607:43 [INFO] [stdout] 32: 0x62f20bdd2de4 - test::run_test::{{closure}}::hdc2695f2276aad3e [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/test/src/lib.rs:637:41 [INFO] [stdout] 33: 0x62f20bdd2de4 - std::sys::backtrace::__rust_begin_short_backtrace::h2555b2113ed90fc9 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/backtrace.rs:158:18 [INFO] [stdout] 34: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::h99800d480987dfb6 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:562:17 [INFO] [stdout] 35: 0x62f20bdd671a - as core::ops::function::FnOnce<()>>::call_once::hbbb69b5294b2221b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/panic/unwind_safe.rs:274:9 [INFO] [stdout] 36: 0x62f20bdd671a - std::panicking::catch_unwind::do_call::h495689350c27df20 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:590:40 [INFO] [stdout] 37: 0x62f20bdd671a - std::panicking::catch_unwind::hf22437c51d1b3740 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panicking.rs:553:19 [INFO] [stdout] 38: 0x62f20bdd671a - std::panic::catch_unwind::habda1a57de8231f8 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/panic.rs:359:14 [INFO] [stdout] 39: 0x62f20bdd671a - std::thread::Builder::spawn_unchecked_::{{closure}}::h5833d8dda8c7e80b [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/thread/mod.rs:560:30 [INFO] [stdout] 40: 0x62f20bdd671a - core::ops::function::FnOnce::call_once{{vtable.shim}}::ha2910283a5058402 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/core/src/ops/function.rs:250:5 [INFO] [stdout] 41: 0x62f20d04a16f - as core::ops::function::FnOnce>::call_once::he4d85cb537521972 [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/alloc/src/boxed.rs:2005:9 [INFO] [stdout] 42: 0x62f20d04a16f - std::sys::thread::unix::Thread::new::thread_start::hbaf586a9e50a98cd [INFO] [stdout] at /rustc/3b4dd9bf1410f8da6329baa36ce5e37673cbbd1f/library/std/src/sys/thread/unix.rs:126:17 [INFO] [stdout] 43: 0x78fade4b5aa4 - [INFO] [stdout] 44: 0x78fade542a64 - clone [INFO] [stdout] 45: 0x0 - [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] failures: [INFO] [stdout] modules::file_splitter::splitter::integration_tests::test_edge_cases [INFO] [stdout] modules::file_splitter::splitter::integration_tests::test_mixed_file_types [INFO] [stdout] modules::file_splitter::splitter::integration_tests::test_performance_characteristics [INFO] [stdout] modules::file_splitter::splitter::tests::test_binary_file_rejection [INFO] [stdout] modules::file_splitter::splitter_types::json_splitter::tests::test_invalid_json_rejection [INFO] [stdout] modules::file_splitter::splitter_types::json_splitter::tests::test_json_array_detection [INFO] [stdout] modules::file_splitter::splitter_types::json_splitter::tests::test_standard_json_detection [INFO] [stdout] [INFO] [stdout] test result: FAILED. 30 passed; 7 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.32s [INFO] [stdout] [INFO] running `Command { std: "docker" "inspect" "5c269d0fc29196fff7b39a0afc81a54ead67e7a95847c233b2892e3b44035595", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "5c269d0fc29196fff7b39a0afc81a54ead67e7a95847c233b2892e3b44035595", kill_on_drop: false }` [INFO] [stdout] 5c269d0fc29196fff7b39a0afc81a54ead67e7a95847c233b2892e3b44035595