[INFO] cloning repository https://github.com/tuaregsand/awesome-web-spider [INFO] running `Command { std: "git" "-c" "credential.helper=" "-c" "credential.helper=/workspace/cargo-home/bin/git-credential-null" "clone" "--bare" "https://github.com/tuaregsand/awesome-web-spider" "/workspace/cache/git-repos/https%3A%2F%2Fgithub.com%2Ftuaregsand%2Fawesome-web-spider", kill_on_drop: false }` [INFO] [stderr] Cloning into bare repository '/workspace/cache/git-repos/https%3A%2F%2Fgithub.com%2Ftuaregsand%2Fawesome-web-spider'... [INFO] running `Command { std: "git" "rev-parse" "HEAD", kill_on_drop: false }` [INFO] [stdout] 52ab35243a2aaaaf716ca7346d917f479883b3fa [INFO] checking tuaregsand/awesome-web-spider against master#779e19d8baa3e3625bd4fc5c85cbb2ad47b43155 for pr-147589-1 [INFO] running `Command { std: "git" "clone" "/workspace/cache/git-repos/https%3A%2F%2Fgithub.com%2Ftuaregsand%2Fawesome-web-spider" "/workspace/builds/worker-4-tc1/source", kill_on_drop: false }` [INFO] [stderr] Cloning into '/workspace/builds/worker-4-tc1/source'... [INFO] [stderr] done. [INFO] started tweaking git repo https://github.com/tuaregsand/awesome-web-spider [INFO] finished tweaking git repo https://github.com/tuaregsand/awesome-web-spider [INFO] tweaked toml for git repo https://github.com/tuaregsand/awesome-web-spider written to /workspace/builds/worker-4-tc1/source/Cargo.toml [INFO] validating manifest of git repo https://github.com/tuaregsand/awesome-web-spider on toolchain 779e19d8baa3e3625bd4fc5c85cbb2ad47b43155 [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+779e19d8baa3e3625bd4fc5c85cbb2ad47b43155" "metadata" "--manifest-path" "Cargo.toml" "--no-deps", kill_on_drop: false }` [INFO] crate git repo https://github.com/tuaregsand/awesome-web-spider already has a lockfile, it will not be regenerated [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+779e19d8baa3e3625bd4fc5c85cbb2ad47b43155" "fetch" "--manifest-path" "Cargo.toml", kill_on_drop: false }` [INFO] [stderr] Updating crates.io index [INFO] [stderr] Downloading crates ... [INFO] [stderr] Downloaded scraper v0.18.1 [INFO] [stderr] Downloaded bloom v0.3.2 [INFO] [stderr] Downloaded robotstxt v0.2.0 [INFO] [stderr] Downloaded fancy-regex v0.12.0 [INFO] [stderr] Downloaded tiktoken-rs v0.5.9 [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:e90291280db7d1fac5b66fc6dad9f9662629e7365a55743daf9bdf73ebc4ea79" "/opt/rustwide/cargo-home/bin/cargo" "+779e19d8baa3e3625bd4fc5c85cbb2ad47b43155" "metadata" "--no-deps" "--format-version=1", kill_on_drop: false }` [INFO] [stdout] d8b528be0601ce3827636a15c4404483ceb7575db45b4f2528c31c93990d1d9a [INFO] running `Command { std: "docker" "start" "-a" "d8b528be0601ce3827636a15c4404483ceb7575db45b4f2528c31c93990d1d9a", kill_on_drop: false }` [INFO] running `Command { std: "docker" "inspect" "d8b528be0601ce3827636a15c4404483ceb7575db45b4f2528c31c93990d1d9a", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "d8b528be0601ce3827636a15c4404483ceb7575db45b4f2528c31c93990d1d9a", kill_on_drop: false }` [INFO] [stdout] d8b528be0601ce3827636a15c4404483ceb7575db45b4f2528c31c93990d1d9a [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc1/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc1/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:e90291280db7d1fac5b66fc6dad9f9662629e7365a55743daf9bdf73ebc4ea79" "/opt/rustwide/cargo-home/bin/cargo" "+779e19d8baa3e3625bd4fc5c85cbb2ad47b43155" "check" "--frozen" "--all" "--all-targets" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] dfb8982f5db6b7118e9078aae52855ce965c17aafb1eea9aada93047d141cdd0 [INFO] running `Command { std: "docker" "start" "-a" "dfb8982f5db6b7118e9078aae52855ce965c17aafb1eea9aada93047d141cdd0", kill_on_drop: false }` [INFO] [stderr] Checking smallvec v1.15.0 [INFO] [stderr] Compiling zerocopy v0.8.25 [INFO] [stderr] Compiling phf_shared v0.11.3 [INFO] [stderr] Compiling getrandom v0.2.16 [INFO] [stderr] Checking yoke v0.8.0 [INFO] [stderr] Compiling cc v1.2.25 [INFO] [stderr] Checking serde v1.0.219 [INFO] [stderr] Checking mio v1.0.4 [INFO] [stderr] Checking socket2 v0.5.10 [INFO] [stderr] Checking futures-util v0.3.31 [INFO] [stderr] Compiling ahash v0.8.12 [INFO] [stderr] Checking phf_shared v0.10.0 [INFO] [stderr] Checking indexmap v2.9.0 [INFO] [stderr] Compiling openssl-macros v0.1.1 [INFO] [stderr] Checking regex-automata v0.4.9 [INFO] [stderr] Compiling rand_core v0.6.4 [INFO] [stderr] Checking parking_lot_core v0.9.11 [INFO] [stderr] Checking zerovec v0.11.2 [INFO] [stderr] Checking zerotrie v0.2.2 [INFO] [stderr] Checking phf v0.10.1 [INFO] [stderr] Checking tracing v0.1.41 [INFO] [stderr] Compiling syn v1.0.109 [INFO] [stderr] Checking getrandom v0.3.3 [INFO] [stderr] Checking dtoa v1.0.10 [INFO] [stderr] Checking mio v0.8.11 [INFO] [stderr] Checking parking_lot v0.12.4 [INFO] [stderr] Compiling num-traits v0.2.19 [INFO] [stderr] Compiling cssparser-macros v0.6.1 [INFO] [stderr] Checking bit-vec v0.6.3 [INFO] [stderr] Checking dtoa-short v0.3.5 [INFO] [stderr] Checking unicode-width v0.1.14 [INFO] [stderr] Compiling strum_macros v0.25.3 [INFO] [stderr] Compiling derive_more v0.99.20 [INFO] [stderr] Checking bit-set v0.5.3 [INFO] [stderr] Checking signal-hook-mio v0.2.4 [INFO] [stderr] Checking clap_builder v4.5.39 [INFO] [stderr] Checking getopts v0.2.21 [INFO] [stderr] Checking tokio v1.45.1 [INFO] [stderr] Checking crossterm v0.27.0 [INFO] [stderr] Checking console v0.15.11 [INFO] [stderr] Checking hashbrown v0.14.5 [INFO] [stderr] Checking itertools v0.11.0 [INFO] [stderr] Checking tracing-log v0.2.0 [INFO] [stderr] Checking lru v0.12.5 [INFO] [stderr] Checking ego-tree v0.6.3 [INFO] [stderr] Checking bit-vec v0.4.4 [INFO] [stderr] Compiling ppv-lite86 v0.2.21 [INFO] [stderr] Checking tinystr v0.8.1 [INFO] [stderr] Checking potential_utf v0.1.2 [INFO] [stderr] Checking icu_locale_core v2.0.0 [INFO] [stderr] Checking bloom v0.3.2 [INFO] [stderr] Checking indicatif v0.17.11 [INFO] [stderr] Checking tracing-subscriber v0.3.19 [INFO] [stderr] Checking robotstxt v0.2.0 [INFO] [stderr] Compiling rand_chacha v0.3.1 [INFO] [stderr] Compiling openssl-sys v0.9.109 [INFO] [stderr] Checking icu_collections v2.0.0 [INFO] [stderr] Checking clap v4.5.39 [INFO] [stderr] Checking icu_provider v2.0.0 [INFO] [stderr] Checking dashmap v5.5.3 [INFO] [stderr] Compiling rand v0.8.5 [INFO] [stderr] Checking icu_normalizer v2.0.0 [INFO] [stderr] Checking icu_properties v2.0.1 [INFO] [stderr] Compiling openssl v0.10.73 [INFO] [stderr] Compiling native-tls v0.2.14 [INFO] [stderr] Checking string_cache v0.8.9 [INFO] [stderr] Checking serde_urlencoded v0.7.1 [INFO] [stderr] Checking serde_json v1.0.140 [INFO] [stderr] Checking serde_yaml v0.9.34+deprecated [INFO] [stderr] Checking strum v0.25.0 [INFO] [stderr] Checking chrono v0.4.41 [INFO] [stderr] Checking futures-executor v0.3.31 [INFO] [stderr] Checking regex v1.11.1 [INFO] [stderr] Checking bstr v1.12.0 [INFO] [stderr] Checking ratatui v0.24.0 [INFO] [stderr] Checking futures v0.3.31 [INFO] [stderr] Compiling phf_generator v0.11.3 [INFO] [stderr] Compiling phf_generator v0.10.0 [INFO] [stderr] Compiling phf_codegen v0.10.0 [INFO] [stderr] Compiling string_cache_codegen v0.5.4 [INFO] [stderr] Compiling phf_macros v0.11.3 [INFO] [stderr] Compiling selectors v0.25.0 [INFO] [stderr] Compiling markup5ever v0.11.0 [INFO] [stderr] Checking fancy-regex v0.12.0 [INFO] [stderr] Checking idna_adapter v1.2.1 [INFO] [stderr] Checking idna v1.0.3 [INFO] [stderr] Checking tiktoken-rs v0.5.9 [INFO] [stderr] Checking phf v0.11.3 [INFO] [stderr] Checking url v2.5.4 [INFO] [stderr] Checking cssparser v0.31.2 [INFO] [stderr] Compiling html5ever v0.26.0 [INFO] [stderr] Checking tokio-util v0.7.15 [INFO] [stderr] Checking h2 v0.3.26 [INFO] [stderr] Checking scraper v0.18.1 [INFO] [stderr] Checking tokio-native-tls v0.3.1 [INFO] [stderr] Checking hyper v0.14.32 [INFO] [stderr] Checking hyper-tls v0.5.0 [INFO] [stderr] Checking reqwest v0.11.27 [INFO] [stderr] Checking web-spider v0.1.0 (/opt/rustwide/workdir) [INFO] [stdout] warning: unused variable: `hash` [INFO] [stdout] --> src/ai_analyzer.rs:195:57 [INFO] [stdout] | [INFO] [stdout] 195 | fn check_duplicate_similarity(&self, content: &str, hash: &str) -> f64 { [INFO] [stdout] | ^^^^ help: if this is intentional, prefix it with an underscore: `_hash` [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: unused variable: `hash` [INFO] [stdout] --> src/ai_analyzer.rs:195:57 [INFO] [stdout] | [INFO] [stdout] 195 | fn check_duplicate_similarity(&self, content: &str, hash: &str) -> f64 { [INFO] [stdout] | ^^^^ help: if this is intentional, prefix it with an underscore: `_hash` [INFO] [stdout] | [INFO] [stdout] = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: field `parent_url` is never read [INFO] [stdout] --> src/crawler.rs:20:9 [INFO] [stdout] | [INFO] [stdout] 17 | pub struct CrawlTask { [INFO] [stdout] | --------- field in this struct [INFO] [stdout] ... [INFO] [stdout] 20 | pub parent_url: Option, [INFO] [stdout] | ^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `CrawlTask` has derived impls for the traits `Clone` and `Debug`, but these are intentionally ignored during dead code analysis [INFO] [stdout] = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: struct `AIContentAnalyzer` is never constructed [INFO] [stdout] --> src/ai_analyzer.rs:69:12 [INFO] [stdout] | [INFO] [stdout] 69 | pub struct AIContentAnalyzer { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: multiple associated items are never used [INFO] [stdout] --> src/ai_analyzer.rs:77:12 [INFO] [stdout] | [INFO] [stdout] 76 | impl AIContentAnalyzer { [INFO] [stdout] | ---------------------- associated items in this implementation [INFO] [stdout] 77 | pub fn new() -> Result { [INFO] [stdout] | ^^^ [INFO] [stdout] ... [INFO] [stdout] 117 | pub async fn analyze_content( [INFO] [stdout] | ^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 173 | fn extract_text_content(&self, document: &Html) -> String { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 186 | fn calculate_content_hash(&self, content: &str) -> String { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 195 | fn check_duplicate_similarity(&self, content: &str, hash: &str) -> f64 { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 209 | fn classify_content_type(&self, text: &str, document: &Html) -> ContentType { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 248 | fn extract_keywords(&self, text: &str) -> Vec { [INFO] [stdout] | ^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 267 | fn analyze_sentiment(&self, text: &str) -> f64 { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 290 | fn calculate_readability(&self, text: &str) -> f64 { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 308 | fn analyze_seo(&self, document: &Html, text: &str) -> SeoAnalysis { [INFO] [stdout] | ^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 376 | fn classify_topics(&self, text: &str) -> Vec { [INFO] [stdout] | ^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 409 | fn detect_language(&self, text: &str) -> Option { [INFO] [stdout] | ^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 440 | fn generate_summary(&self, text: &str) -> String { [INFO] [stdout] | ^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 465 | fn analyze_technical_aspects( [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: method `get_domain` is never used [INFO] [stdout] --> src/config.rs:17:12 [INFO] [stdout] | [INFO] [stdout] 16 | impl CrawlerConfig { [INFO] [stdout] | ------------------ method in this implementation [INFO] [stdout] 17 | pub fn get_domain(&self) -> Option { [INFO] [stdout] | ^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: field `parent_url` is never read [INFO] [stdout] --> src/crawler.rs:20:9 [INFO] [stdout] | [INFO] [stdout] 17 | pub struct CrawlTask { [INFO] [stdout] | --------- field in this struct [INFO] [stdout] ... [INFO] [stdout] 20 | pub parent_url: Option, [INFO] [stdout] | ^^^^^^^^^^ [INFO] [stdout] | [INFO] [stdout] = note: `CrawlTask` has derived impls for the traits `Clone` and `Debug`, but these are intentionally ignored during dead code analysis [INFO] [stdout] = note: `#[warn(dead_code)]` (part of `#[warn(unused)]`) on by default [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: struct `AIContentAnalyzer` is never constructed [INFO] [stdout] --> src/ai_analyzer.rs:69:12 [INFO] [stdout] | [INFO] [stdout] 69 | pub struct AIContentAnalyzer { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: multiple associated items are never used [INFO] [stdout] --> src/ai_analyzer.rs:77:12 [INFO] [stdout] | [INFO] [stdout] 76 | impl AIContentAnalyzer { [INFO] [stdout] | ---------------------- associated items in this implementation [INFO] [stdout] 77 | pub fn new() -> Result { [INFO] [stdout] | ^^^ [INFO] [stdout] ... [INFO] [stdout] 117 | pub async fn analyze_content( [INFO] [stdout] | ^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 173 | fn extract_text_content(&self, document: &Html) -> String { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 186 | fn calculate_content_hash(&self, content: &str) -> String { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 195 | fn check_duplicate_similarity(&self, content: &str, hash: &str) -> f64 { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 209 | fn classify_content_type(&self, text: &str, document: &Html) -> ContentType { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 248 | fn extract_keywords(&self, text: &str) -> Vec { [INFO] [stdout] | ^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 267 | fn analyze_sentiment(&self, text: &str) -> f64 { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 290 | fn calculate_readability(&self, text: &str) -> f64 { [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 308 | fn analyze_seo(&self, document: &Html, text: &str) -> SeoAnalysis { [INFO] [stdout] | ^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 376 | fn classify_topics(&self, text: &str) -> Vec { [INFO] [stdout] | ^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 409 | fn detect_language(&self, text: &str) -> Option { [INFO] [stdout] | ^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 440 | fn generate_summary(&self, text: &str) -> String { [INFO] [stdout] | ^^^^^^^^^^^^^^^^ [INFO] [stdout] ... [INFO] [stdout] 465 | fn analyze_technical_aspects( [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: method `get_domain` is never used [INFO] [stdout] --> src/config.rs:17:12 [INFO] [stdout] | [INFO] [stdout] 16 | impl CrawlerConfig { [INFO] [stdout] | ------------------ method in this implementation [INFO] [stdout] 17 | pub fn get_domain(&self) -> Option { [INFO] [stdout] | ^^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stderr] Finished `dev` profile [unoptimized + debuginfo] target(s) in 32.24s [INFO] running `Command { std: "docker" "inspect" "dfb8982f5db6b7118e9078aae52855ce965c17aafb1eea9aada93047d141cdd0", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "dfb8982f5db6b7118e9078aae52855ce965c17aafb1eea9aada93047d141cdd0", kill_on_drop: false }` [INFO] [stdout] dfb8982f5db6b7118e9078aae52855ce965c17aafb1eea9aada93047d141cdd0