[INFO] fetching crate hydroflow_plus 0.10.0... [INFO] checking hydroflow_plus-0.10.0 against try#a4980311fb7bb9e7893708e6bd3fbbfb2819fd3d for pr-145342-1 [INFO] extracting crate hydroflow_plus 0.10.0 into /workspace/builds/worker-4-tc2/source [INFO] started tweaking crates.io crate hydroflow_plus 0.10.0 [INFO] finished tweaking crates.io crate hydroflow_plus 0.10.0 [INFO] tweaked toml for crates.io crate hydroflow_plus 0.10.0 written to /workspace/builds/worker-4-tc2/source/Cargo.toml [INFO] validating manifest of crates.io crate hydroflow_plus 0.10.0 on toolchain a4980311fb7bb9e7893708e6bd3fbbfb2819fd3d [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+a4980311fb7bb9e7893708e6bd3fbbfb2819fd3d" "metadata" "--manifest-path" "Cargo.toml" "--no-deps", kill_on_drop: false }` [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+a4980311fb7bb9e7893708e6bd3fbbfb2819fd3d" "generate-lockfile" "--manifest-path" "Cargo.toml", kill_on_drop: false }` [INFO] [stderr] Updating crates.io index [INFO] [stderr] Locking 247 packages to latest compatible versions [INFO] [stderr] Adding bincode v1.3.3 (available: v2.0.1) [INFO] [stderr] Adding ctor v0.2.9 (available: v0.5.0) [INFO] [stderr] Adding hydro_deploy v0.10.0 (available: v0.14.0) [INFO] [stderr] Adding proc-macro-crate v1.3.1 (available: v3.3.0) [INFO] [stderr] Adding stageleft v0.5.0 (available: v0.9.6) [INFO] [stderr] Adding stageleft_tool v0.4.0 (available: v0.9.6) [INFO] [stderr] Adding toml v0.8.23 (available: v0.9.5) [INFO] running `Command { std: CARGO_HOME="/workspace/cargo-home" RUSTUP_HOME="/workspace/rustup-home" "/workspace/cargo-home/bin/cargo" "+a4980311fb7bb9e7893708e6bd3fbbfb2819fd3d" "fetch" "--manifest-path" "Cargo.toml", kill_on_drop: false }` [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7ad1b28ee6f5f7f699f6cf7015098d6ccdd96d6f2d78dd06228f5b4c9faf309c" "/opt/rustwide/cargo-home/bin/cargo" "+a4980311fb7bb9e7893708e6bd3fbbfb2819fd3d" "metadata" "--no-deps" "--format-version=1", kill_on_drop: false }` [INFO] [stdout] dee34cec5cff5872319a2ed5663fe4b174a47f083d638f3cec8e6a42cb15273f [INFO] running `Command { std: "docker" "start" "-a" "dee34cec5cff5872319a2ed5663fe4b174a47f083d638f3cec8e6a42cb15273f", kill_on_drop: false }` [INFO] running `Command { std: "docker" "inspect" "dee34cec5cff5872319a2ed5663fe4b174a47f083d638f3cec8e6a42cb15273f", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "dee34cec5cff5872319a2ed5663fe4b174a47f083d638f3cec8e6a42cb15273f", kill_on_drop: false }` [INFO] [stdout] dee34cec5cff5872319a2ed5663fe4b174a47f083d638f3cec8e6a42cb15273f [INFO] running `Command { std: "docker" "create" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc2/target:/opt/rustwide/target:rw,Z" "-v" "/var/lib/crater-agent-workspace/builds/worker-4-tc2/source:/opt/rustwide/workdir:ro,Z" "-v" "/var/lib/crater-agent-workspace/cargo-home:/opt/rustwide/cargo-home:ro,Z" "-v" "/var/lib/crater-agent-workspace/rustup-home:/opt/rustwide/rustup-home:ro,Z" "-e" "SOURCE_DIR=/opt/rustwide/workdir" "-e" "CARGO_TARGET_DIR=/opt/rustwide/target" "-e" "CARGO_INCREMENTAL=0" "-e" "RUST_BACKTRACE=full" "-e" "RUSTFLAGS=--cap-lints=forbid" "-e" "RUSTDOCFLAGS=--cap-lints=forbid" "-e" "CARGO_HOME=/opt/rustwide/cargo-home" "-e" "RUSTUP_HOME=/opt/rustwide/rustup-home" "-w" "/opt/rustwide/workdir" "-m" "1610612736" "--user" "0:0" "--network" "none" "ghcr.io/rust-lang/crates-build-env/linux@sha256:7ad1b28ee6f5f7f699f6cf7015098d6ccdd96d6f2d78dd06228f5b4c9faf309c" "/opt/rustwide/cargo-home/bin/cargo" "+a4980311fb7bb9e7893708e6bd3fbbfb2819fd3d" "check" "--frozen" "--all" "--all-targets" "--message-format=json", kill_on_drop: false }` [INFO] [stdout] 5afa19b2952e29819cbfb4e0ac601e2027c0fdefe8b30d3ba7d0a5d4cd1fedc6 [INFO] running `Command { std: "docker" "start" "-a" "5afa19b2952e29819cbfb4e0ac601e2027c0fdefe8b30d3ba7d0a5d4cd1fedc6", kill_on_drop: false }` [INFO] [stderr] Compiling proc-macro2 v1.0.101 [INFO] [stderr] Compiling unicode-ident v1.0.18 [INFO] [stderr] Compiling libc v0.2.175 [INFO] [stderr] Compiling version_check v0.9.5 [INFO] [stderr] Checking cfg-if v1.0.3 [INFO] [stderr] Compiling typenum v1.18.0 [INFO] [stderr] Compiling zerocopy v0.8.26 [INFO] [stderr] Checking pin-project-lite v0.2.16 [INFO] [stderr] Checking once_cell v1.21.3 [INFO] [stderr] Checking memchr v2.7.5 [INFO] [stderr] Compiling autocfg v1.5.0 [INFO] [stderr] Checking futures-core v0.3.31 [INFO] [stderr] Compiling parking_lot_core v0.9.11 [INFO] [stderr] Compiling serde v1.0.219 [INFO] [stderr] Compiling generic-array v0.14.7 [INFO] [stderr] Compiling getrandom v0.3.3 [INFO] [stderr] Checking smallvec v1.15.1 [INFO] [stderr] Checking scopeguard v1.2.0 [INFO] [stderr] Compiling lock_api v0.4.13 [INFO] [stderr] Checking futures-io v0.3.31 [INFO] [stderr] Compiling crossbeam-utils v0.8.21 [INFO] [stderr] Compiling shlex v1.3.0 [INFO] [stderr] Checking bitflags v2.9.3 [INFO] [stderr] Compiling cc v1.2.34 [INFO] [stderr] Compiling ahash v0.8.12 [INFO] [stderr] Compiling pkg-config v0.3.32 [INFO] [stderr] Compiling quote v1.0.40 [INFO] [stderr] Compiling syn v2.0.106 [INFO] [stderr] Compiling equivalent v1.0.2 [INFO] [stderr] Compiling hashbrown v0.15.5 [INFO] [stderr] Compiling rustix v1.0.8 [INFO] [stderr] Checking futures-sink v0.3.31 [INFO] [stderr] Checking slab v0.4.11 [INFO] [stderr] Compiling toml_datetime v0.6.11 [INFO] [stderr] Compiling vcpkg v0.2.15 [INFO] [stderr] Compiling indexmap v2.11.0 [INFO] [stderr] Checking linux-raw-sys v0.9.4 [INFO] [stderr] Checking allocator-api2 v0.2.21 [INFO] [stderr] Checking bytes v1.10.1 [INFO] [stderr] Checking signal-hook-registry v1.4.6 [INFO] [stderr] Checking socket2 v0.6.0 [INFO] [stderr] Checking parking_lot v0.12.4 [INFO] [stderr] Checking mio v1.0.4 [INFO] [stderr] Checking hashbrown v0.14.5 [INFO] [stderr] Compiling crypto-common v0.1.6 [INFO] [stderr] Compiling block-buffer v0.10.4 [INFO] [stderr] Checking utf8parse v0.2.2 [INFO] [stderr] Checking fastrand v2.3.0 [INFO] [stderr] Compiling heck v0.4.1 [INFO] [stderr] Checking parking v2.2.1 [INFO] [stderr] Checking anstyle-parse v0.2.7 [INFO] [stderr] Compiling digest v0.10.7 [INFO] [stderr] Compiling openssl-src v300.5.2+3.5.2 [INFO] [stderr] Checking futures-channel v0.3.31 [INFO] [stderr] Checking concurrent-queue v2.5.0 [INFO] [stderr] Checking anstyle v1.0.11 [INFO] [stderr] Checking is_terminal_polyfill v1.70.1 [INFO] [stderr] Checking anstyle-query v1.1.4 [INFO] [stderr] Checking colorchoice v1.0.4 [INFO] [stderr] Checking pin-utils v0.1.0 [INFO] [stderr] Checking futures-task v0.3.31 [INFO] [stderr] Checking itoa v1.0.15 [INFO] [stderr] Compiling cpufeatures v0.2.17 [INFO] [stderr] Compiling winnow v0.5.40 [INFO] [stderr] Compiling sha2 v0.10.9 [INFO] [stderr] Checking anstream v0.6.20 [INFO] [stderr] Compiling openssl-sys v0.9.109 [INFO] [stderr] Checking event-listener v5.4.1 [INFO] [stderr] Compiling tokio v1.47.1 [INFO] [stderr] Compiling libz-sys v1.1.22 [INFO] [stderr] Checking clap_lex v0.7.5 [INFO] [stderr] Compiling hex v0.4.3 [INFO] [stderr] Compiling serde_json v1.0.143 [INFO] [stderr] Checking strsim v0.11.1 [INFO] [stderr] Compiling heck v0.5.0 [INFO] [stderr] Checking clap_builder v4.5.44 [INFO] [stderr] Compiling toml_edit v0.19.15 [INFO] [stderr] Checking event-listener-strategy v0.5.4 [INFO] [stderr] Compiling winnow v0.7.13 [INFO] [stderr] Checking ryu v1.0.20 [INFO] [stderr] Checking either v1.15.0 [INFO] [stderr] Compiling prettyplease v0.2.37 [INFO] [stderr] Compiling thiserror v1.0.69 [INFO] [stderr] Compiling aho-corasick v1.1.3 [INFO] [stderr] Compiling libssh2-sys v0.3.1 [INFO] [stderr] Compiling serde_derive v1.0.219 [INFO] [stderr] Compiling async-trait v0.1.89 [INFO] [stderr] Compiling tokio-macros v2.5.0 [INFO] [stderr] Compiling sealed v0.5.0 [INFO] [stderr] Compiling futures-macro v0.3.31 [INFO] [stderr] Compiling sha256 v1.6.0 [INFO] [stderr] Compiling proc-macro-crate v1.3.1 [INFO] [stderr] Compiling clap_derive v4.5.45 [INFO] [stderr] Compiling toml_edit v0.22.27 [INFO] [stderr] Checking futures-util v0.3.31 [INFO] [stderr] Compiling thiserror-impl v1.0.69 [INFO] [stderr] Compiling pin-project-internal v1.1.10 [INFO] [stderr] Checking getrandom v0.2.16 [INFO] [stderr] Compiling slotmap v1.0.7 [INFO] [stderr] Compiling regex-syntax v0.8.6 [INFO] [stderr] Compiling ref-cast v1.0.24 [INFO] [stderr] Compiling object v0.36.7 [INFO] [stderr] Checking clap v4.5.45 [INFO] [stderr] Checking rand_core v0.6.4 [INFO] [stderr] Checking pin-project v1.1.10 [INFO] [stderr] Compiling proc-macro-crate v3.3.0 [INFO] [stderr] Compiling variadics v0.0.8 [INFO] [stderr] Compiling async-recursion v1.1.1 [INFO] [stderr] Compiling ref-cast-impl v1.0.24 [INFO] [stderr] Checking futures-executor v0.3.31 [INFO] [stderr] Checking futures v0.3.31 [INFO] [stderr] Compiling hydroflow_lang v0.10.0 [INFO] [stderr] Checking tempfile v3.21.0 [INFO] [stderr] Checking polling v3.10.0 [INFO] [stderr] Checking itertools v0.10.5 [INFO] [stderr] Checking async-lock v3.4.1 [INFO] [stderr] Compiling syn-inline-mod v0.6.0 [INFO] [stderr] Checking futures-lite v2.6.1 [INFO] [stderr] Checking ppv-lite86 v0.2.21 [INFO] [stderr] Checking tokio-util v0.7.16 [INFO] [stderr] Compiling regex-automata v0.4.10 [INFO] [stderr] Checking unicode-width v0.2.1 [INFO] [stderr] Compiling portable-atomic v1.11.1 [INFO] [stderr] Checking tokio-stream v0.1.17 [INFO] [stderr] Checking log v0.4.27 [INFO] [stderr] Checking adler2 v2.0.1 [INFO] [stderr] Compiling semver v1.0.26 [INFO] [stderr] Compiling camino v1.1.11 [INFO] [stderr] Compiling cfg_aliases v0.2.1 [INFO] [stderr] Checking gimli v0.31.1 [INFO] [stderr] Checking hydroflow_deploy_integration v0.10.0 [INFO] [stderr] Compiling nix v0.29.0 [INFO] [stderr] Checking miniz_oxide v0.8.9 [INFO] [stderr] Checking env_filter v0.1.3 [INFO] [stderr] Checking console v0.15.11 [INFO] [stderr] Checking rand_chacha v0.3.1 [INFO] [stderr] Checking async-io v2.5.0 [INFO] [stderr] Compiling stageleft_tool v0.4.0 [INFO] [stderr] Compiling lattices_macro v0.5.10 [INFO] [stderr] Compiling variadics_macro v0.5.6 [INFO] [stderr] Checking variadics v0.0.7 [INFO] [stderr] Compiling regex v1.11.2 [INFO] [stderr] Compiling auto_impl v1.3.0 [INFO] [stderr] Compiling tracing-attributes v0.1.30 [INFO] [stderr] Checking addr2line v0.24.2 [INFO] [stderr] Compiling try_match_inner v0.5.2 [INFO] [stderr] Checking tracing-core v0.1.34 [INFO] [stderr] Checking cc-traits v2.0.0 [INFO] [stderr] Checking arrayvec v0.7.6 [INFO] [stderr] Compiling lazy_static v1.5.0 [INFO] [stderr] Compiling anyhow v1.0.99 [INFO] [stderr] Checking bytemuck v1.23.2 [INFO] [stderr] Checking rustc-demangle v0.1.26 [INFO] [stderr] Checking num-format v0.4.4 [INFO] [stderr] Checking rgb v0.8.52 [INFO] [stderr] Compiling str_inflector v0.12.0 [INFO] [stderr] Checking tracing v0.1.41 [INFO] [stderr] Checking lattices v0.5.9 [INFO] [stderr] Compiling try_match v0.4.2 [INFO] [stderr] Checking pusherator v0.0.9 [INFO] [stderr] Compiling hydroflow_plus v0.10.0 (/opt/rustwide/workdir) [INFO] [stderr] Checking async-signal v0.2.12 [INFO] [stderr] Checking rand v0.8.5 [INFO] [stderr] Checking env_logger v0.11.8 [INFO] [stderr] Checking cargo-platform v0.1.9 [INFO] [stderr] Checking bincode v1.3.3 [INFO] [stderr] Compiling stageleft_macro v0.4.0 [INFO] [stderr] Checking async-channel v2.5.0 [INFO] [stderr] Checking dashmap v6.1.0 [INFO] [stderr] Checking is-terminal v0.4.16 [INFO] [stderr] Checking crossbeam-channel v0.5.15 [INFO] [stderr] Checking quick-xml v0.26.0 [INFO] [stderr] Checking backtrace v0.3.75 [INFO] [stderr] Checking str_stack v0.1.0 [INFO] [stderr] Checking nameof v1.3.0 [INFO] [stderr] Checking rustc-hash v1.1.0 [INFO] [stderr] Checking byteorder v1.5.0 [INFO] [stderr] Checking number_prefix v0.4.0 [INFO] [stderr] Checking web-time v1.1.0 [INFO] [stderr] Checking async-task v4.7.1 [INFO] [stderr] Checking indicatif v0.17.11 [INFO] [stderr] Checking nanoid v0.4.0 [INFO] [stderr] Checking inferno v0.11.21 [INFO] [stderr] Checking stageleft v0.5.0 [INFO] [stderr] Checking async-process v2.4.0 [INFO] [stderr] Checking cargo_metadata v0.18.1 [INFO] [stderr] Compiling buildstructor v0.5.4 [INFO] [stderr] Checking dyn-clone v1.0.20 [INFO] [stderr] Checking similar v2.7.0 [INFO] [stderr] Checking memo-map v0.3.3 [INFO] [stderr] Checking dunce v1.0.5 [INFO] [stderr] Checking shell-escape v0.1.5 [INFO] [stderr] Checking insta v1.43.1 [INFO] [stderr] Compiling ctor v0.2.9 [INFO] [stderr] Checking hydroflow v0.10.0 [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> /opt/rustwide/target/debug/build/hydroflow_plus-41cd71e800d48fc4/out/lib_pub.rs:1:76054 [INFO] [stdout] | [INFO] [stdout] 1 | ...untimeData < & DeployPorts < HydroflowPlusMeta > > , of_cluster : usize ,) -> impl Quoted < & Vec < u32 > > + Copy { q ! (cli . meta .... [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ------------------------------------ [INFO] [stdout] | | | | [INFO] [stdout] | | | the same lifetime is elided here [INFO] [stdout] | the lifetime is elided here the same lifetime is hidden here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] = note: `#[warn(mismatched_lifetime_syntaxes)]` on by default [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 1 | pub use hydroflow :: scheduled :: graph :: Hydroflow ; pub use hydroflow :: * ; pub use stageleft :: * ; pub mod runtime_support { pub use bincode ; } pub mod runtime_context { pub use std :: marker :: PhantomData ; pub use hydroflow :: scheduled :: context :: Context ; pub use proc_macro2 :: TokenStream ; pub use quote :: quote ; pub use stageleft :: runtime_support :: FreeVariable ; pub use crate :: runtime_context :: RuntimeContext ; # [cfg (stageleft_macro)] impl RuntimeContext < '_ > { pub fn new () -> Self { Self { _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl Copy for RuntimeContext < '_ > { } # [cfg (stageleft_macro)] impl Default for RuntimeContext < '_ > { fn default () -> Self { Self :: new () } } # [cfg (stageleft_macro)] impl < 'a > FreeVariable < & 'a Context > for RuntimeContext < 'a > { fn to_tokens (self) -> (Option < TokenStream > , Option < TokenStream >) { (None , Some (quote ! (& context))) } } } pub use runtime_context :: RuntimeContext ; pub mod stream { pub use std :: cell :: RefCell ; pub use std :: hash :: Hash ; pub use std :: marker :: PhantomData ; pub use std :: ops :: Deref ; pub use std :: rc :: Rc ; pub use hydroflow :: bytes :: Bytes ; pub use hydroflow :: futures :: Sink ; pub use hydroflow_lang :: parse :: Pipeline ; pub use serde :: de :: DeserializeOwned ; pub use serde :: Serialize ; pub use stageleft :: { q , IntoQuotedMut , Quoted } ; pub use syn :: parse_quote ; pub use crate :: __staged :: builder :: FLOW_USED_MESSAGE ; pub use crate :: __staged :: cycle :: { CycleCollection , CycleComplete , DeferTick , ForwardRefMarker , TickCycleMarker } ; pub use crate :: __staged :: ir :: { DebugInstantiate , HfPlusLeaf , HfPlusNode , TeeNode } ; pub use crate :: __staged :: location :: cluster :: ClusterSelfId ; pub use crate :: __staged :: location :: external_process :: { ExternalBincodeStream , ExternalBytesPort } ; pub use crate :: __staged :: location :: { check_matching_location , CanSend , ExternalProcess , Location , LocationId , NoTick , Tick , } ; pub use crate :: __staged :: staging_util :: get_this_crate ; pub use crate :: __staged :: { Cluster , ClusterId , Optional , Process , Singleton } ; pub use crate :: stream :: Unbounded ; pub use crate :: stream :: Bounded ; pub use crate :: stream :: Stream ; # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Stream < T , L , B > { fn location_kind (& self) -> LocationId { self . location . id () } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > DeferTick for Stream < T , Tick < L > , Bounded > { fn defer_tick (self) -> Self { Stream :: defer_tick (self) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleCollection < 'a , TickCycleMarker > for Stream < T , Tick < L > , Bounded > { type Location = Tick < L > ; fn create_source (ident : syn :: Ident , location : Tick < L >) -> Self { let location_id = location . id () ; Stream :: new (location , HfPlusNode :: CycleSource { ident , location_kind : location_id , } ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleComplete < 'a , TickCycleMarker > for Stream < T , Tick < L > , Bounded > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (self . ir_node . into_inner ()) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > CycleCollection < 'a , ForwardRefMarker > for Stream < T , L , B > { type Location = L ; fn create_source (ident : syn :: Ident , location : L) -> Self { let location_id = location . id () ; Stream :: new (location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: CycleSource { ident , location_kind : location_id , })) ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > CycleComplete < 'a , ForwardRefMarker > for Stream < T , L , B > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Stream < T , L , B > { pub (crate) fn new (location : L , ir_node : HfPlusNode) -> Self { Stream { location , ir_node : RefCell :: new (ir_node) , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < 'a , T : Clone , L : Location < 'a > , B > Clone for Stream < T , L , B > { fn clone (& self) -> Self { if ! matches ! (self . ir_node . borrow () . deref () , HfPlusNode :: Tee { .. }) { let orig_ir_node = self . ir_node . replace (HfPlusNode :: Placeholder) ; * self . ir_node . borrow_mut () = HfPlusNode :: Tee { inner : TeeNode (Rc :: new (RefCell :: new (orig_ir_node))) , } ; } if let HfPlusNode :: Tee { inner } = self . ir_node . borrow () . deref () { Stream { location : self . location . clone () , ir_node : HfPlusNode :: Tee { inner : TeeNode (inner . 0 . clone ()) , } . into () , _phantom : PhantomData , } } else { unreachable ! () } } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Stream < T , L , B > { pub fn map < U , F : Fn (T) -> U + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Stream < U , L , B > { Stream :: new (self . location , HfPlusNode :: Map { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn cloned (self) -> Stream < T , L , B > where T : Clone , { self . map (q ! (| d | d . clone ())) } pub fn flat_map < U , I : IntoIterator < Item = U > , F : Fn (T) -> I + 'a > (self , f : impl IntoQuotedMut < 'a , F > ,) -> Stream < U , L , B > { Stream :: new (self . location , HfPlusNode :: FlatMap { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn flatten < U > (self) -> Stream < U , L , B > where T : IntoIterator < Item = U > , { self . flat_map (q ! (| d | d)) } pub fn filter < F : Fn (& T) -> bool + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Stream < T , L , B > { Stream :: new (self . location , HfPlusNode :: Filter { f : f . splice_fn1_borrow () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn filter_map < U , F : Fn (T) -> Option < U > + 'a > (self , f : impl IntoQuotedMut < 'a , F > ,) -> Stream < U , L , B > { Stream :: new (self . location , HfPlusNode :: FilterMap { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn cross_singleton < O > (self , other : impl Into < Optional < O , L , Bounded > > ,) -> Stream < (T , O) , L , B > where O : Clone , { let other : Optional < O , L , Bounded > = other . into () ; check_matching_location (& self . location , & other . location) ; Stream :: new (self . location , HfPlusNode :: CrossSingleton (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } # [doc = " Allow this stream through if the other stream has elements, otherwise the output is empty."] pub fn continue_if < U > (self , signal : Optional < U , L , Bounded >) -> Stream < T , L , B > { self . cross_singleton (signal . map (q ! (| _u | ()))) . map (q ! (| (d , _signal) | d)) } # [doc = " Allow this stream through if the other stream is empty, otherwise the output is empty."] pub fn continue_unless < U > (self , other : Optional < U , L , Bounded >) -> Stream < T , L , B > { self . continue_if (other . into_stream () . count () . filter (q ! (| c | * c == 0))) } pub fn cross_product < O > (self , other : Stream < O , L , B >) -> Stream < (T , O) , L , B > where T : Clone , O : Clone , { check_matching_location (& self . location , & other . location) ; Stream :: new (self . location , HfPlusNode :: CrossProduct (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } pub fn union (self , other : Stream < T , L , B >) -> Stream < T , L , B > { check_matching_location (& self . location , & other . location) ; Stream :: new (self . location , HfPlusNode :: Union (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } pub fn enumerate (self) -> Stream < (usize , T) , L , B > { Stream :: new (self . location , HfPlusNode :: Enumerate (Box :: new (self . ir_node . into_inner ())) ,) } pub fn unique (self) -> Stream < T , L , B > where T : Eq + Hash , { Stream :: new (self . location , HfPlusNode :: Unique (Box :: new (self . ir_node . into_inner ())) ,) } pub fn filter_not_in (self , other : Stream < T , L , Bounded >) -> Stream < T , L , Bounded > where T : Eq + Hash , { check_matching_location (& self . location , & other . location) ; Stream :: new (self . location , HfPlusNode :: Difference (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } pub fn first (self) -> Optional < T , L , Bounded > { Optional :: new (self . location , self . ir_node . into_inner ()) } pub fn inspect < F : Fn (& T) + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Stream < T , L , B > { if L :: is_top_level () { Stream :: new (self . location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: Inspect { f : f . splice_fn1_borrow () . into () , input : Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , })) ,) } else { Stream :: new (self . location , HfPlusNode :: Inspect { f : f . splice_fn1_borrow () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } } pub fn fold < A , I : Fn () -> A + 'a , F : Fn (& mut A , T) > (self , init : impl IntoQuotedMut < 'a , I > , comb : impl IntoQuotedMut < 'a , F > ,) -> Singleton < A , L , B > { let mut core = HfPlusNode :: Fold { init : init . splice_fn0 () . into () , acc : comb . splice_fn2_borrow_mut () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ; if L :: is_top_level () { core = HfPlusNode :: Persist (Box :: new (core)) ; } Singleton :: new (self . location , core) } pub fn reduce < F : Fn (& mut T , T) + 'a > (self , comb : impl IntoQuotedMut < 'a , F > ,) -> Optional < T , L , B > { let mut core = HfPlusNode :: Reduce { f : comb . splice_fn2_borrow_mut () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ; if L :: is_top_level () { core = HfPlusNode :: Persist (Box :: new (core)) ; } Optional :: new (self . location , core) } pub fn max (self) -> Optional < T , L , B > where T : Ord , { self . reduce (q ! (| curr , new | { if new > * curr { * curr = new ; } })) } pub fn min (self) -> Optional < T , L , B > where T : Ord , { self . reduce (q ! (| curr , new | { if new < * curr { * curr = new ; } })) } pub fn count (self) -> Singleton < usize , L , B > { self . fold (q ! (|| 0usize) , q ! (| count , _ | * count += 1)) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > Stream < T , L , Bounded > { pub fn sort (self) -> Stream < T , L , Bounded > where T : Ord , { Stream :: new (self . location , HfPlusNode :: Sort (Box :: new (self . ir_node . into_inner ())) ,) } } # [cfg (stageleft_macro)] impl < 'a , K , V1 , L : Location < 'a > , B > Stream < (K , V1) , L , B > { pub fn join < V2 > (self , n : Stream < (K , V2) , L , B >) -> Stream < (K , (V1 , V2)) , L , B > where K : Eq + Hash , { check_matching_location (& self . location , & n . location) ; Stream :: new (self . location , HfPlusNode :: Join (Box :: new (self . ir_node . into_inner ()) , Box :: new (n . ir_node . into_inner ()) ,) ,) } pub fn anti_join (self , n : Stream < K , L , Bounded >) -> Stream < (K , V1) , L , B > where K : Eq + Hash , { check_matching_location (& self . location , & n . location) ; Stream :: new (self . location , HfPlusNode :: AntiJoin (Box :: new (self . ir_node . into_inner ()) , Box :: new (n . ir_node . into_inner ()) ,) ,) } } # [cfg (stageleft_macro)] impl < 'a , K : Eq + Hash , V , L : Location < 'a > > Stream < (K , V) , Tick < L > , Bounded > { pub fn fold_keyed < A , I : Fn () -> A + 'a , F : Fn (& mut A , V) + 'a > (self , init : impl IntoQuotedMut < 'a , I > , comb : impl IntoQuotedMut < 'a , F > ,) -> Stream < (K , A) , Tick < L > , Bounded > { Stream :: new (self . location , HfPlusNode :: FoldKeyed { init : init . splice_fn0 () . into () , acc : comb . splice_fn2_borrow_mut () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn reduce_keyed < F : Fn (& mut V , V) + 'a > (self , comb : impl IntoQuotedMut < 'a , F > ,) -> Stream < (K , V) , Tick < L > , Bounded > { Stream :: new (self . location , HfPlusNode :: ReduceKeyed { f : comb . splice_fn2_borrow_mut () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > Stream < T , L , B > { pub fn tick_batch (self , tick : & Tick < L >) -> Stream < T , Tick < L > , Bounded > { Stream :: new (tick . clone () , HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn tick_prefix (self , tick : & Tick < L >) -> Stream < T , Tick < L > , Bounded > where T : Clone , { self . tick_batch (tick) . persist () } pub fn sample_every (self , interval : impl Quoted < 'a , std :: time :: Duration > + Copy + 'a ,) -> Stream < T , L , Unbounded > { let samples = self . location . source_interval (interval) ; let tick = self . location . tick () ; self . tick_batch (& tick) . continue_if (samples . tick_batch (& tick) . first ()) . all_ticks () } pub fn for_each < F : Fn (T) + 'a > (self , f : impl IntoQuotedMut < 'a , F >) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: ForEach { input : Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , f : f . splice_fn1 () . into () , }) ; } pub fn dest_sink < S : Unpin + Sink < T > + 'a > (self , sink : impl Quoted < 'a , S >) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: DestSink { sink : sink . splice_typed () . into () , input : Box :: new (self . ir_node . into_inner ()) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > Stream < T , Tick < L > , Bounded > { pub fn all_ticks (self) -> Stream < T , L , Unbounded > { Stream :: new (self . location . outer () . clone () , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn persist (self) -> Stream < T , Tick < L > , Bounded > where T : Clone , { Stream :: new (self . location , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn defer_tick (self) -> Stream < T , Tick < L > , Bounded > { Stream :: new (self . location , HfPlusNode :: DeferTick (Box :: new (self . ir_node . into_inner ())) ,) } pub fn delta (self) -> Stream < T , Tick < L > , Bounded > { Stream :: new (self . location , HfPlusNode :: Delta (Box :: new (self . ir_node . into_inner ())) ,) } } pub fn serialize_bincode < T : Serialize > (is_demux : bool) -> Pipeline { let root = get_this_crate () ; let t_type : syn :: Type = stageleft :: quote_type :: < T > () ; if is_demux { parse_quote ! { map (| (id , data) : (# root :: ClusterId < _ >, # t_type) | { (id . raw_id , # root :: runtime_support :: bincode :: serialize ::<# t_type > (& data) . unwrap () . into ()) }) } } else { parse_quote ! { map (| data | { # root :: runtime_support :: bincode :: serialize ::<# t_type > (& data) . unwrap () . into () }) } } } pub fn deserialize_bincode < T : DeserializeOwned > (tagged : Option < syn :: Type >) -> Pipeline { let root = get_this_crate () ; let t_type : syn :: Type = stageleft :: quote_type :: < T > () ; if let Some (c_type) = tagged { parse_quote ! { map (| res | { let (id , b) = res . unwrap () ; (# root :: ClusterId ::<# c_type >:: from_raw (id) , # root :: runtime_support :: bincode :: deserialize ::<# t_type > (& b) . unwrap ()) }) } } else { parse_quote ! { map (| res | { # root :: runtime_support :: bincode :: deserialize ::<# t_type > (& res . unwrap ()) . unwrap () }) } } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > Stream < T , L , B > { pub fn decouple_process < P2 > (self , other : & Process < 'a , P2 > ,) -> Stream < T , Process < 'a , P2 > , Unbounded > where L : CanSend < 'a , Process < 'a , P2 > , In < T > = T , Out < T > = T > , T : Clone + Serialize + DeserializeOwned , { self . send_bincode :: < Process < 'a , P2 > , T > (other) } pub fn decouple_cluster < C2 , Tag > (self , other : & Cluster < 'a , C2 > ,) -> Stream < T , Cluster < 'a , C2 > , Unbounded > where L : CanSend < 'a , Cluster < 'a , C2 > , In < T > = (ClusterId < C2 > , T) , Out < T > = (Tag , T) > , T : Clone + Serialize + DeserializeOwned , { let self_node_id = match self . location_kind () { LocationId :: Cluster (cluster_id) => ClusterSelfId { id : cluster_id , _phantom : PhantomData , } , _ => panic ! ("decouple_cluster must be called on a cluster") , } ; self . map (q ! (move | b | (self_node_id , b . clone ()))) . send_bincode_interleaved (other) } pub fn send_bincode < L2 : Location < 'a > , CoreType > (self , other : & L2 ,) -> Stream < L :: Out < CoreType > , L2 , Unbounded > where L : CanSend < 'a , L2 , In < CoreType > = T > , CoreType : Serialize + DeserializeOwned , { let serialize_pipeline = Some (serialize_bincode :: < CoreType > (L :: is_demux ())) ; let deserialize_pipeline = Some (deserialize_bincode :: < CoreType > (L :: tagged_type ())) ; Stream :: new (other . clone () , HfPlusNode :: Network { from_location : self . location_kind () , from_key : None , to_location : other . id () , to_key : None , serialize_pipeline , instantiate_fn : DebugInstantiate :: Building () , deserialize_pipeline , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn send_bincode_external < L2 : 'a , CoreType > (self , other : & ExternalProcess < L2 > ,) -> ExternalBincodeStream < L :: Out < CoreType > > where L : CanSend < 'a , ExternalProcess < 'a , L2 > , In < CoreType > = T , Out < CoreType > = CoreType > , CoreType : Serialize + DeserializeOwned , { let serialize_pipeline = Some (serialize_bincode :: < CoreType > (L :: is_demux ())) ; let mut flow_state_borrow = self . location . flow_state () . borrow_mut () ; let external_key = flow_state_borrow . next_external_out ; flow_state_borrow . next_external_out += 1 ; let leaves = flow_state_borrow . leaves . as_mut () . expect ("Attempted to add a leaf to a flow that has already been finalized. No leaves can be added after the flow has been compiled()") ; let dummy_f : syn :: Expr = syn :: parse_quote ! (()) ; leaves . push (HfPlusLeaf :: ForEach { f : dummy_f . into () , input : Box :: new (HfPlusNode :: Network { from_location : self . location_kind () , from_key : None , to_location : other . id () , to_key : Some (external_key) , serialize_pipeline , instantiate_fn : DebugInstantiate :: Building () , deserialize_pipeline : None , input : Box :: new (self . ir_node . into_inner ()) , }) , }) ; ExternalBincodeStream { process_id : other . id , port_id : external_key , _phantom : PhantomData , } } pub fn send_bytes < L2 : Location < 'a > > (self , other : & L2) -> Stream < L :: Out < Bytes > , L2 , Unbounded > where L : CanSend < 'a , L2 , In < Bytes > = T > , { let root = get_this_crate () ; Stream :: new (other . clone () , HfPlusNode :: Network { from_location : self . location_kind () , from_key : None , to_location : other . id () , to_key : None , serialize_pipeline : None , instantiate_fn : DebugInstantiate :: Building () , deserialize_pipeline : if let Some (c_type) = L :: tagged_type () { Some (parse_quote ! (map (| (id , b) | (# root :: ClusterId <# c_type >:: from_raw (id) , b . unwrap () . freeze ()))) ,) } else { Some (parse_quote ! (map (| b | b . unwrap () . freeze ()))) } , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn send_bytes_external < L2 : 'a > (self , other : & ExternalProcess < L2 >) -> ExternalBytesPort where L : CanSend < 'a , ExternalProcess < 'a , L2 > , In < Bytes > = T , Out < Bytes > = Bytes > , { let mut flow_state_borrow = self . location . flow_state () . borrow_mut () ; let external_key = flow_state_borrow . next_external_out ; flow_state_borrow . next_external_out += 1 ; let leaves = flow_state_borrow . leaves . as_mut () . expect ("Attempted to add a leaf to a flow that has already been finalized. No leaves can be added after the flow has been compiled()") ; let dummy_f : syn :: Expr = syn :: parse_quote ! (()) ; leaves . push (HfPlusLeaf :: ForEach { f : dummy_f . into () , input : Box :: new (HfPlusNode :: Network { from_location : self . location_kind () , from_key : None , to_location : other . id () , to_key : Some (external_key) , serialize_pipeline : None , instantiate_fn : DebugInstantiate :: Building () , deserialize_pipeline : None , input : Box :: new (self . ir_node . into_inner ()) , }) , }) ; ExternalBytesPort { process_id : other . id , port_id : external_key , } } pub fn send_bincode_interleaved < L2 : Location < 'a > , Tag , CoreType > (self , other : & L2 ,) -> Stream < CoreType , L2 , Unbounded > where L : CanSend < 'a , L2 , In < CoreType > = T , Out < CoreType > = (Tag , CoreType) > , CoreType : Serialize + DeserializeOwned , { self . send_bincode :: < L2 , CoreType > (other) . map (q ! (| (_ , b) | b)) } pub fn send_bytes_interleaved < L2 : Location < 'a > , Tag > (self , other : & L2 ,) -> Stream < Bytes , L2 , Unbounded > where L : CanSend < 'a , L2 , In < Bytes > = T , Out < Bytes > = (Tag , Bytes) > , { self . send_bytes :: < L2 > (other) . map (q ! (| (_ , b) | b)) } pub fn broadcast_bincode < C2 > (self , other : & Cluster < 'a , C2 > ,) -> Stream < L :: Out < T > , Cluster < 'a , C2 > , Unbounded > where L : CanSend < 'a , Cluster < 'a , C2 > , In < T > = (ClusterId < C2 > , T) > , T : Clone + Serialize + DeserializeOwned , { let ids = other . members () ; self . flat_map (q ! (| b | ids . iter () . map (move | id | (:: std :: clone :: Clone :: clone (id) , :: std :: clone :: Clone :: clone (& b))))) . send_bincode (other) } pub fn broadcast_bincode_interleaved < C2 , Tag > (self , other : & Cluster < 'a , C2 > ,) -> Stream < T , Cluster < 'a , C2 > , Unbounded > where L : CanSend < 'a , Cluster < 'a , C2 > , In < T > = (ClusterId < C2 > , T) , Out < T > = (Tag , T) > + 'a , T : Clone + Serialize + DeserializeOwned , { self . broadcast_bincode (other) . map (q ! (| (_ , b) | b)) } pub fn broadcast_bytes < C2 > (self , other : & Cluster < 'a , C2 > ,) -> Stream < L :: Out < Bytes > , Cluster < 'a , C2 > , Unbounded > where L : CanSend < 'a , Cluster < 'a , C2 > , In < Bytes > = (ClusterId < C2 > , T) > + 'a , T : Clone , { let ids = other . members () ; self . flat_map (q ! (| b | ids . iter () . map (move | id | (:: std :: clone :: Clone :: clone (id) , :: std :: clone :: Clone :: clone (& b))))) . send_bytes (other) } pub fn broadcast_bytes_interleaved < C2 , Tag > (self , other : & Cluster < 'a , C2 > ,) -> Stream < Bytes , Cluster < 'a , C2 > , Unbounded > where L : CanSend < 'a , Cluster < 'a , C2 > , In < Bytes > = (ClusterId < C2 > , T) , Out < Bytes > = (Tag , Bytes) > + 'a , T : Clone , { self . broadcast_bytes (other) . map (q ! (| (_ , b) | b)) } } # [cfg (stageleft_macro)] pub mod tests { pub use hydro_deploy :: Deployment ; pub use hydroflow :: futures :: StreamExt ; pub use serde :: { Deserialize , Serialize } ; pub use stageleft :: q ; pub use crate :: __staged :: location :: Location ; pub use crate :: __staged :: FlowBuilder ; pub struct P1 { } pub struct P2 { } # [derive (Serialize , Deserialize , Debug)] pub struct SendOverNetwork { pub n : u32 , } # [tokio :: test] pub async fn first_ten_distributed () { let mut deployment = Deployment :: new () ; let flow = FlowBuilder :: new () ; let first_node = flow . process :: < P1 > () ; let second_node = flow . process :: < P2 > () ; let external = flow . external_process :: < P2 > () ; let numbers = first_node . source_iter (q ! (0 .. 10)) ; let out_port = numbers . map (q ! (| n | SendOverNetwork { n })) . send_bincode (& second_node) . send_bincode_external (& external) ; let nodes = flow . with_process (& first_node , deployment . Localhost ()) . with_process (& second_node , deployment . Localhost ()) . with_external (& external , deployment . Localhost ()) . deploy (& mut deployment) ; deployment . deploy () . await . unwrap () ; let mut external_out = nodes . connect_source_bincode (out_port) . await ; deployment . start () . await . unwrap () ; for i in 0 .. 10 { assert_eq ! (external_out . next () . await . unwrap () . n , i) ; } } } } pub use stream :: { Bounded , Stream , Unbounded } ; pub mod singleton { pub use std :: cell :: RefCell ; pub use std :: marker :: PhantomData ; pub use std :: ops :: Deref ; pub use std :: rc :: Rc ; pub use stageleft :: { q , IntoQuotedMut , Quoted } ; pub use crate :: __staged :: builder :: FLOW_USED_MESSAGE ; pub use crate :: __staged :: cycle :: { CycleCollection , CycleCollectionWithInitial , CycleComplete , DeferTick , ForwardRefMarker , TickCycleMarker , } ; pub use crate :: __staged :: ir :: { HfPlusLeaf , HfPlusNode , TeeNode } ; pub use crate :: __staged :: location :: { check_matching_location , Location , LocationId , NoTick , Tick } ; pub use crate :: __staged :: stream :: { Bounded , Unbounded } ; pub use crate :: __staged :: { Optional , Stream } ; pub use crate :: singleton :: Singleton ; # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Singleton < T , L , B > { pub (crate) fn new (location : L , ir_node : HfPlusNode) -> Self { Singleton { location , ir_node : RefCell :: new (ir_node) , _phantom : PhantomData , } } fn location_kind (& self) -> LocationId { self . location . id () } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > From < Singleton < T , L , Bounded > > for Singleton < T , L , Unbounded > { fn from (singleton : Singleton < T , L , Bounded >) -> Self { Singleton :: new (singleton . location , singleton . ir_node . into_inner ()) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > DeferTick for Singleton < T , Tick < L > , Bounded > { fn defer_tick (self) -> Self { Singleton :: defer_tick (self) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleCollectionWithInitial < 'a , TickCycleMarker > for Singleton < T , Tick < L > , Bounded > { type Location = Tick < L > ; fn create_source (ident : syn :: Ident , initial : Self , location : Tick < L >) -> Self { let location_id = location . id () ; Singleton :: new (location , HfPlusNode :: Union (Box :: new (HfPlusNode :: CycleSource { ident , location_kind : location_id , }) , initial . ir_node . into_inner () . into () ,) ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleComplete < 'a , TickCycleMarker > for Singleton < T , Tick < L > , Bounded > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (self . ir_node . into_inner ()) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleCollection < 'a , ForwardRefMarker > for Singleton < T , Tick < L > , Bounded > { type Location = Tick < L > ; fn create_source (ident : syn :: Ident , location : Tick < L >) -> Self { let location_id = location . id () ; Singleton :: new (location , HfPlusNode :: CycleSource { ident , location_kind : location_id , } ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleComplete < 'a , ForwardRefMarker > for Singleton < T , Tick < L > , Bounded > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (self . ir_node . into_inner ()) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T : Clone , L : Location < 'a > , B > Clone for Singleton < T , L , B > { fn clone (& self) -> Self { if ! matches ! (self . ir_node . borrow () . deref () , HfPlusNode :: Tee { .. }) { let orig_ir_node = self . ir_node . replace (HfPlusNode :: Placeholder) ; * self . ir_node . borrow_mut () = HfPlusNode :: Tee { inner : TeeNode (Rc :: new (RefCell :: new (orig_ir_node))) , } ; } if let HfPlusNode :: Tee { inner } = self . ir_node . borrow () . deref () { Singleton { location : self . location . clone () , ir_node : HfPlusNode :: Tee { inner : TeeNode (inner . 0 . clone ()) , } . into () , _phantom : PhantomData , } } else { unreachable ! () } } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Singleton < T , L , B > { pub fn into_stream (self) -> Stream < T , L , Bounded > { Stream :: new (self . location , self . ir_node . into_inner ()) } pub fn map < U , F : Fn (T) -> U + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Singleton < U , L , B > { Singleton :: new (self . location , HfPlusNode :: Map { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn flat_map < U , I : IntoIterator < Item = U > , F : Fn (T) -> I + 'a > (self , f : impl IntoQuotedMut < 'a , F > ,) -> Stream < U , L , B > { Stream :: new (self . location , HfPlusNode :: FlatMap { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn filter < F : Fn (& T) -> bool + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Optional < T , L , B > { Optional :: new (self . location , HfPlusNode :: Filter { f : f . splice_fn1_borrow () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn filter_map < U , F : Fn (T) -> Option < U > + 'a > (self , f : impl IntoQuotedMut < 'a , F > ,) -> Optional < U , L , B > { Optional :: new (self . location , HfPlusNode :: FilterMap { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn zip < Other > (self , other : Other) -> < Self as ZipResult < 'a , Other > > :: Out where Self : ZipResult < 'a , Other , Location = L > , { check_matching_location (& self . location , & Self :: other_location (& other)) ; if L :: is_top_level () { Self :: make (self . location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: CrossSingleton (Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , Box :: new (HfPlusNode :: Unpersist (Box :: new (Self :: other_ir_node (other)))) ,))) ,) } else { Self :: make (self . location , HfPlusNode :: CrossSingleton (Box :: new (self . ir_node . into_inner ()) , Box :: new (Self :: other_ir_node (other)) ,) ,) } } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > Singleton < T , L , Bounded > { pub fn continue_if < U > (self , signal : Optional < U , L , Bounded >) -> Optional < T , L , Bounded > { self . zip (signal . map (q ! (| _u | ()))) . map (q ! (| (d , _signal) | d)) } pub fn continue_unless < U > (self , other : Optional < U , L , Bounded >) -> Optional < T , L , Bounded > { self . continue_if (other . into_stream () . count () . filter (q ! (| c | * c == 0))) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > Singleton < T , L , B > { pub fn latest_tick (self , tick : & Tick < L >) -> Singleton < T , Tick < L > , Bounded > { Singleton :: new (tick . clone () , HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn tick_samples (self) -> Stream < T , L , Unbounded > { let tick = self . location . tick () ; self . latest_tick (& tick) . all_ticks () } pub fn sample_every (self , interval : impl Quoted < 'a , std :: time :: Duration > + Copy + 'a ,) -> Stream < T , L , Unbounded > { let samples = self . location . source_interval (interval) ; let tick = self . location . tick () ; self . latest_tick (& tick) . continue_if (samples . tick_batch (& tick) . first ()) . all_ticks () } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > Singleton < T , Tick < L > , Bounded > { pub fn all_ticks (self) -> Stream < T , L , Unbounded > { Stream :: new (self . location . outer () . clone () , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn latest (self) -> Singleton < T , L , Unbounded > { Singleton :: new (self . location . outer () . clone () , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn defer_tick (self) -> Singleton < T , Tick < L > , Bounded > { Singleton :: new (self . location , HfPlusNode :: DeferTick (Box :: new (self . ir_node . into_inner ())) ,) } pub fn persist (self) -> Stream < T , Tick < L > , Bounded > { Stream :: new (self . location , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn delta (self) -> Optional < T , Tick < L > , Bounded > { Optional :: new (self . location , HfPlusNode :: Delta (Box :: new (self . ir_node . into_inner ())) ,) } } pub use crate :: singleton :: ZipResult ; # [cfg (stageleft_macro)] impl < 'a , T , U : Clone , L : Location < 'a > , B > ZipResult < 'a , Singleton < U , L , B > > for Singleton < T , L , B > { type Out = Singleton < (T , U) , L , B > ; type Location = L ; fn other_location (other : & Singleton < U , L , B >) -> L { other . location . clone () } fn other_ir_node (other : Singleton < U , L , B >) -> HfPlusNode { other . ir_node . into_inner () } fn make (location : L , ir_node : HfPlusNode) -> Self :: Out { Singleton :: new (location , ir_node) } } # [cfg (stageleft_macro)] impl < 'a , T , U : Clone , L : Location < 'a > , B > ZipResult < 'a , Optional < U , L , B > > for Singleton < T , L , B > { type Out = Optional < (T , U) , L , B > ; type Location = L ; fn other_location (other : & Optional < U , L , B >) -> L { other . location . clone () } fn other_ir_node (other : Optional < U , L , B >) -> HfPlusNode { other . ir_node . into_inner () } fn make (location : L , ir_node : HfPlusNode) -> Self :: Out { Optional :: new (location , ir_node) } } } pub use singleton :: Singleton ; pub mod optional { pub use std :: cell :: RefCell ; pub use std :: marker :: PhantomData ; pub use std :: ops :: Deref ; pub use std :: rc :: Rc ; pub use stageleft :: { q , IntoQuotedMut , Quoted } ; pub use syn :: parse_quote ; pub use crate :: __staged :: builder :: FLOW_USED_MESSAGE ; pub use crate :: __staged :: cycle :: { CycleCollection , CycleComplete , DeferTick , ForwardRefMarker , TickCycleMarker } ; pub use crate :: __staged :: ir :: { HfPlusLeaf , HfPlusNode , HfPlusSource , TeeNode } ; pub use crate :: __staged :: location :: { check_matching_location , LocationId , NoTick } ; pub use crate :: __staged :: { Bounded , Location , Singleton , Stream , Tick , Unbounded } ; pub use crate :: optional :: Optional ; # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Optional < T , L , B > { pub (crate) fn new (location : L , ir_node : HfPlusNode) -> Self { Optional { location , ir_node : RefCell :: new (ir_node) , _phantom : PhantomData , } } pub fn some (singleton : Singleton < T , L , B >) -> Self { Optional :: new (singleton . location , singleton . ir_node . into_inner ()) } fn location_kind (& self) -> LocationId { self . location . id () } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > DeferTick for Optional < T , Tick < L > , Bounded > { fn defer_tick (self) -> Self { Optional :: defer_tick (self) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleCollection < 'a , TickCycleMarker > for Optional < T , Tick < L > , Bounded > { type Location = Tick < L > ; fn create_source (ident : syn :: Ident , location : Tick < L >) -> Self { let location_id = location . id () ; Optional :: new (location , HfPlusNode :: CycleSource { ident , location_kind : location_id , } ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleComplete < 'a , TickCycleMarker > for Optional < T , Tick < L > , Bounded > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (self . ir_node . into_inner ()) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleCollection < 'a , ForwardRefMarker > for Optional < T , Tick < L > , Bounded > { type Location = Tick < L > ; fn create_source (ident : syn :: Ident , location : Tick < L >) -> Self { let location_id = location . id () ; Optional :: new (location , HfPlusNode :: CycleSource { ident , location_kind : location_id , } ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleComplete < 'a , ForwardRefMarker > for Optional < T , Tick < L > , Bounded > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (self . ir_node . into_inner ()) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > CycleCollection < 'a , ForwardRefMarker > for Optional < T , L , B > { type Location = L ; fn create_source (ident : syn :: Ident , location : L) -> Self { let location_id = location . id () ; Optional :: new (location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: CycleSource { ident , location_kind : location_id , })) ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > CycleComplete < 'a , ForwardRefMarker > for Optional < T , L , B > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > From < Optional < T , L , Bounded > > for Optional < T , L , Unbounded > { fn from (singleton : Optional < T , L , Bounded >) -> Self { Optional :: new (singleton . location , singleton . ir_node . into_inner ()) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > From < Singleton < T , L , B > > for Optional < T , L , B > { fn from (singleton : Singleton < T , L , B >) -> Self { Optional :: some (singleton) } } # [cfg (stageleft_macro)] impl < 'a , T : Clone , L : Location < 'a > , B > Clone for Optional < T , L , B > { fn clone (& self) -> Self { if ! matches ! (self . ir_node . borrow () . deref () , HfPlusNode :: Tee { .. }) { let orig_ir_node = self . ir_node . replace (HfPlusNode :: Placeholder) ; * self . ir_node . borrow_mut () = HfPlusNode :: Tee { inner : TeeNode (Rc :: new (RefCell :: new (orig_ir_node))) , } ; } if let HfPlusNode :: Tee { inner } = self . ir_node . borrow () . deref () { Optional { location : self . location . clone () , ir_node : HfPlusNode :: Tee { inner : TeeNode (inner . 0 . clone ()) , } . into () , _phantom : PhantomData , } } else { unreachable ! () } } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Optional < T , L , B > { pub fn into_stream (self) -> Stream < T , L , B > { if L :: is_top_level () { panic ! ("Converting an optional to a stream is not yet supported at the top level") ; } Stream :: new (self . location , self . ir_node . into_inner ()) } pub fn map < U , F : Fn (T) -> U + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Optional < U , L , B > { Optional :: new (self . location , HfPlusNode :: Map { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn flat_map < U , I : IntoIterator < Item = U > , F : Fn (T) -> I + 'a > (self , f : impl IntoQuotedMut < 'a , F > ,) -> Stream < U , L , B > { Stream :: new (self . location , HfPlusNode :: FlatMap { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn filter < F : Fn (& T) -> bool + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Optional < T , L , B > { Optional :: new (self . location , HfPlusNode :: Filter { f : f . splice_fn1_borrow () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn filter_map < U , F : Fn (T) -> Option < U > + 'a > (self , f : impl IntoQuotedMut < 'a , F > ,) -> Optional < U , L , B > { Optional :: new (self . location , HfPlusNode :: FilterMap { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn union (self , other : Optional < T , L , B >) -> Optional < T , L , B > { check_matching_location (& self . location , & other . location) ; if L :: is_top_level () { Optional :: new (self . location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: Union (Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , Box :: new (HfPlusNode :: Unpersist (Box :: new (other . ir_node . into_inner ()))) ,))) ,) } else { Optional :: new (self . location , HfPlusNode :: Union (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } } pub fn zip < O > (self , other : impl Into < Optional < O , L , B > >) -> Optional < (T , O) , L , B > where O : Clone , { let other : Optional < O , L , B > = other . into () ; check_matching_location (& self . location , & other . location) ; if L :: is_top_level () { Optional :: new (self . location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: CrossSingleton (Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , Box :: new (HfPlusNode :: Unpersist (Box :: new (other . ir_node . into_inner ()))) ,))) ,) } else { Optional :: new (self . location , HfPlusNode :: CrossSingleton (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } } pub fn unwrap_or (self , other : Singleton < T , L , B >) -> Singleton < T , L , B > { check_matching_location (& self . location , & other . location) ; if L :: is_top_level () { Singleton :: new (self . location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: Union (Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , Box :: new (HfPlusNode :: Unpersist (Box :: new (other . ir_node . into_inner ()))) ,))) ,) } else { Singleton :: new (self . location , HfPlusNode :: Union (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } } pub fn into_singleton (self) -> Singleton < Option < T > , L , B > where T : Clone , { let none : syn :: Expr = parse_quote ! ([:: std :: option :: Option :: None]) ; let core_ir = HfPlusNode :: Persist (Box :: new (HfPlusNode :: Source { source : HfPlusSource :: Iter (none . into ()) , location_kind : self . location . id () . root () . clone () , })) ; let none_singleton = if L :: is_top_level () { Singleton :: new (self . location . clone () , HfPlusNode :: Persist (Box :: new (core_ir)) ,) } else { Singleton :: new (self . location . clone () , core_ir) } ; self . map (q ! (| v | Some (v))) . unwrap_or (none_singleton) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > Optional < T , L , Bounded > { pub fn continue_if < U > (self , signal : Optional < U , L , Bounded >) -> Optional < T , L , Bounded > { self . zip (signal . map (q ! (| _u | ()))) . map (q ! (| (d , _signal) | d)) } pub fn continue_unless < U > (self , other : Optional < U , L , Bounded >) -> Optional < T , L , Bounded > { self . continue_if (other . into_stream () . count () . filter (q ! (| c | * c == 0))) } pub fn then < U > (self , value : Singleton < U , L , Bounded >) -> Optional < U , L , Bounded > { value . continue_if (self) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > Optional < T , L , B > { pub fn latest_tick (self , tick : & Tick < L >) -> Optional < T , Tick < L > , Bounded > { Optional :: new (tick . clone () , HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn tick_samples (self) -> Stream < T , L , Unbounded > { let tick = self . location . tick () ; self . latest_tick (& tick) . all_ticks () } pub fn sample_every (self , interval : impl Quoted < 'a , std :: time :: Duration > + Copy + 'a ,) -> Stream < T , L , Unbounded > { let samples = self . location . source_interval (interval) ; let tick = self . location . tick () ; self . latest_tick (& tick) . continue_if (samples . tick_batch (& tick) . first ()) . all_ticks () } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > Optional < T , Tick < L > , Bounded > { pub fn all_ticks (self) -> Stream < T , L , Unbounded > { Stream :: new (self . location . outer () . clone () , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn latest (self) -> Optional < T , L , Unbounded > { Optional :: new (self . location . outer () . clone () , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn defer_tick (self) -> Optional < T , Tick < L > , Bounded > { Optional :: new (self . location , HfPlusNode :: DeferTick (Box :: new (self . ir_node . into_inner ())) ,) } pub fn persist (self) -> Stream < T , Tick < L > , Bounded > { Stream :: new (self . location , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn delta (self) -> Optional < T , Tick < L > , Bounded > { Optional :: new (self . location , HfPlusNode :: Delta (Box :: new (self . ir_node . into_inner ())) ,) } } } pub use optional :: Optional ; pub mod location { pub use std :: fmt :: Debug ; pub use std :: marker :: PhantomData ; pub use std :: time :: Duration ; pub use hydroflow :: futures :: stream :: Stream as FuturesStream ; pub use hydroflow :: { tokio , tokio_stream } ; pub use proc_macro2 :: Span ; pub use stageleft :: { q , Quoted } ; pub use super :: builder :: FlowState ; pub use crate :: __staged :: cycle :: { CycleCollection , ForwardRef , ForwardRefMarker } ; pub use crate :: __staged :: ir :: { HfPlusNode , HfPlusSource } ; pub use crate :: __staged :: { Singleton , Stream , Unbounded } ; pub mod external_process { pub use std :: marker :: PhantomData ; pub use hydroflow :: bytes :: Bytes ; pub use serde :: de :: DeserializeOwned ; pub use serde :: Serialize ; pub use super :: { Location , LocationId , NoTick } ; pub use crate :: __staged :: builder :: FlowState ; pub use crate :: __staged :: ir :: { HfPlusNode , HfPlusSource } ; pub use crate :: __staged :: { Stream , Unbounded } ; pub use crate :: location :: external_process :: ExternalBytesPort ; pub use crate :: location :: external_process :: ExternalBincodeSink ; pub use crate :: location :: external_process :: ExternalBincodeStream ; pub use crate :: location :: external_process :: ExternalProcess ; # [cfg (stageleft_macro)] impl < P > Clone for ExternalProcess < '_ , P > { fn clone (& self) -> Self { ExternalProcess { id : self . id , flow_state : self . flow_state . clone () , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < 'a , P > Location < 'a > for ExternalProcess < 'a , P > { fn id (& self) -> LocationId { LocationId :: ExternalProcess (self . id) } fn flow_state (& self) -> & FlowState { & self . flow_state } fn is_top_level () -> bool { true } } # [cfg (stageleft_macro)] impl < 'a , P > ExternalProcess < 'a , P > { pub fn source_external_bytes < L : Location < 'a > + NoTick > (& self , to : & L ,) -> (ExternalBytesPort , Stream < Bytes , L , Unbounded >) { let next_external_port_id = { let mut flow_state = self . flow_state . borrow_mut () ; let id = flow_state . next_external_out ; flow_state . next_external_out += 1 ; id } ; (ExternalBytesPort { process_id : self . id , port_id : next_external_port_id , } , Stream :: new (to . clone () , HfPlusNode :: Persist (Box :: new (HfPlusNode :: Network { from_location : LocationId :: ExternalProcess (self . id) , from_key : Some (next_external_port_id) , to_location : to . id () , to_key : None , serialize_pipeline : None , instantiate_fn : crate :: ir :: DebugInstantiate :: Building () , deserialize_pipeline : Some (syn :: parse_quote ! (map (| b | b . unwrap () . freeze ()))) , input : Box :: new (HfPlusNode :: Source { source : HfPlusSource :: ExternalNetwork () , location_kind : LocationId :: ExternalProcess (self . id) , }) , })) ,) ,) } pub fn source_external_bincode < L : Location < 'a > + NoTick , T : Serialize + DeserializeOwned > (& self , to : & L ,) -> (ExternalBincodeSink < T > , Stream < T , L , Unbounded >) { let next_external_port_id = { let mut flow_state = self . flow_state . borrow_mut () ; let id = flow_state . next_external_out ; flow_state . next_external_out += 1 ; id } ; (ExternalBincodeSink { process_id : self . id , port_id : next_external_port_id , _phantom : PhantomData , } , Stream :: new (to . clone () , HfPlusNode :: Persist (Box :: new (HfPlusNode :: Network { from_location : LocationId :: ExternalProcess (self . id) , from_key : Some (next_external_port_id) , to_location : to . id () , to_key : None , serialize_pipeline : None , instantiate_fn : crate :: ir :: DebugInstantiate :: Building () , deserialize_pipeline : Some (crate :: stream :: deserialize_bincode :: < T > (None)) , input : Box :: new (HfPlusNode :: Source { source : HfPlusSource :: ExternalNetwork () , location_kind : LocationId :: ExternalProcess (self . id) , }) , })) ,) ,) } } } pub use external_process :: ExternalProcess ; pub mod process { pub use std :: marker :: PhantomData ; pub use super :: { Location , LocationId } ; pub use crate :: __staged :: builder :: FlowState ; pub use crate :: location :: process :: Process ; # [cfg (stageleft_macro)] impl < P > Clone for Process < '_ , P > { fn clone (& self) -> Self { Process { id : self . id , flow_state : self . flow_state . clone () , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < 'a , P > Location < 'a > for Process < 'a , P > { fn id (& self) -> LocationId { LocationId :: Process (self . id) } fn flow_state (& self) -> & FlowState { & self . flow_state } fn is_top_level () -> bool { true } } } pub use process :: Process ; pub mod cluster { pub use std :: fmt :: { Debug , Display } ; pub use std :: hash :: Hash ; pub use std :: marker :: PhantomData ; pub use proc_macro2 :: { Span , TokenStream } ; pub use quote :: quote ; pub use serde :: { Deserialize , Serialize } ; pub use stageleft :: runtime_support :: FreeVariable ; pub use stageleft :: { quote_type , Quoted } ; pub use super :: { Location , LocationId } ; pub use crate :: __staged :: builder :: FlowState ; pub use crate :: __staged :: staging_util :: get_this_crate ; pub use crate :: location :: cluster :: Cluster ; # [cfg (stageleft_macro)] impl < 'a , C > Cluster < 'a , C > { pub fn self_id (& self) -> impl Quoted < 'a , ClusterId < C > > + Copy + 'a { ClusterSelfId { id : self . id , _phantom : PhantomData , } } pub fn members (& self) -> impl Quoted < 'a , & 'a Vec < ClusterId < C > > > + Copy + 'a { ClusterIds { id : self . id , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < C > Clone for Cluster < '_ , C > { fn clone (& self) -> Self { Cluster { id : self . id , flow_state : self . flow_state . clone () , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < 'a , C > Location < 'a > for Cluster < 'a , C > { fn id (& self) -> LocationId { LocationId :: Cluster (self . id) } fn flow_state (& self) -> & FlowState { & self . flow_state } fn is_top_level () -> bool { true } } pub use crate :: location :: cluster :: ClusterId ; # [cfg (stageleft_macro)] impl < C > Debug for ClusterId < C > { fn fmt (& self , f : & mut std :: fmt :: Formatter < '_ >) -> std :: fmt :: Result { write ! (f , "ClusterId::<{}>({})" , std :: any :: type_name ::< C > () , self . raw_id) } } # [cfg (stageleft_macro)] impl < C > Display for ClusterId < C > { fn fmt (& self , f : & mut std :: fmt :: Formatter < '_ >) -> std :: fmt :: Result { write ! (f , "ClusterId::<{}>({})" , std :: any :: type_name ::< C > () , self . raw_id) } } # [cfg (stageleft_macro)] impl < C > Clone for ClusterId < C > { fn clone (& self) -> Self { * self } } # [cfg (stageleft_macro)] impl < C > Copy for ClusterId < C > { } # [cfg (stageleft_macro)] impl < C > Serialize for ClusterId < C > { fn serialize < S > (& self , serializer : S) -> Result < S :: Ok , S :: Error > where S : serde :: ser :: Serializer , { self . raw_id . serialize (serializer) } } # [cfg (stageleft_macro)] impl < 'de , C > Deserialize < 'de > for ClusterId < C > { fn deserialize < D > (deserializer : D) -> Result < Self , D :: Error > where D : serde :: de :: Deserializer < 'de > , { u32 :: deserialize (deserializer) . map (| id | ClusterId { raw_id : id , _phantom : PhantomData , }) } } # [cfg (stageleft_macro)] impl < C > PartialEq for ClusterId < C > { fn eq (& self , other : & Self) -> bool { self . raw_id == other . raw_id } } # [cfg (stageleft_macro)] impl < C > Eq for ClusterId < C > { } # [cfg (stageleft_macro)] impl < C > PartialOrd for ClusterId < C > { fn partial_cmp (& self , other : & Self) -> Option < std :: cmp :: Ordering > { Some (self . cmp (other)) } } # [cfg (stageleft_macro)] impl < C > Ord for ClusterId < C > { fn cmp (& self , other : & Self) -> std :: cmp :: Ordering { self . raw_id . cmp (& other . raw_id) } } # [cfg (stageleft_macro)] impl < C > Hash for ClusterId < C > { fn hash < H : std :: hash :: Hasher > (& self , state : & mut H) { self . raw_id . hash (state) } } # [cfg (stageleft_macro)] impl < C > ClusterId < C > { pub fn from_raw (id : u32) -> Self { ClusterId { raw_id : id , _phantom : PhantomData , } } } pub use crate :: location :: cluster :: ClusterIds ; # [cfg (stageleft_macro)] impl < C > Clone for ClusterIds < '_ , C > { fn clone (& self) -> Self { * self } } # [cfg (stageleft_macro)] impl < C > Copy for ClusterIds < '_ , C > { } # [cfg (stageleft_macro)] impl < 'a , C > FreeVariable < & 'a Vec < ClusterId < C > > > for ClusterIds < 'a , C > { fn to_tokens (self) -> (Option < TokenStream > , Option < TokenStream >) where Self : Sized , { let ident = syn :: Ident :: new (& format ! ("__hydroflow_plus_cluster_ids_{}" , self . id) , Span :: call_site () ,) ; let root = get_this_crate () ; let c_type = quote_type :: < C > () ; (None , Some (quote ! { unsafe { :: std :: mem :: transmute ::< _ , &:: std :: vec :: Vec <# root :: ClusterId <# c_type >>> (# ident) } } ,) ,) } } # [cfg (stageleft_macro)] impl < 'a , C > Quoted < 'a , & 'a Vec < ClusterId < C > > > for ClusterIds < 'a , C > { } pub use crate :: location :: cluster :: ClusterSelfId ; # [cfg (stageleft_macro)] impl < C > Clone for ClusterSelfId < '_ , C > { fn clone (& self) -> Self { * self } } # [cfg (stageleft_macro)] impl < C > Copy for ClusterSelfId < '_ , C > { } # [cfg (stageleft_macro)] impl < C > FreeVariable < ClusterId < C > > for ClusterSelfId < '_ , C > { fn to_tokens (self) -> (Option < TokenStream > , Option < TokenStream >) where Self : Sized , { let ident = syn :: Ident :: new (& format ! ("__hydroflow_plus_cluster_self_id_{}" , self . id) , Span :: call_site () ,) ; let root = get_this_crate () ; let c_type : syn :: Type = quote_type :: < C > () ; (None , Some (quote ! { # root :: ClusterId ::<# c_type >:: from_raw (# ident) }) ,) } } # [cfg (stageleft_macro)] impl < 'a , C > Quoted < 'a , ClusterId < C > > for ClusterSelfId < 'a , C > { } } pub use cluster :: { Cluster , ClusterId } ; pub mod can_send { pub use stageleft :: quote_type ; pub use super :: { Cluster , ClusterId , ExternalProcess , Location , Process } ; pub use crate :: location :: can_send :: CanSend ; # [cfg (stageleft_macro)] impl < 'a , P1 , P2 > CanSend < 'a , Process < 'a , P2 > > for Process < 'a , P1 > { type In < T > = T ; type Out < T > = T ; fn is_demux () -> bool { false } fn tagged_type () -> Option < syn :: Type > { None } } # [cfg (stageleft_macro)] impl < 'a , P1 , C2 > CanSend < 'a , Cluster < 'a , C2 > > for Process < 'a , P1 > { type In < T > = (ClusterId < C2 > , T) ; type Out < T > = T ; fn is_demux () -> bool { true } fn tagged_type () -> Option < syn :: Type > { None } } # [cfg (stageleft_macro)] impl < 'a , C1 , P2 > CanSend < 'a , Process < 'a , P2 > > for Cluster < 'a , C1 > { type In < T > = T ; type Out < T > = (ClusterId < C1 > , T) ; fn is_demux () -> bool { false } fn tagged_type () -> Option < syn :: Type > { Some (quote_type :: < C1 > ()) } } # [cfg (stageleft_macro)] impl < 'a , C1 , C2 > CanSend < 'a , Cluster < 'a , C2 > > for Cluster < 'a , C1 > { type In < T > = (ClusterId < C2 > , T) ; type Out < T > = (ClusterId < C1 > , T) ; fn is_demux () -> bool { true } fn tagged_type () -> Option < syn :: Type > { Some (quote_type :: < C1 > ()) } } # [cfg (stageleft_macro)] impl < 'a , P1 , E2 > CanSend < 'a , ExternalProcess < 'a , E2 > > for Process < 'a , P1 > { type In < T > = T ; type Out < T > = T ; fn is_demux () -> bool { false } fn tagged_type () -> Option < syn :: Type > { None } } } pub use can_send :: CanSend ; pub mod tick { pub use std :: marker :: PhantomData ; pub use proc_macro2 :: Span ; pub use stageleft :: { q , Quoted } ; pub use super :: { Cluster , Location , LocationId , Process } ; pub use crate :: __staged :: builder :: FlowState ; pub use crate :: __staged :: cycle :: { CycleCollection , CycleCollectionWithInitial , DeferTick , ForwardRef , ForwardRefMarker , TickCycle , TickCycleMarker , } ; pub use crate :: __staged :: ir :: { HfPlusNode , HfPlusSource } ; pub use crate :: __staged :: { Bounded , Optional , Singleton , Stream } ; pub use crate :: location :: tick :: NoTick ; # [cfg (stageleft_macro)] impl < T > NoTick for Process < '_ , T > { } # [cfg (stageleft_macro)] impl < T > NoTick for Cluster < '_ , T > { } pub use crate :: location :: tick :: Tick ; # [cfg (stageleft_macro)] impl < 'a , L : Location < 'a > > Location < 'a > for Tick < L > { fn id (& self) -> LocationId { LocationId :: Tick (self . id , Box :: new (self . l . id ())) } fn flow_state (& self) -> & FlowState { self . l . flow_state () } fn is_top_level () -> bool { false } } # [cfg (stageleft_macro)] impl < 'a , L : Location < 'a > > Tick < L > { pub fn outer (& self) -> & L { & self . l } pub fn spin_batch (& self , batch_size : impl Quoted < 'a , usize > + Copy + 'a ,) -> Stream < () , Self , Bounded > where L : NoTick , { self . l . spin () . flat_map (q ! (move | _ | 0 .. batch_size)) . map (q ! (| _ | ())) . tick_batch (self) } pub fn singleton < T : Clone > (& self , e : impl Quoted < 'a , T >) -> Singleton < T , Self , Bounded > where L : NoTick , { self . outer () . singleton (e) . latest_tick (self) } pub fn singleton_first_tick < T : Clone > (& self , e : impl Quoted < 'a , T > ,) -> Optional < T , Self , Bounded > where L : NoTick , { let e_arr = q ! ([e]) ; let e = e_arr . splice_untyped () ; Optional :: new (self . clone () , HfPlusNode :: Source { source : HfPlusSource :: Iter (e . into ()) , location_kind : self . l . id () , } ,) } pub fn forward_ref < S : CycleCollection < 'a , ForwardRefMarker , Location = Self > > (& self ,) -> (ForwardRef < 'a , S > , S) where L : NoTick , { let next_id = { let on_id = match self . l . id () { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (_) => panic ! () , } ; let mut flow_state = self . flow_state () . borrow_mut () ; let next_id_entry = flow_state . cycle_counts . entry (on_id) . or_default () ; let id = * next_id_entry ; * next_id_entry += 1 ; id } ; let ident = syn :: Ident :: new (& format ! ("cycle_{}" , next_id) , Span :: call_site ()) ; (ForwardRef { ident : ident . clone () , _phantom : PhantomData , } , S :: create_source (ident , self . clone ()) ,) } pub fn cycle < S : CycleCollection < 'a , TickCycleMarker , Location = Self > + DeferTick > (& self ,) -> (TickCycle < 'a , S > , S) where L : NoTick , { let next_id = { let on_id = match self . l . id () { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (_) => panic ! () , } ; let mut flow_state = self . flow_state () . borrow_mut () ; let next_id_entry = flow_state . cycle_counts . entry (on_id) . or_default () ; let id = * next_id_entry ; * next_id_entry += 1 ; id } ; let ident = syn :: Ident :: new (& format ! ("cycle_{}" , next_id) , Span :: call_site ()) ; (TickCycle { ident : ident . clone () , _phantom : PhantomData , } , S :: create_source (ident , self . clone ()) ,) } pub fn cycle_with_initial < S : CycleCollectionWithInitial < 'a , TickCycleMarker , Location = Self > + DeferTick , > (& self , initial : S ,) -> (TickCycle < 'a , S > , S) where L : NoTick , { let next_id = { let on_id = match self . l . id () { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (_) => panic ! () , } ; let mut flow_state = self . flow_state () . borrow_mut () ; let next_id_entry = flow_state . cycle_counts . entry (on_id) . or_default () ; let id = * next_id_entry ; * next_id_entry += 1 ; id } ; let ident = syn :: Ident :: new (& format ! ("cycle_{}" , next_id) , Span :: call_site ()) ; (TickCycle { ident : ident . clone () , _phantom : PhantomData , } , S :: create_source (ident , initial , self . clone ()) ,) } } } pub use tick :: { NoTick , Tick } ; pub use crate :: location :: LocationId ; # [cfg (stageleft_macro)] impl LocationId { pub fn root (& self) -> & LocationId { match self { LocationId :: Process (_) => self , LocationId :: Cluster (_) => self , LocationId :: Tick (_ , id) => id . root () , LocationId :: ExternalProcess (_) => self , } } pub fn raw_id (& self) -> usize { match self { LocationId :: Process (id) => * id , LocationId :: Cluster (id) => * id , LocationId :: Tick (_ , _) => panic ! ("cannot get raw id for tick") , LocationId :: ExternalProcess (id) => * id , } } } pub fn check_matching_location < 'a , L : Location < 'a > > (l1 : & L , l2 : & L) { assert_eq ! (l1 . id () , l2 . id () , "locations do not match") ; } pub use crate :: location :: Location ; } pub use location :: { Cluster , ClusterId , Location , Process , Tick } ; pub mod deploy { pub use std :: future :: Future ; pub use std :: io :: Error ; pub use std :: pin :: Pin ; pub use hydroflow :: bytes :: Bytes ; pub use hydroflow :: futures :: { Sink , Stream } ; pub use hydroflow_lang :: graph :: HydroflowGraph ; pub use serde :: de :: DeserializeOwned ; pub use serde :: Serialize ; pub use stageleft :: Quoted ; # [cfg (feature = "deploy_runtime")] pub mod macro_runtime { pub use std :: cell :: RefCell ; pub use std :: pin :: Pin ; pub use std :: rc :: Rc ; pub use hydroflow :: util :: deploy :: DeployPorts ; pub use stageleft :: { Quoted , RuntimeData } ; pub use super :: HydroflowPlusMeta ; pub use crate :: __staged :: deploy :: { ClusterSpec , Deploy , ExternalSpec , Node , ProcessSpec , RegisterPort } ; pub use crate :: __staged :: lang :: graph :: HydroflowGraph ; pub use crate :: deploy :: macro_runtime :: DeployRuntime ; # [cfg (stageleft_macro)] impl < 'a > Deploy < 'a > for DeployRuntime { type InstantiateEnv = () ; type CompileEnv = RuntimeData < & 'a DeployPorts < HydroflowPlusMeta > > ; type Process = DeployRuntimeNode ; type Cluster = DeployRuntimeCluster ; type ExternalProcess = DeployRuntimeNode ; type Port = String ; type ExternalRawPort = () ; type Meta = () ; type GraphId = usize ; fn has_trivial_node () -> bool { true } fn trivial_process (_id : usize) -> Self :: Process { DeployRuntimeNode { next_port : Rc :: new (RefCell :: new (0)) , } } fn trivail_cluster (_id : usize) -> Self :: Cluster { DeployRuntimeCluster { next_port : Rc :: new (RefCell :: new (0)) , } } fn allocate_process_port (process : & Self :: Process) -> Self :: Port { process . next_port () } fn allocate_cluster_port (cluster : & Self :: Cluster) -> Self :: Port { cluster . next_port () } fn allocate_external_port (_external : & Self :: ExternalProcess) -> Self :: Port { panic ! () ; } fn o2o_sink_source (env : & Self :: CompileEnv , _p1 : & Self :: Process , p1_port : & Self :: Port , _p2 : & Self :: Process , p2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { super :: deploy_runtime :: deploy_o2o (* env , p1_port . as_str () , p2_port . as_str ()) } fn o2o_connect (_p1 : & Self :: Process , _p1_port : & Self :: Port , _p2 : & Self :: Process , _p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { Box :: new (| | panic ! ()) } fn o2m_sink_source (env : & Self :: CompileEnv , _p1 : & Self :: Process , p1_port : & Self :: Port , _c2 : & Self :: Cluster , c2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { super :: deploy_runtime :: deploy_o2m (* env , p1_port . as_str () , c2_port . as_str ()) } fn o2m_connect (_p1 : & Self :: Process , _p1_port : & Self :: Port , _c2 : & Self :: Cluster , _c2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { Box :: new (| | panic ! ()) } fn m2o_sink_source (env : & Self :: CompileEnv , _c1 : & Self :: Cluster , c1_port : & Self :: Port , _p2 : & Self :: Process , p2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { super :: deploy_runtime :: deploy_m2o (* env , c1_port . as_str () , p2_port . as_str ()) } fn m2o_connect (_c1 : & Self :: Cluster , _c1_port : & Self :: Port , _p2 : & Self :: Process , _p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { Box :: new (| | panic ! ()) } fn m2m_sink_source (env : & Self :: CompileEnv , _c1 : & Self :: Cluster , c1_port : & Self :: Port , _c2 : & Self :: Cluster , c2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { super :: deploy_runtime :: deploy_m2m (* env , c1_port . as_str () , c2_port . as_str ()) } fn m2m_connect (_c1 : & Self :: Cluster , _c1_port : & Self :: Port , _c2 : & Self :: Cluster , _c2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { Box :: new (| | panic ! ()) } fn e2o_source (_compile_env : & Self :: CompileEnv , _p1 : & Self :: ExternalProcess , _p1_port : & Self :: Port , _p2 : & Self :: Process , _p2_port : & Self :: Port ,) -> syn :: Expr { panic ! () } fn e2o_connect (_p1 : & Self :: ExternalProcess , _p1_port : & Self :: Port , _p2 : & Self :: Process , _p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { panic ! () } fn o2e_sink (_compile_env : & Self :: CompileEnv , _p1 : & Self :: Process , _p1_port : & Self :: Port , _p2 : & Self :: ExternalProcess , _p2_port : & Self :: Port ,) -> syn :: Expr { panic ! () } fn o2e_connect (_p1 : & Self :: Process , _p1_port : & Self :: Port , _p2 : & Self :: ExternalProcess , _p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { panic ! () } fn cluster_ids (env : & Self :: CompileEnv , of_cluster : usize ,) -> impl Quoted < 'a , & 'a Vec < u32 > > + Copy + 'a { super :: deploy_runtime :: cluster_members (* env , of_cluster) } fn cluster_self_id (env : & Self :: CompileEnv) -> impl Quoted < 'a , u32 > + Copy + 'a { super :: deploy_runtime :: cluster_self_id (* env) } } pub use crate :: deploy :: macro_runtime :: DeployRuntimeNode ; # [cfg (stageleft_macro)] impl < 'a > RegisterPort < 'a , DeployRuntime > for DeployRuntimeNode { fn register (& self , _key : usize , _port : < DeployRuntime as Deploy > :: Port) { panic ! () } fn raw_port (& self , _key : usize) -> < DeployRuntime as Deploy > :: ExternalRawPort { panic ! () } # [expect (clippy :: manual_async_fn , reason = "buggy Clippy lint for lifetime bounds")] fn as_bytes_sink (& self , _key : usize ,) -> impl std :: future :: Future < Output = Pin < Box < dyn crate :: futures :: Sink < crate :: bytes :: Bytes , Error = std :: io :: Error > > > , > + 'a { async { panic ! () } } # [expect (clippy :: manual_async_fn , reason = "buggy Clippy lint for lifetime bounds")] fn as_bincode_sink < T : serde :: Serialize + 'static > (& self , _key : usize ,) -> impl std :: future :: Future < Output = Pin < Box < dyn crate :: futures :: Sink < T , Error = std :: io :: Error > > > , > + 'a { async { panic ! () } } # [expect (clippy :: manual_async_fn , reason = "buggy Clippy lint for lifetime bounds")] fn as_bytes_source (& self , _key : usize ,) -> impl std :: future :: Future < Output = Pin < Box < dyn hydroflow :: futures :: Stream < Item = hydroflow :: bytes :: Bytes > > > , > + 'a { async { panic ! () } } # [expect (clippy :: manual_async_fn , reason = "buggy Clippy lint for lifetime bounds")] fn as_bincode_source < T : serde :: de :: DeserializeOwned + 'static > (& self , _key : usize ,) -> impl std :: future :: Future < Output = Pin < Box < dyn hydroflow :: futures :: Stream < Item = T > > > > + 'a { async { panic ! () } } } # [cfg (stageleft_macro)] impl Node for DeployRuntimeNode { type Port = String ; type Meta = () ; type InstantiateEnv = () ; fn next_port (& self) -> String { let next_send_port = * self . next_port . borrow () ; * self . next_port . borrow_mut () += 1 ; format ! ("port_{}" , next_send_port) } fn update_meta (& mut self , _meta : & Self :: Meta) { } fn instantiate (& self , _env : & mut Self :: InstantiateEnv , _meta : & mut Self :: Meta , _graph : HydroflowGraph , _extra_stmts : Vec < syn :: Stmt > ,) { panic ! (".deploy() cannot be called on a DeployRuntimeNode") ; } } pub use crate :: deploy :: macro_runtime :: DeployRuntimeCluster ; # [cfg (stageleft_macro)] impl Node for DeployRuntimeCluster { type Port = String ; type Meta = () ; type InstantiateEnv = () ; fn next_port (& self) -> String { let next_send_port = * self . next_port . borrow () ; * self . next_port . borrow_mut () += 1 ; format ! ("port_{}" , next_send_port) } fn update_meta (& mut self , _meta : & Self :: Meta) { } fn instantiate (& self , _env : & mut Self :: InstantiateEnv , _meta : & mut Self :: Meta , _graph : HydroflowGraph , _extra_stmts : Vec < syn :: Stmt > ,) { panic ! (".deploy() cannot be called on a DeployRuntimeCluster") ; } } # [cfg (stageleft_macro)] impl ProcessSpec < '_ , DeployRuntime > for () { fn build (self , _id : usize , _name_hint : & str) -> DeployRuntimeNode { DeployRuntimeNode { next_port : Rc :: new (RefCell :: new (0)) , } } } # [cfg (stageleft_macro)] impl ClusterSpec < '_ , DeployRuntime > for () { fn build (self , _id : usize , _name_hint : & str) -> DeployRuntimeCluster { DeployRuntimeCluster { next_port : Rc :: new (RefCell :: new (0)) , } } } # [cfg (stageleft_macro)] impl ExternalSpec < '_ , DeployRuntime > for () { fn build (self , _id : usize , _name_hint : & str) -> DeployRuntimeNode { panic ! () } } } # [cfg (feature = "deploy")] pub mod trybuild { pub use std :: fs ; pub use std :: path :: PathBuf ; pub use stageleft :: internal :: quote ; pub use trybuild_internals_api :: cargo :: { self , Metadata } ; pub use trybuild_internals_api :: env :: Update ; pub use trybuild_internals_api :: run :: { PathDependency , Project } ; pub use trybuild_internals_api :: { dependencies , features , path , Runner } ; pub use crate :: __staged :: lang :: graph :: { partition_graph , HydroflowGraph } ; pub static IS_TEST : std :: sync :: atomic :: AtomicBool = std :: sync :: atomic :: AtomicBool :: new (false) ; pub fn init_test () { IS_TEST . store (true , std :: sync :: atomic :: Ordering :: Relaxed) ; } pub fn compile_graph_trybuild (graph : HydroflowGraph , extra_stmts : Vec < syn :: Stmt >) -> syn :: File { let partitioned_graph = partition_graph (graph) . expect ("Failed to partition (cycle detected).") ; let mut diagnostics = Vec :: new () ; let tokens = partitioned_graph . as_code (& quote ! { hydroflow_plus } , true , quote ! () , & mut diagnostics) ; let source_ast : syn :: File = syn :: parse_quote ! { #! [feature (box_patterns)] #! [allow (unused_crate_dependencies , missing_docs)] use hydroflow_plus ::*; # [allow (unused)] fn __hfplus_runtime <'a > (__hydroflow_plus_trybuild_cli : &'a hydroflow_plus :: util :: deploy :: DeployPorts < hydroflow_plus :: deploy :: HydroflowPlusMeta >) -> hydroflow_plus :: Hydroflow <'a > { # (# extra_stmts) * # tokens } # [tokio :: main] async fn main () { let ports = hydroflow_plus :: util :: deploy :: init_no_ack_start () . await ; let flow = __hfplus_runtime (& ports) ; println ! ("ack start") ; hydroflow_plus :: util :: deploy :: launch_flow (flow) . await ; } } ; source_ast } pub fn create_trybuild (source : & str , bin : & str , is_test : bool ,) -> Result < (PathBuf , PathBuf , Option < Vec < String > >) , trybuild_internals_api :: error :: Error > { let Metadata { target_directory : target_dir , workspace_root : workspace , packages , } = cargo :: metadata () ? ; let source_dir = cargo :: manifest_dir () ? ; let mut source_manifest = dependencies :: get_manifest (& source_dir) ? ; if ! is_test { source_manifest . dev_dependencies . clear () ; } let mut features = features :: find () ; let path_dependencies = source_manifest . dependencies . iter () . filter_map (| (name , dep) | { let path = dep . path . as_ref () ? ; if packages . iter () . any (| p | & p . name == name) { None } else { Some (PathDependency { name : name . clone () , normalized_path : path . canonicalize () . ok () ? , }) } }) . collect () ; let crate_name = source_manifest . package . name . clone () ; let project_dir = path ! (target_dir / "hfplus_trybuild" / crate_name /) ; fs :: create_dir_all (& project_dir) ? ; let project_name = format ! ("{}-hfplus-trybuild" , crate_name) ; let mut manifest = Runner :: make_manifest (& workspace , & project_name , & source_dir , & packages , & [] , source_manifest ,) ? ; manifest . features . remove ("stageleft_devel") ; if let Some (enabled_features) = & mut features { enabled_features . retain (| feature | manifest . features . contains_key (feature) || feature == "default") ; manifest . features . get_mut ("default") . iter_mut () . for_each (| v | { v . retain (| f | f != "stageleft_devel") ; }) ; } let project = Project { dir : project_dir , source_dir , target_dir , name : project_name , update : Update :: env () ? , has_pass : false , has_compile_fail : false , features , workspace , path_dependencies , manifest , keep_going : false , } ; let manifest_toml = toml :: to_string (& project . manifest) ? ; fs :: write (path ! (project . dir / "Cargo.toml") , manifest_toml) ? ; fs :: create_dir_all (path ! (project . dir / "src" / "bin")) ? ; let out_path = path ! (project . dir / "src" / "bin" / format ! ("{bin}.rs")) ; if ! out_path . exists () || fs :: read_to_string (& out_path) ? != source { fs :: write (path ! (project . dir / "src" / "bin" / format ! ("{bin}.rs")) , source ,) ? ; } let workspace_cargo_lock = path ! (project . workspace / "Cargo.lock") ; if workspace_cargo_lock . exists () { let _ = fs :: copy (workspace_cargo_lock , path ! (project . dir / "Cargo.lock")) ; } else { let _ = cargo :: cargo (& project) . arg ("generate-lockfile") . status () ; } let workspace_dot_cargo_config_toml = path ! (project . workspace / ".cargo" / "config.toml") ; if workspace_dot_cargo_config_toml . exists () { let dot_cargo_folder = path ! (project . dir / ".cargo") ; fs :: create_dir_all (& dot_cargo_folder) ? ; let _ = fs :: copy (workspace_dot_cargo_config_toml , path ! (dot_cargo_folder / "config.toml") ,) ; } Ok ((project . dir . as_ref () . into () , path ! (project . target_dir / "hfplus_trybuild") , project . features ,)) } } pub use macro_runtime :: * ; # [cfg (feature = "deploy")] pub use trybuild :: init_test ; # [cfg (feature = "deploy_runtime")] pub mod deploy_runtime { pub use std :: collections :: HashMap ; pub use hydroflow :: util :: deploy :: { ConnectedDemux , ConnectedDirect , ConnectedSink , ConnectedSource , ConnectedTagged , DeployPorts , } ; pub use serde :: { Deserialize , Serialize } ; pub use stageleft :: { q , Quoted , RuntimeData } ; pub use crate :: deploy :: deploy_runtime :: HydroflowPlusMeta ; pub fn cluster_members (cli : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , of_cluster : usize ,) -> impl Quoted <'_, & Vec < u32 > > + Copy { q ! (cli . meta . clusters . get (& of_cluster) . unwrap ()) } pub fn cluster_self_id (cli : RuntimeData < & DeployPorts < HydroflowPlusMeta > > ,) -> impl Quoted < u32 > + Copy { q ! (cli . meta . cluster_id . expect ("Tried to read Cluster ID on a non-cluster node")) } pub fn deploy_o2o (env : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , p1_port : & str , p2_port : & str ,) -> (syn :: Expr , syn :: Expr) { ({ q ! ({ env . port (p1_port) . connect_local_blocking ::< ConnectedDirect > () . into_sink () }) . splice_untyped () } , { q ! ({ env . port (p2_port) . connect_local_blocking ::< ConnectedDirect > () . into_source () }) . splice_untyped () } ,) } pub fn deploy_o2m (env : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , p1_port : & str , c2_port : & str ,) -> (syn :: Expr , syn :: Expr) { ({ q ! ({ env . port (p1_port) . connect_local_blocking ::< ConnectedDemux < ConnectedDirect >> () . into_sink () }) . splice_untyped () } , { q ! ({ env . port (c2_port) . connect_local_blocking ::< ConnectedDirect > () . into_source () }) . splice_untyped () } ,) } pub fn deploy_m2o (env : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , c1_port : & str , p2_port : & str ,) -> (syn :: Expr , syn :: Expr) { ({ q ! ({ env . port (c1_port) . connect_local_blocking ::< ConnectedDirect > () . into_sink () }) . splice_untyped () } , { q ! ({ env . port (p2_port) . connect_local_blocking ::< ConnectedTagged < ConnectedDirect >> () . into_source () }) . splice_untyped () } ,) } pub fn deploy_m2m (env : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , c1_port : & str , c2_port : & str ,) -> (syn :: Expr , syn :: Expr) { ({ q ! ({ env . port (c1_port) . connect_local_blocking ::< ConnectedDemux < ConnectedDirect >> () . into_sink () }) . splice_untyped () } , { q ! ({ env . port (c2_port) . connect_local_blocking ::< ConnectedTagged < ConnectedDirect >> () . into_source () }) . splice_untyped () } ,) } pub fn deploy_e2o (env : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , _e1_port : & str , p2_port : & str ,) -> syn :: Expr { q ! ({ env . port (p2_port) . connect_local_blocking ::< ConnectedDirect > () . into_source () }) . splice_untyped () } pub fn deploy_o2e (env : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , p1_port : & str , _e2_port : & str ,) -> syn :: Expr { q ! ({ env . port (p1_port) . connect_local_blocking ::< ConnectedDirect > () . into_sink () }) . splice_untyped () } } # [cfg (feature = "deploy_runtime")] pub use deploy_runtime :: HydroflowPlusMeta ; # [cfg (feature = "deploy")] pub mod deploy_graph { pub use std :: cell :: RefCell ; pub use std :: collections :: HashMap ; pub use std :: future :: Future ; pub use std :: io :: Error ; pub use std :: pin :: Pin ; pub use std :: rc :: Rc ; pub use std :: sync :: Arc ; pub use hydro_deploy :: custom_service :: CustomClientPort ; pub use hydro_deploy :: hydroflow_crate :: ports :: { DemuxSink , HydroflowSink , HydroflowSource , TaggedSource , } ; pub use hydro_deploy :: hydroflow_crate :: tracing_options :: TracingOptions ; pub use hydro_deploy :: hydroflow_crate :: HydroflowCrateService ; pub use hydro_deploy :: { CustomService , Deployment , Host , HydroflowCrate } ; pub use hydroflow :: futures :: StreamExt ; pub use hydroflow :: util :: deploy :: { ConnectedSink , ConnectedSource } ; pub use nameof :: name_of ; pub use serde :: de :: DeserializeOwned ; pub use serde :: Serialize ; pub use sha2 :: { Digest , Sha256 } ; pub use stageleft :: { Quoted , RuntimeData } ; pub use syn :: visit_mut :: VisitMut ; pub use tokio :: sync :: RwLock ; pub use trybuild_internals_api :: path ; pub use super :: deploy_runtime :: * ; pub use super :: trybuild :: { compile_graph_trybuild , create_trybuild } ; pub use super :: { ClusterSpec , Deploy , ExternalSpec , IntoProcessSpec , Node , ProcessSpec , RegisterPort } ; pub use crate :: __staged :: futures :: SinkExt ; pub use crate :: __staged :: lang :: graph :: HydroflowGraph ; pub use crate :: deploy :: deploy_graph :: HydroDeploy ; # [cfg (stageleft_macro)] impl < 'a > Deploy < 'a > for HydroDeploy { type InstantiateEnv = Deployment ; type CompileEnv = () ; type Process = DeployNode ; type Cluster = DeployCluster ; type ExternalProcess = DeployExternal ; type Meta = HashMap < usize , Vec < u32 > > ; type GraphId = () ; type Port = String ; type ExternalRawPort = CustomClientPort ; fn allocate_process_port (process : & Self :: Process) -> Self :: Port { process . next_port () } fn allocate_cluster_port (cluster : & Self :: Cluster) -> Self :: Port { cluster . next_port () } fn allocate_external_port (external : & Self :: ExternalProcess) -> Self :: Port { external . next_port () } fn o2o_sink_source (_env : & () , _p1 : & Self :: Process , p1_port : & Self :: Port , _p2 : & Self :: Process , p2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { let p1_port = p1_port . as_str () ; let p2_port = p2_port . as_str () ; deploy_o2o (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , p1_port , p2_port ,) } fn o2o_connect (p1 : & Self :: Process , p1_port : & Self :: Port , p2 : & Self :: Process , p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { let p1 = p1 . clone () ; let p1_port = p1_port . clone () ; let p2 = p2 . clone () ; let p2_port = p2_port . clone () ; Box :: new (move | | { let self_underlying_borrow = p1 . underlying . borrow () ; let self_underlying = self_underlying_borrow . as_ref () . unwrap () ; let source_port = self_underlying . try_read () . unwrap () . get_port (p1_port . clone () , self_underlying) ; let other_underlying_borrow = p2 . underlying . borrow () ; let other_underlying = other_underlying_borrow . as_ref () . unwrap () ; let recipient_port = other_underlying . try_read () . unwrap () . get_port (p2_port . clone () , other_underlying) ; source_port . send_to (& recipient_port) }) } fn o2m_sink_source (_env : & () , _p1 : & Self :: Process , p1_port : & Self :: Port , _c2 : & Self :: Cluster , c2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { let p1_port = p1_port . as_str () ; let c2_port = c2_port . as_str () ; deploy_o2m (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , p1_port , c2_port ,) } fn o2m_connect (p1 : & Self :: Process , p1_port : & Self :: Port , c2 : & Self :: Cluster , c2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { let p1 = p1 . clone () ; let p1_port = p1_port . clone () ; let c2 = c2 . clone () ; let c2_port = c2_port . clone () ; Box :: new (move | | { let self_underlying_borrow = p1 . underlying . borrow () ; let self_underlying = self_underlying_borrow . as_ref () . unwrap () ; let source_port = self_underlying . try_read () . unwrap () . get_port (p1_port . clone () , self_underlying) ; let recipient_port = DemuxSink { demux : c2 . members . borrow () . iter () . enumerate () . map (| (id , c) | { let n = c . underlying . try_read () . unwrap () ; (id as u32 , Arc :: new (n . get_port (c2_port . clone () , & c . underlying)) as Arc < dyn HydroflowSink + 'static > ,) }) . collect () , } ; source_port . send_to (& recipient_port) }) } fn m2o_sink_source (_env : & () , _c1 : & Self :: Cluster , c1_port : & Self :: Port , _p2 : & Self :: Process , p2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { let c1_port = c1_port . as_str () ; let p2_port = p2_port . as_str () ; deploy_m2o (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , c1_port , p2_port ,) } fn m2o_connect (c1 : & Self :: Cluster , c1_port : & Self :: Port , p2 : & Self :: Process , p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { let c1 = c1 . clone () ; let c1_port = c1_port . clone () ; let p2 = p2 . clone () ; let p2_port = p2_port . clone () ; Box :: new (move | | { let other_underlying_borrow = p2 . underlying . borrow () ; let other_underlying = other_underlying_borrow . as_ref () . unwrap () ; let recipient_port = other_underlying . try_read () . unwrap () . get_port (p2_port . clone () , other_underlying) . merge () ; for (i , node) in c1 . members . borrow () . iter () . enumerate () { let source_port = node . underlying . try_read () . unwrap () . get_port (c1_port . clone () , & node . underlying) ; TaggedSource { source : Arc :: new (source_port) , tag : i as u32 , } . send_to (& recipient_port) ; } }) } fn m2m_sink_source (_env : & () , _c1 : & Self :: Cluster , c1_port : & Self :: Port , _c2 : & Self :: Cluster , c2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { let c1_port = c1_port . as_str () ; let c2_port = c2_port . as_str () ; deploy_m2m (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , c1_port , c2_port ,) } fn m2m_connect (c1 : & Self :: Cluster , c1_port : & Self :: Port , c2 : & Self :: Cluster , c2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { let c1 = c1 . clone () ; let c1_port = c1_port . clone () ; let c2 = c2 . clone () ; let c2_port = c2_port . clone () ; Box :: new (move | | { for (i , sender) in c1 . members . borrow () . iter () . enumerate () { let source_port = sender . underlying . try_read () . unwrap () . get_port (c1_port . clone () , & sender . underlying) ; let recipient_port = DemuxSink { demux : c2 . members . borrow () . iter () . enumerate () . map (| (id , c) | { let n = c . underlying . try_read () . unwrap () ; (id as u32 , Arc :: new (n . get_port (c2_port . clone () , & c . underlying) . merge ()) as Arc < dyn HydroflowSink + 'static > ,) }) . collect () , } ; TaggedSource { source : Arc :: new (source_port) , tag : i as u32 , } . send_to (& recipient_port) ; } }) } fn e2o_source (_compile_env : & Self :: CompileEnv , _p1 : & Self :: ExternalProcess , p1_port : & Self :: Port , _p2 : & Self :: Process , p2_port : & Self :: Port ,) -> syn :: Expr { let p1_port = p1_port . as_str () ; let p2_port = p2_port . as_str () ; deploy_e2o (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , p1_port , p2_port ,) } fn e2o_connect (p1 : & Self :: ExternalProcess , p1_port : & Self :: Port , p2 : & Self :: Process , p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { let p1 = p1 . clone () ; let p1_port = p1_port . clone () ; let p2 = p2 . clone () ; let p2_port = p2_port . clone () ; Box :: new (move | | { let self_underlying_borrow = p1 . underlying . borrow () ; let self_underlying = self_underlying_borrow . as_ref () . unwrap () ; let source_port = self_underlying . try_read () . unwrap () . declare_client (self_underlying) ; let other_underlying_borrow = p2 . underlying . borrow () ; let other_underlying = other_underlying_borrow . as_ref () . unwrap () ; let recipient_port = other_underlying . try_read () . unwrap () . get_port (p2_port . clone () , other_underlying) ; source_port . send_to (& recipient_port) ; p1 . client_ports . borrow_mut () . insert (p1_port . clone () , source_port) ; }) } fn o2e_sink (_compile_env : & Self :: CompileEnv , _p1 : & Self :: Process , p1_port : & Self :: Port , _p2 : & Self :: ExternalProcess , p2_port : & Self :: Port ,) -> syn :: Expr { let p1_port = p1_port . as_str () ; let p2_port = p2_port . as_str () ; deploy_o2e (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , p1_port , p2_port ,) } fn o2e_connect (p1 : & Self :: Process , p1_port : & Self :: Port , p2 : & Self :: ExternalProcess , p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { let p1 = p1 . clone () ; let p1_port = p1_port . clone () ; let p2 = p2 . clone () ; let p2_port = p2_port . clone () ; Box :: new (move | | { let self_underlying_borrow = p1 . underlying . borrow () ; let self_underlying = self_underlying_borrow . as_ref () . unwrap () ; let source_port = self_underlying . try_read () . unwrap () . get_port (p1_port . clone () , self_underlying) ; let other_underlying_borrow = p2 . underlying . borrow () ; let other_underlying = other_underlying_borrow . as_ref () . unwrap () ; let recipient_port = other_underlying . try_read () . unwrap () . declare_client (other_underlying) ; source_port . send_to (& recipient_port) ; p2 . client_ports . borrow_mut () . insert (p2_port . clone () , recipient_port) ; }) } fn cluster_ids (_env : & Self :: CompileEnv , of_cluster : usize ,) -> impl Quoted < 'a , & 'a Vec < u32 > > + Copy + 'a { cluster_members (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , of_cluster ,) } fn cluster_self_id (_env : & Self :: CompileEnv) -> impl Quoted < 'a , u32 > + Copy + 'a { cluster_self_id (RuntimeData :: new ("__hydroflow_plus_trybuild_cli")) } } pub use crate :: deploy :: deploy_graph :: DeployCrateWrapper ; pub use crate :: deploy :: deploy_graph :: TrybuildHost ; # [cfg (stageleft_macro)] impl From < Arc < dyn Host > > for TrybuildHost { fn from (host : Arc < dyn Host >) -> Self { Self { host , display_name : None , rustflags : None , tracing : None , name_hint : None , cluster_idx : None , } } } # [cfg (stageleft_macro)] impl < H : Host + 'static > From < Arc < H > > for TrybuildHost { fn from (host : Arc < H >) -> Self { Self { host , display_name : None , rustflags : None , tracing : None , name_hint : None , cluster_idx : None , } } } # [cfg (stageleft_macro)] impl TrybuildHost { pub fn new (host : Arc < dyn Host >) -> Self { Self { host , display_name : None , rustflags : None , tracing : None , name_hint : None , cluster_idx : None , } } pub fn display_name (self , display_name : impl Into < String >) -> Self { if self . display_name . is_some () { panic ! ("{} already set" , name_of ! (display_name in Self)) ; } Self { display_name : Some (display_name . into ()) , .. self } } pub fn rustflags (self , rustflags : impl Into < String >) -> Self { if self . rustflags . is_some () { panic ! ("{} already set" , name_of ! (rustflags in Self)) ; } Self { rustflags : Some (rustflags . into ()) , .. self } } pub fn tracing (self , tracing : TracingOptions) -> Self { if self . tracing . is_some () { panic ! ("{} already set" , name_of ! (tracing in Self)) ; } Self { tracing : Some (tracing) , .. self } } } # [cfg (stageleft_macro)] impl IntoProcessSpec < '_ , HydroDeploy > for Arc < dyn Host > { type ProcessSpec = TrybuildHost ; fn into_process_spec (self) -> TrybuildHost { TrybuildHost { host : self , display_name : None , rustflags : None , tracing : None , name_hint : None , cluster_idx : None , } } } # [cfg (stageleft_macro)] impl < H : Host + 'static > IntoProcessSpec < '_ , HydroDeploy > for Arc < H > { type ProcessSpec = TrybuildHost ; fn into_process_spec (self) -> TrybuildHost { TrybuildHost { host : self , display_name : None , rustflags : None , tracing : None , name_hint : None , cluster_idx : None , } } } pub use crate :: deploy :: deploy_graph :: DeployExternal ; # [cfg (stageleft_macro)] impl DeployExternal { pub fn take_port (& self , key : usize) -> CustomClientPort { self . client_ports . borrow_mut () . remove (self . allocated_ports . borrow () . get (& key) . unwrap ()) . unwrap () } } # [cfg (stageleft_macro)] impl < 'a > RegisterPort < 'a , HydroDeploy > for DeployExternal { fn register (& self , key : usize , port : < HydroDeploy as Deploy > :: Port) { self . allocated_ports . borrow_mut () . insert (key , port) ; } fn raw_port (& self , key : usize) -> < HydroDeploy as Deploy > :: ExternalRawPort { self . client_ports . borrow_mut () . remove (self . allocated_ports . borrow () . get (& key) . unwrap ()) . unwrap () } fn as_bytes_sink (& self , key : usize ,) -> impl Future < Output = Pin < Box < dyn crate :: futures :: Sink < crate :: bytes :: Bytes , Error = Error > > > > + 'a { let port = self . raw_port (key) ; async move { let sink = port . connect () . await . into_sink () ; sink as Pin < Box < dyn crate :: futures :: Sink < crate :: bytes :: Bytes , Error = Error > > > } } fn as_bincode_sink < T : Serialize + 'static > (& self , key : usize ,) -> impl Future < Output = Pin < Box < dyn crate :: futures :: Sink < T , Error = Error > > > > + 'a { let port = self . raw_port (key) ; async move { let sink = port . connect () . await . into_sink () ; Box :: pin (sink . with (| item | async move { Ok (bincode :: serialize (& item) . unwrap () . into ()) })) as Pin < Box < dyn crate :: futures :: Sink < T , Error = Error > > > } } fn as_bytes_source (& self , key : usize ,) -> impl Future < Output = Pin < Box < dyn crate :: futures :: Stream < Item = crate :: bytes :: Bytes > > > > + 'a { let port = self . raw_port (key) ; async move { let source = port . connect () . await . into_source () ; Box :: pin (source . map (| r | r . unwrap () . freeze ())) as Pin < Box < dyn crate :: futures :: Stream < Item = crate :: bytes :: Bytes > > > } } fn as_bincode_source < T : DeserializeOwned + 'static > (& self , key : usize ,) -> impl Future < Output = Pin < Box < dyn crate :: futures :: Stream < Item = T > > > > + 'a { let port = self . raw_port (key) ; async move { let source = port . connect () . await . into_source () ; Box :: pin (source . map (| item | bincode :: deserialize (& item . unwrap ()) . unwrap ())) as Pin < Box < dyn crate :: futures :: Stream < Item = T > > > } } } # [cfg (stageleft_macro)] impl Node for DeployExternal { type Port = String ; type Meta = HashMap < usize , Vec < u32 > > ; type InstantiateEnv = Deployment ; fn next_port (& self) -> Self :: Port { let next_port = * self . next_port . borrow () ; * self . next_port . borrow_mut () += 1 ; format ! ("port_{}" , next_port) } fn instantiate (& self , env : & mut Self :: InstantiateEnv , _meta : & mut Self :: Meta , _graph : HydroflowGraph , _extra_stmts : Vec < syn :: Stmt > ,) { let service = env . CustomService (self . host . clone () , vec ! []) ; * self . underlying . borrow_mut () = Some (service) ; } fn update_meta (& mut self , _meta : & Self :: Meta) { } } # [cfg (stageleft_macro)] impl ExternalSpec < '_ , HydroDeploy > for Arc < dyn Host > { fn build (self , _id : usize , _name_hint : & str) -> DeployExternal { DeployExternal { next_port : Rc :: new (RefCell :: new (0)) , host : self , underlying : Rc :: new (RefCell :: new (None)) , allocated_ports : Rc :: new (RefCell :: new (HashMap :: new ())) , client_ports : Rc :: new (RefCell :: new (HashMap :: new ())) , } } } # [cfg (stageleft_macro)] impl < H : Host + 'static > ExternalSpec < '_ , HydroDeploy > for Arc < H > { fn build (self , _id : usize , _name_hint : & str) -> DeployExternal { DeployExternal { next_port : Rc :: new (RefCell :: new (0)) , host : self , underlying : Rc :: new (RefCell :: new (None)) , allocated_ports : Rc :: new (RefCell :: new (HashMap :: new ())) , client_ports : Rc :: new (RefCell :: new (HashMap :: new ())) , } } } pub use crate :: deploy :: deploy_graph :: CrateOrTrybuild ; pub use crate :: deploy :: deploy_graph :: DeployNode ; # [cfg (stageleft_macro)] impl DeployCrateWrapper for DeployNode { fn underlying (& self) -> Arc < RwLock < HydroflowCrateService > > { self . underlying . borrow () . as_ref () . unwrap () . clone () } } # [cfg (stageleft_macro)] impl Node for DeployNode { type Port = String ; type Meta = HashMap < usize , Vec < u32 > > ; type InstantiateEnv = Deployment ; fn next_port (& self) -> String { let next_port = * self . next_port . borrow () ; * self . next_port . borrow_mut () += 1 ; format ! ("port_{}" , next_port) } fn update_meta (& mut self , meta : & Self :: Meta) { let underlying_node = self . underlying . borrow () ; let mut n = underlying_node . as_ref () . unwrap () . try_write () . unwrap () ; n . update_meta (HydroflowPlusMeta { clusters : meta . clone () , cluster_id : None , subgraph_id : self . id , }) ; } fn instantiate (& self , env : & mut Self :: InstantiateEnv , _meta : & mut Self :: Meta , graph : HydroflowGraph , extra_stmts : Vec < syn :: Stmt > ,) { let service = match self . service_spec . borrow_mut () . take () . unwrap () { CrateOrTrybuild :: Crate (c) => c , CrateOrTrybuild :: Trybuild (trybuild) => { let (bin_name , (dir , target_dir , features)) = create_graph_trybuild (graph , extra_stmts , & trybuild . name_hint) ; create_trybuild_service (trybuild , & dir , & target_dir , & features , & bin_name) } } ; * self . underlying . borrow_mut () = Some (env . add_service (service)) ; } } pub use crate :: deploy :: deploy_graph :: DeployClusterNode ; # [cfg (stageleft_macro)] impl DeployCrateWrapper for DeployClusterNode { fn underlying (& self) -> Arc < RwLock < HydroflowCrateService > > { self . underlying . clone () } } pub use crate :: deploy :: deploy_graph :: DeployCluster ; # [cfg (stageleft_macro)] impl DeployCluster { pub fn members (& self) -> Vec < DeployClusterNode > { self . members . borrow () . clone () } } # [cfg (stageleft_macro)] impl Node for DeployCluster { type Port = String ; type Meta = HashMap < usize , Vec < u32 > > ; type InstantiateEnv = Deployment ; fn next_port (& self) -> String { let next_port = * self . next_port . borrow () ; * self . next_port . borrow_mut () += 1 ; format ! ("port_{}" , next_port) } fn instantiate (& self , env : & mut Self :: InstantiateEnv , meta : & mut Self :: Meta , graph : HydroflowGraph , extra_stmts : Vec < syn :: Stmt > ,) { let has_trybuild = self . cluster_spec . borrow () . as_ref () . unwrap () . iter () . any (| spec | matches ! (spec , CrateOrTrybuild :: Trybuild { .. })) ; let maybe_trybuild = if has_trybuild { Some (create_graph_trybuild (graph , extra_stmts , & self . name_hint)) } else { None } ; let cluster_nodes = self . cluster_spec . borrow_mut () . take () . unwrap () . into_iter () . map (| spec | { let service = match spec { CrateOrTrybuild :: Crate (c) => c , CrateOrTrybuild :: Trybuild (trybuild) => { let (bin_name , (dir , target_dir , features)) = maybe_trybuild . as_ref () . unwrap () ; create_trybuild_service (trybuild , dir , target_dir , features , bin_name) } } ; env . add_service (service) }) . collect :: < Vec < _ > > () ; meta . insert (self . id , (0 .. (cluster_nodes . len () as u32)) . collect ()) ; * self . members . borrow_mut () = cluster_nodes . into_iter () . map (| n | DeployClusterNode { underlying : n }) . collect () ; } fn update_meta (& mut self , meta : & Self :: Meta) { for (cluster_id , node) in self . members . borrow () . iter () . enumerate () { let mut n = node . underlying . try_write () . unwrap () ; n . update_meta (HydroflowPlusMeta { clusters : meta . clone () , cluster_id : Some (cluster_id as u32) , subgraph_id : self . id , }) ; } } } pub use crate :: deploy :: deploy_graph :: DeployProcessSpec ; # [cfg (stageleft_macro)] impl DeployProcessSpec { pub fn new (t : HydroflowCrate) -> Self { Self (t) } } # [cfg (stageleft_macro)] impl ProcessSpec < '_ , HydroDeploy > for DeployProcessSpec { fn build (self , id : usize , _name_hint : & str) -> DeployNode { DeployNode { id , next_port : Rc :: new (RefCell :: new (0)) , service_spec : Rc :: new (RefCell :: new (Some (CrateOrTrybuild :: Crate (self . 0)))) , underlying : Rc :: new (RefCell :: new (None)) , } } } # [cfg (stageleft_macro)] impl ProcessSpec < '_ , HydroDeploy > for TrybuildHost { fn build (mut self , id : usize , name_hint : & str) -> DeployNode { self . name_hint = Some (format ! ("{} (process {id})" , name_hint)) ; DeployNode { id , next_port : Rc :: new (RefCell :: new (0)) , service_spec : Rc :: new (RefCell :: new (Some (CrateOrTrybuild :: Trybuild (self)))) , underlying : Rc :: new (RefCell :: new (None)) , } } } pub use crate :: deploy :: deploy_graph :: DeployClusterSpec ; # [cfg (stageleft_macro)] impl DeployClusterSpec { pub fn new (crates : Vec < HydroflowCrate >) -> Self { Self (crates) } } # [cfg (stageleft_macro)] impl ClusterSpec < '_ , HydroDeploy > for DeployClusterSpec { fn build (self , id : usize , _name_hint : & str) -> DeployCluster { DeployCluster { id , next_port : Rc :: new (RefCell :: new (0)) , cluster_spec : Rc :: new (RefCell :: new (Some (self . 0 . into_iter () . map (CrateOrTrybuild :: Crate) . collect () ,))) , members : Rc :: new (RefCell :: new (vec ! [])) , name_hint : None , } } } # [cfg (stageleft_macro)] impl < T : Into < TrybuildHost > , I : IntoIterator < Item = T > > ClusterSpec < '_ , HydroDeploy > for I { fn build (self , id : usize , name_hint : & str) -> DeployCluster { let name_hint = format ! ("{} (cluster {id})" , name_hint) ; DeployCluster { id , next_port : Rc :: new (RefCell :: new (0)) , cluster_spec : Rc :: new (RefCell :: new (Some (self . into_iter () . enumerate () . map (| (idx , b) | { let mut b = b . into () ; b . name_hint = Some (name_hint . clone ()) ; b . cluster_idx = Some (idx) ; CrateOrTrybuild :: Trybuild (b) }) . collect () ,))) , members : Rc :: new (RefCell :: new (vec ! [])) , name_hint : Some (name_hint) , } } } pub fn clean_name_hint (name_hint : & str) -> String { name_hint . replace ("::" , "__") . replace (" " , "_") . replace ("," , "_") . replace ("<" , "_") . replace (">" , "") . replace ("(" , "") . replace (")" , "") } pub use crate :: deploy :: deploy_graph :: ReplaceCrateNameWithStaged ; # [cfg (stageleft_macro)] impl VisitMut for ReplaceCrateNameWithStaged { fn visit_type_path_mut (& mut self , i : & mut syn :: TypePath) { if let Some (first) = i . path . segments . first () { if first . ident == self . crate_name { let tail = i . path . segments . iter () . skip (1) . collect :: < Vec < _ > > () ; * i = syn :: parse_quote ! (crate :: __staged # (::# tail) *) ; } } syn :: visit_mut :: visit_type_path_mut (self , i) ; } } pub use crate :: deploy :: deploy_graph :: ReplaceCrateWithOrig ; # [cfg (stageleft_macro)] impl VisitMut for ReplaceCrateWithOrig { fn visit_item_use_mut (& mut self , i : & mut syn :: ItemUse) { if let syn :: UseTree :: Path (p) = & mut i . tree { if p . ident == "crate" { p . ident = syn :: Ident :: new (& self . crate_name , p . ident . span ()) ; i . leading_colon = Some (Default :: default ()) ; } } syn :: visit_mut :: visit_item_use_mut (self , i) ; } } pub fn create_graph_trybuild (graph : HydroflowGraph , extra_stmts : Vec < syn :: Stmt > , name_hint : & Option < String > ,) -> (String , (std :: path :: PathBuf , std :: path :: PathBuf , Option < Vec < String > >) ,) { let source_dir = trybuild_internals_api :: cargo :: manifest_dir () . unwrap () ; let source_manifest = trybuild_internals_api :: dependencies :: get_manifest (& source_dir) . unwrap () ; let crate_name = & source_manifest . package . name . to_string () . replace ("-" , "_") ; let is_test = super :: trybuild :: IS_TEST . load (std :: sync :: atomic :: Ordering :: Relaxed) ; let mut generated_code = compile_graph_trybuild (graph , extra_stmts) ; ReplaceCrateNameWithStaged { crate_name : crate_name . clone () , } . visit_file_mut (& mut generated_code) ; let mut inlined_staged = stageleft_tool :: gen_staged_trybuild (& path ! (source_dir / "src" / "lib.rs") , crate_name . clone () , is_test ,) ; ReplaceCrateWithOrig { crate_name : crate_name . clone () , } . visit_file_mut (& mut inlined_staged) ; let source = prettyplease :: unparse (& syn :: parse_quote ! { # generated_code # [allow (unused , ambiguous_glob_reexports , clippy :: suspicious_else_formatting , unexpected_cfgs , reason = "generated code")] pub mod __staged { # inlined_staged } }) ; let mut hasher = Sha256 :: new () ; hasher . update (& source) ; let hash = format ! ("{:X}" , hasher . finalize ()) . chars () . take (8) . collect :: < String > () ; let bin_name = if let Some (name_hint) = & name_hint { format ! ("{}_{}" , clean_name_hint (name_hint) , & hash) } else { hash } ; let trybuild_created = create_trybuild (& source , & bin_name , is_test) . unwrap () ; (bin_name , trybuild_created) } pub fn create_trybuild_service (trybuild : TrybuildHost , dir : & std :: path :: PathBuf , target_dir : & std :: path :: PathBuf , features : & Option < Vec < String > > , bin_name : & str ,) -> HydroflowCrate { let mut ret = HydroflowCrate :: new (dir , trybuild . host) . target_dir (target_dir) . bin (bin_name) . no_default_features () ; if let Some (display_name) = trybuild . display_name { ret = ret . display_name (display_name) ; } else if let Some (name_hint) = trybuild . name_hint { if let Some (cluster_idx) = trybuild . cluster_idx { ret = ret . display_name (format ! ("{} / {}" , name_hint , cluster_idx)) ; } else { ret = ret . display_name (name_hint) ; } } if let Some (rustflags) = trybuild . rustflags { ret = ret . rustflags (rustflags) ; } if let Some (tracing) = trybuild . tracing { ret = ret . tracing (tracing) ; } if let Some (features) = features { ret = ret . features (features . clone ()) ; } ret } } # [cfg (feature = "deploy")] pub use deploy_graph :: * ; pub mod in_memory_graph { pub use hydroflow_lang :: graph :: HydroflowGraph ; pub use super :: { LocalDeploy , Node , ProcessSpec } ; pub use crate :: deploy :: in_memory_graph :: SingleProcessGraph ; # [cfg (stageleft_macro)] impl LocalDeploy < '_ > for SingleProcessGraph { type Process = SingleNode ; type Cluster = SingleNode ; type ExternalProcess = SingleNode ; type Meta = () ; type GraphId = () ; fn has_trivial_node () -> bool { true } fn trivial_process (_id : usize) -> Self :: Process { SingleNode { } } fn trivial_cluster (_id : usize) -> Self :: Cluster { SingleNode { } } } # [cfg (stageleft_macro)] impl ProcessSpec < '_ , SingleProcessGraph > for () { fn build (self , _id : usize , _name_hint : & str) -> SingleNode { SingleNode { } } } pub use crate :: deploy :: in_memory_graph :: SingleNode ; # [cfg (stageleft_macro)] impl Node for SingleNode { type Port = () ; type Meta = () ; type InstantiateEnv = () ; fn next_port (& self) { panic ! () ; } fn update_meta (& mut self , _meta : & Self :: Meta) { } fn instantiate (& self , _env : & mut Self :: InstantiateEnv , _meta : & mut Self :: Meta , _graph : HydroflowGraph , _extra_stmts : Vec < syn :: Stmt > ,) { } } pub use crate :: deploy :: in_memory_graph :: MultiGraph ; # [cfg (stageleft_macro)] impl LocalDeploy < '_ > for MultiGraph { type Process = MultiNode ; type Cluster = MultiNode ; type ExternalProcess = MultiNode ; type Meta = () ; type GraphId = usize ; fn has_trivial_node () -> bool { true } fn trivial_process (_id : usize) -> Self :: Process { MultiNode { } } fn trivial_cluster (_id : usize) -> Self :: Cluster { MultiNode { } } } # [cfg (stageleft_macro)] impl ProcessSpec < '_ , MultiGraph > for () { fn build (self , _id : usize , _name_hint : & str) -> MultiNode { MultiNode { } } } pub use crate :: deploy :: in_memory_graph :: MultiNode ; # [cfg (stageleft_macro)] impl Node for MultiNode { type Port = () ; type Meta = () ; type InstantiateEnv = () ; fn next_port (& self) { panic ! () ; } fn update_meta (& mut self , _meta : & Self :: Meta) { } fn instantiate (& self , _env : & mut Self :: InstantiateEnv , _meta : & mut Self :: Meta , _graph : HydroflowGraph , _extra_stmts : Vec < syn :: Stmt > ,) { } } } pub use in_memory_graph :: * ; pub use crate :: deploy :: LocalDeploy ; pub use crate :: deploy :: Deploy ; # [cfg (stageleft_macro)] impl < 'a , T : Deploy < 'a , Process = N , Cluster = C , ExternalProcess = E , Meta = M , GraphId = R > , N : Node < Meta = M > , C : Node < Meta = M > , E : Node < Meta = M > , M : Default , R , > LocalDeploy < 'a > for T { type Process = N ; type Cluster = C ; type ExternalProcess = E ; type Meta = M ; type GraphId = R ; fn has_trivial_node () -> bool { < T as Deploy < 'a > > :: has_trivial_node () } fn trivial_process (id : usize) -> Self :: Process { < T as Deploy < 'a > > :: trivial_process (id) } fn trivial_cluster (id : usize) -> Self :: Cluster { < T as Deploy < 'a > > :: trivail_cluster (id) } } pub use crate :: deploy :: ProcessSpec ; pub use crate :: deploy :: IntoProcessSpec ; # [cfg (stageleft_macro)] impl < 'a , D : LocalDeploy < 'a > + ? Sized , T : ProcessSpec < 'a , D > > IntoProcessSpec < 'a , D > for T { type ProcessSpec = T ; fn into_process_spec (self) -> Self :: ProcessSpec { self } } pub use crate :: deploy :: ClusterSpec ; pub use crate :: deploy :: ExternalSpec ; pub use crate :: deploy :: Node ; pub use crate :: deploy :: RegisterPort ; } pub use deploy :: { ClusterSpec , Deploy , ProcessSpec } ; pub mod cycle { pub use std :: marker :: PhantomData ; pub use crate :: __staged :: location :: Location ; pub use crate :: cycle :: ForwardRefMarker ; pub use crate :: cycle :: TickCycleMarker ; pub use crate :: cycle :: DeferTick ; pub use crate :: cycle :: CycleComplete ; pub use crate :: cycle :: CycleCollection ; pub use crate :: cycle :: CycleCollectionWithInitial ; pub use crate :: cycle :: ForwardRef ; # [cfg (stageleft_macro)] impl < 'a , S : CycleComplete < 'a , ForwardRefMarker > > ForwardRef < 'a , S > { pub fn complete (self , stream : S) { let ident = self . ident ; S :: complete (stream , ident) } } pub use crate :: cycle :: TickCycle ; # [cfg (stageleft_macro)] impl < 'a , S : CycleComplete < 'a , TickCycleMarker > + DeferTick > TickCycle < 'a , S > { pub fn complete_next_tick (self , stream : S) { let ident = self . ident ; S :: complete (stream . defer_tick () , ident) } } } pub mod builder { pub use std :: cell :: RefCell ; pub use std :: collections :: HashMap ; pub use std :: marker :: PhantomData ; pub use std :: rc :: Rc ; pub use compiled :: CompiledFlow ; pub use deploy :: { DeployFlow , DeployResult } ; pub use stageleft :: * ; pub use crate :: __staged :: deploy :: { ExternalSpec , IntoProcessSpec , LocalDeploy } ; pub use crate :: __staged :: ir :: HfPlusLeaf ; pub use crate :: __staged :: location :: { Cluster , ExternalProcess , Process } ; pub use crate :: __staged :: { ClusterSpec , Deploy , RuntimeContext } ; pub mod built { pub use std :: collections :: { BTreeMap , HashMap } ; pub use std :: marker :: PhantomData ; pub use hydroflow_lang :: graph :: { eliminate_extra_unions_tees , HydroflowGraph } ; pub use super :: compiled :: CompiledFlow ; pub use super :: deploy :: { DeployFlow , DeployResult } ; pub use crate :: __staged :: deploy :: { ClusterSpec , Deploy , ExternalSpec , IntoProcessSpec , LocalDeploy } ; pub use crate :: __staged :: ir :: HfPlusLeaf ; pub use crate :: __staged :: location :: { Cluster , ExternalProcess , Process } ; pub use crate :: builder :: built :: BuiltFlow ; # [cfg (stageleft_macro)] impl Drop for BuiltFlow < '_ > { fn drop (& mut self) { if ! self . used { panic ! ("Dropped BuiltFlow without instantiating, you may have forgotten to call `compile` or `deploy`.") ; } } } pub fn build_inner (ir : & mut Vec < HfPlusLeaf >) -> BTreeMap < usize , HydroflowGraph > { let mut builders = BTreeMap :: new () ; let mut built_tees = HashMap :: new () ; let mut next_stmt_id = 0 ; for leaf in ir { leaf . emit (& mut builders , & mut built_tees , & mut next_stmt_id) ; } builders . into_iter () . map (| (k , v) | { let (mut flat_graph , _ , _) = v . build () ; eliminate_extra_unions_tees (& mut flat_graph) ; (k , flat_graph) }) . collect () } # [cfg (stageleft_macro)] impl < 'a > BuiltFlow < 'a > { pub fn ir (& self) -> & Vec < HfPlusLeaf > { & self . ir } pub fn optimize_with (mut self , f : impl FnOnce (Vec < HfPlusLeaf >) -> Vec < HfPlusLeaf >) -> Self { self . used = true ; BuiltFlow { ir : f (std :: mem :: take (& mut self . ir)) , processes : std :: mem :: take (& mut self . processes) , clusters : std :: mem :: take (& mut self . clusters) , used : false , _phantom : PhantomData , } } pub fn with_default_optimize < D : LocalDeploy < 'a > > (self) -> DeployFlow < 'a , D > { self . optimize_with (crate :: rewrites :: persist_pullup :: persist_pullup) . into_deploy () } fn into_deploy < D : LocalDeploy < 'a > > (mut self) -> DeployFlow < 'a , D > { self . used = true ; let processes = if D :: has_trivial_node () { self . processes . iter () . map (| id | (* id , D :: trivial_process (* id))) . collect () } else { HashMap :: new () } ; let clusters = if D :: has_trivial_node () { self . clusters . iter () . map (| id | (* id , D :: trivial_cluster (* id))) . collect () } else { HashMap :: new () } ; DeployFlow { ir : std :: mem :: take (& mut self . ir) , nodes : processes , clusters , externals : HashMap :: new () , used : false , _phantom : PhantomData , } } pub fn with_process < P , D : LocalDeploy < 'a > > (self , process : & Process < P > , spec : impl IntoProcessSpec < 'a , D > ,) -> DeployFlow < 'a , D > { self . into_deploy () . with_process (process , spec) } pub fn with_external < P , D : LocalDeploy < 'a > > (self , process : & ExternalProcess < P > , spec : impl ExternalSpec < 'a , D > ,) -> DeployFlow < 'a , D > { self . into_deploy () . with_external (process , spec) } pub fn with_cluster < C , D : LocalDeploy < 'a > > (self , cluster : & Cluster < C > , spec : impl ClusterSpec < 'a , D > ,) -> DeployFlow < 'a , D > { self . into_deploy () . with_cluster (cluster , spec) } pub fn compile < D : Deploy < 'a > + 'a > (self , env : & D :: CompileEnv) -> CompiledFlow < 'a , D :: GraphId > { self . into_deploy :: < D > () . compile (env) } pub fn compile_no_network < D : LocalDeploy < 'a > + 'a > (self) -> CompiledFlow < 'a , D :: GraphId > { self . into_deploy :: < D > () . compile_no_network () } pub fn deploy < D : Deploy < 'a , CompileEnv = () > + 'a > (self , env : & mut D :: InstantiateEnv ,) -> DeployResult < 'a , D > { self . into_deploy :: < D > () . deploy (env) } } } pub mod compiled { pub use std :: collections :: BTreeMap ; pub use std :: marker :: PhantomData ; pub use hydroflow :: scheduled :: graph :: Hydroflow ; pub use hydroflow_lang :: graph :: { partition_graph , HydroflowGraph } ; pub use proc_macro2 :: TokenStream ; pub use quote :: quote ; pub use stageleft :: runtime_support :: FreeVariable ; pub use stageleft :: Quoted ; pub use crate :: builder :: compiled :: CompiledFlow ; # [cfg (stageleft_macro)] impl < ID > CompiledFlow < '_ , ID > { pub fn hydroflow_ir (& self) -> & BTreeMap < usize , HydroflowGraph > { & self . hydroflow_ir } pub fn take_ir (self) -> BTreeMap < usize , HydroflowGraph > { self . hydroflow_ir } } # [cfg (stageleft_macro)] impl < 'a > CompiledFlow < 'a , usize > { pub fn with_dynamic_id (self , id : impl Quoted < 'a , usize >) -> CompiledFlowWithId < 'a > { let hydroflow_crate = proc_macro_crate :: crate_name ("hydroflow_plus") . expect ("hydroflow_plus should be present in `Cargo.toml`") ; let root = match hydroflow_crate { proc_macro_crate :: FoundCrate :: Itself => quote ! { hydroflow_plus } , proc_macro_crate :: FoundCrate :: Name (name) => { let ident = syn :: Ident :: new (& name , proc_macro2 :: Span :: call_site ()) ; quote ! { # ident } } } ; let mut conditioned_tokens = None ; for (subgraph_id , flat_graph) in self . hydroflow_ir { let partitioned_graph = partition_graph (flat_graph) . expect ("Failed to partition (cycle detected).") ; let mut diagnostics = Vec :: new () ; let tokens = partitioned_graph . as_code (& root , true , quote :: quote ! () , & mut diagnostics) ; let my_extra_stmts = self . extra_stmts . get (& subgraph_id) . cloned () . unwrap_or_default () ; if let Some (conditioned_tokens) = conditioned_tokens . as_mut () { * conditioned_tokens = syn :: parse_quote ! { # conditioned_tokens else if __given_id == # subgraph_id { # (# my_extra_stmts) * # tokens } } ; } else { conditioned_tokens = Some (syn :: parse_quote ! { if __given_id == # subgraph_id { # (# my_extra_stmts) * # tokens } }) ; } } let conditioned_tokens : TokenStream = conditioned_tokens . unwrap () ; let id = id . splice_untyped () ; CompiledFlowWithId { tokens : syn :: parse_quote ! ({ let __given_id = # id ; # conditioned_tokens else { panic ! ("Invalid node id: {}" , __given_id) ; } }) , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < 'a > Quoted < 'a , Hydroflow < 'a > > for CompiledFlow < 'a , () > { } # [cfg (stageleft_macro)] impl < 'a > FreeVariable < Hydroflow < 'a > > for CompiledFlow < 'a , () > { fn to_tokens (mut self) -> (Option < TokenStream > , Option < TokenStream >) { let hydroflow_crate = proc_macro_crate :: crate_name ("hydroflow_plus") . expect ("hydroflow_plus should be present in `Cargo.toml`") ; let root = match hydroflow_crate { proc_macro_crate :: FoundCrate :: Itself => quote ! { hydroflow_plus } , proc_macro_crate :: FoundCrate :: Name (name) => { let ident = syn :: Ident :: new (& name , proc_macro2 :: Span :: call_site ()) ; quote ! { # ident } } } ; if self . hydroflow_ir . len () != 1 { panic ! ("Expected exactly one subgraph in the Hydroflow IR") ; } let flat_graph = self . hydroflow_ir . remove (& 0) . unwrap () ; let partitioned_graph = partition_graph (flat_graph) . expect ("Failed to partition (cycle detected).") ; let mut diagnostics = Vec :: new () ; let tokens = partitioned_graph . as_code (& root , true , quote :: quote ! () , & mut diagnostics) ; (None , Some (tokens)) } } pub use crate :: builder :: compiled :: CompiledFlowWithId ; # [cfg (stageleft_macro)] impl < 'a > Quoted < 'a , Hydroflow < 'a > > for CompiledFlowWithId < 'a > { } # [cfg (stageleft_macro)] impl < 'a > FreeVariable < Hydroflow < 'a > > for CompiledFlowWithId < 'a > { fn to_tokens (self) -> (Option < TokenStream > , Option < TokenStream >) { (None , Some (self . tokens)) } } } pub mod deploy { pub use std :: collections :: { BTreeMap , HashMap } ; pub use std :: io :: Error ; pub use std :: marker :: PhantomData ; pub use std :: pin :: Pin ; pub use hydroflow :: bytes :: Bytes ; pub use hydroflow :: futures :: { Sink , Stream } ; pub use proc_macro2 :: Span ; pub use serde :: de :: DeserializeOwned ; pub use serde :: Serialize ; pub use stageleft :: Quoted ; pub use super :: built :: build_inner ; pub use super :: compiled :: CompiledFlow ; pub use crate :: __staged :: deploy :: { ExternalSpec , IntoProcessSpec , LocalDeploy , Node , RegisterPort } ; pub use crate :: __staged :: ir :: HfPlusLeaf ; pub use crate :: __staged :: location :: external_process :: { ExternalBincodeSink , ExternalBincodeStream , ExternalBytesPort , } ; pub use crate :: __staged :: location :: { ExternalProcess , Location , LocationId } ; pub use crate :: __staged :: { Cluster , ClusterSpec , Deploy , Process , ProcessSpec } ; pub use crate :: builder :: deploy :: DeployFlow ; # [cfg (stageleft_macro)] impl < 'a , D : LocalDeploy < 'a > > Drop for DeployFlow < 'a , D > { fn drop (& mut self) { if ! self . used { panic ! ("Dropped DeployFlow without instantiating, you may have forgotten to call `compile` or `deploy`.") ; } } } # [cfg (stageleft_macro)] impl < 'a , D : LocalDeploy < 'a > > DeployFlow < 'a , D > { pub fn ir (& self) -> & Vec < HfPlusLeaf > { & self . ir } pub fn with_process < P > (mut self , process : & Process < P > , spec : impl IntoProcessSpec < 'a , D > ,) -> Self { let tag_name = std :: any :: type_name :: < P > () . to_string () ; self . nodes . insert (process . id , spec . into_process_spec () . build (process . id , & tag_name) ,) ; self } pub fn with_external < P > (mut self , process : & ExternalProcess < P > , spec : impl ExternalSpec < 'a , D > ,) -> Self { let tag_name = std :: any :: type_name :: < P > () . to_string () ; self . externals . insert (process . id , spec . build (process . id , & tag_name)) ; self } pub fn with_cluster < C > (mut self , cluster : & Cluster < C > , spec : impl ClusterSpec < 'a , D >) -> Self { let tag_name = std :: any :: type_name :: < C > () . to_string () ; self . clusters . insert (cluster . id , spec . build (cluster . id , & tag_name)) ; self } pub fn compile_no_network (mut self) -> CompiledFlow < 'a , D :: GraphId > { self . used = true ; CompiledFlow { hydroflow_ir : build_inner (& mut self . ir) , extra_stmts : BTreeMap :: new () , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < 'a , D : Deploy < 'a > > DeployFlow < 'a , D > { pub fn compile (mut self , env : & D :: CompileEnv) -> CompiledFlow < 'a , D :: GraphId > { self . used = true ; let mut seen_tees : HashMap < _ , _ > = HashMap :: new () ; let mut flow_state_networked : Vec < HfPlusLeaf > = std :: mem :: take (& mut self . ir) . into_iter () . map (| leaf | { leaf . compile_network :: < D > (env , & mut seen_tees , & self . nodes , & self . clusters , & self . externals ,) }) . collect () ; let extra_stmts = self . extra_stmts (env) ; CompiledFlow { hydroflow_ir : build_inner (& mut flow_state_networked) , extra_stmts , _phantom : PhantomData , } } fn extra_stmts (& self , env : & < D as Deploy < 'a > > :: CompileEnv) -> BTreeMap < usize , Vec < syn :: Stmt > > { let all_locations_count = self . nodes . len () + self . clusters . len () ; let mut extra_stmts : BTreeMap < usize , Vec < syn :: Stmt > > = BTreeMap :: new () ; for & c_id in self . clusters . keys () { let self_id_ident = syn :: Ident :: new (& format ! ("__hydroflow_plus_cluster_self_id_{}" , c_id) , Span :: call_site () ,) ; let self_id_expr = D :: cluster_self_id (env) . splice_untyped () ; extra_stmts . entry (c_id) . or_default () . push (syn :: parse_quote ! { let # self_id_ident = # self_id_expr ; }) ; for other_location in 0 .. all_locations_count { let other_id_ident = syn :: Ident :: new (& format ! ("__hydroflow_plus_cluster_ids_{}" , c_id) , Span :: call_site () ,) ; let other_id_expr = D :: cluster_ids (env , c_id) . splice_untyped () ; extra_stmts . entry (other_location) . or_default () . push (syn :: parse_quote ! { let # other_id_ident = # other_id_expr ; }) ; } } extra_stmts } } # [cfg (stageleft_macro)] impl < 'a , D : Deploy < 'a , CompileEnv = () > > DeployFlow < 'a , D > { # [must_use] pub fn deploy (mut self , env : & mut D :: InstantiateEnv) -> DeployResult < 'a , D > { self . used = true ; let mut seen_tees_instantiate : HashMap < _ , _ > = HashMap :: new () ; let mut flow_state_networked : Vec < HfPlusLeaf > = std :: mem :: take (& mut self . ir) . into_iter () . map (| leaf | { leaf . compile_network :: < D > (& () , & mut seen_tees_instantiate , & self . nodes , & self . clusters , & self . externals ,) }) . collect () ; let mut compiled = build_inner (& mut flow_state_networked) ; let mut extra_stmts = self . extra_stmts (& ()) ; let mut meta = D :: Meta :: default () ; let (mut processes , mut clusters , mut externals) = (std :: mem :: take (& mut self . nodes) . into_iter () . map (| (node_id , node) | { node . instantiate (env , & mut meta , compiled . remove (& node_id) . unwrap () , extra_stmts . remove (& node_id) . unwrap_or_default () ,) ; (node_id , node) }) . collect :: < HashMap < _ , _ > > () , std :: mem :: take (& mut self . clusters) . into_iter () . map (| (cluster_id , cluster) | { cluster . instantiate (env , & mut meta , compiled . remove (& cluster_id) . unwrap () , extra_stmts . remove (& cluster_id) . unwrap_or_default () ,) ; (cluster_id , cluster) }) . collect :: < HashMap < _ , _ > > () , std :: mem :: take (& mut self . externals) . into_iter () . map (| (external_id , external) | { external . instantiate (env , & mut meta , compiled . remove (& external_id) . unwrap () , extra_stmts . remove (& external_id) . unwrap_or_default () ,) ; (external_id , external) }) . collect :: < HashMap < _ , _ > > () ,) ; for node in processes . values_mut () { node . update_meta (& meta) ; } for cluster in clusters . values_mut () { cluster . update_meta (& meta) ; } for external in externals . values_mut () { external . update_meta (& meta) ; } let mut seen_tees_connect = HashMap :: new () ; for leaf in flow_state_networked { leaf . connect_network (& mut seen_tees_connect) ; } DeployResult { processes , clusters , externals , } } } pub use crate :: builder :: deploy :: DeployResult ; # [cfg (stageleft_macro)] impl < 'a , D : Deploy < 'a > > DeployResult < 'a , D > { pub fn get_process < P > (& self , p : & Process < P >) -> & D :: Process { let id = match p . id () { LocationId :: Process (id) => id , _ => panic ! ("Process ID expected") , } ; self . processes . get (& id) . unwrap () } pub fn get_cluster < C > (& self , c : & Cluster < 'a , C >) -> & D :: Cluster { let id = match c . id () { LocationId :: Cluster (id) => id , _ => panic ! ("Cluster ID expected") , } ; self . clusters . get (& id) . unwrap () } pub fn get_external < P > (& self , p : & ExternalProcess < P >) -> & D :: ExternalProcess { self . externals . get (& p . id) . unwrap () } pub fn raw_port (& self , port : ExternalBytesPort) -> D :: ExternalRawPort { self . externals . get (& port . process_id) . unwrap () . raw_port (port . port_id) } pub async fn connect_sink_bytes (& self , port : ExternalBytesPort ,) -> Pin < Box < dyn Sink < Bytes , Error = Error > > > { self . externals . get (& port . process_id) . unwrap () . as_bytes_sink (port . port_id) . await } pub async fn connect_sink_bincode < T : Serialize + DeserializeOwned + 'static > (& self , port : ExternalBincodeSink < T > ,) -> Pin < Box < dyn Sink < T , Error = Error > > > { self . externals . get (& port . process_id) . unwrap () . as_bincode_sink (port . port_id) . await } pub async fn connect_source_bytes (& self , port : ExternalBytesPort ,) -> Pin < Box < dyn Stream < Item = Bytes > > > { self . externals . get (& port . process_id) . unwrap () . as_bytes_source (port . port_id) . await } pub async fn connect_source_bincode < T : Serialize + DeserializeOwned + 'static > (& self , port : ExternalBincodeStream < T > ,) -> Pin < Box < dyn Stream < Item = T > > > { self . externals . get (& port . process_id) . unwrap () . as_bincode_source (port . port_id) . await } } } pub use crate :: builder :: FlowStateInner ; pub type FlowState = Rc < RefCell < FlowStateInner > > ; pub const FLOW_USED_MESSAGE : & str = "Attempted to add a leaf to a flow that has already been finalized. No leaves can be added after the flow has been compiled." ; pub use crate :: builder :: FlowBuilder ; # [cfg (stageleft_macro)] impl Drop for FlowBuilder < '_ > { fn drop (& mut self) { if ! self . finalized { panic ! ("Dropped FlowBuilder without finalizing, you may have forgotten to call `with_default_optimize`, `optimize_with`, or `finalize`.") ; } } } # [cfg (stageleft_macro)] impl QuotedContext for FlowBuilder < '_ > { fn create () -> Self { FlowBuilder :: new () } } # [cfg (stageleft_macro)] impl < 'a > FlowBuilder < 'a > { # [expect (clippy :: new_without_default , reason = "call `new` explicitly, not `default`")] pub fn new () -> FlowBuilder < 'a > { FlowBuilder { flow_state : Rc :: new (RefCell :: new (FlowStateInner { leaves : Some (vec ! []) , next_external_out : 0 , cycle_counts : HashMap :: new () , next_clock_id : 0 , })) , nodes : RefCell :: new (vec ! []) , clusters : RefCell :: new (vec ! []) , next_node_id : RefCell :: new (0) , finalized : false , _phantom : PhantomData , } } pub fn finalize (mut self) -> built :: BuiltFlow < 'a > { self . finalized = true ; built :: BuiltFlow { ir : self . flow_state . borrow_mut () . leaves . take () . unwrap () , processes : self . nodes . replace (vec ! []) , clusters : self . clusters . replace (vec ! []) , used : false , _phantom : PhantomData , } } pub fn with_default_optimize < D : LocalDeploy < 'a > > (self) -> DeployFlow < 'a , D > { self . finalize () . with_default_optimize () } pub fn optimize_with (self , f : impl FnOnce (Vec < HfPlusLeaf >) -> Vec < HfPlusLeaf > ,) -> built :: BuiltFlow < 'a > { self . finalize () . optimize_with (f) } pub fn flow_state (& self) -> & FlowState { & self . flow_state } pub fn process < P > (& self) -> Process < 'a , P > { let mut next_node_id = self . next_node_id . borrow_mut () ; let id = * next_node_id ; * next_node_id += 1 ; self . nodes . borrow_mut () . push (id) ; Process { id , flow_state : self . flow_state () . clone () , _phantom : PhantomData , } } pub fn external_process < P > (& self) -> ExternalProcess < 'a , P > { let mut next_node_id = self . next_node_id . borrow_mut () ; let id = * next_node_id ; * next_node_id += 1 ; self . nodes . borrow_mut () . push (id) ; ExternalProcess { id , flow_state : self . flow_state () . clone () , _phantom : PhantomData , } } pub fn cluster < C > (& self) -> Cluster < 'a , C > { let mut next_node_id = self . next_node_id . borrow_mut () ; let id = * next_node_id ; * next_node_id += 1 ; self . clusters . borrow_mut () . push (id) ; Cluster { id , flow_state : self . flow_state () . clone () , _phantom : PhantomData , } } pub fn runtime_context (& self) -> RuntimeContext < 'a > { RuntimeContext :: new () } pub fn with_process < P , D : LocalDeploy < 'a > > (self , process : & Process < P > , spec : impl IntoProcessSpec < 'a , D > ,) -> DeployFlow < 'a , D > { self . with_default_optimize () . with_process (process , spec) } pub fn with_external < P , D : LocalDeploy < 'a > > (self , process : & ExternalProcess < P > , spec : impl ExternalSpec < 'a , D > ,) -> DeployFlow < 'a , D > { self . with_default_optimize () . with_external (process , spec) } pub fn with_cluster < C , D : LocalDeploy < 'a > > (self , cluster : & Cluster < C > , spec : impl ClusterSpec < 'a , D > ,) -> DeployFlow < 'a , D > { self . with_default_optimize () . with_cluster (cluster , spec) } pub fn compile < D : Deploy < 'a > + 'a > (self , env : & D :: CompileEnv) -> CompiledFlow < 'a , D :: GraphId > { self . with_default_optimize :: < D > () . compile (env) } pub fn compile_no_network < D : LocalDeploy < 'a > + 'a > (self) -> CompiledFlow < 'a , D :: GraphId > { self . with_default_optimize :: < D > () . compile_no_network () } pub fn deploy < D : Deploy < 'a , CompileEnv = () > + 'a > (self , env : & mut D :: InstantiateEnv ,) -> DeployResult < 'a , D > { self . with_default_optimize () . deploy (env) } } } pub use builder :: FlowBuilder ; pub mod ir { pub use core :: panic ; pub use std :: cell :: RefCell ; pub use std :: collections :: { BTreeMap , HashMap } ; pub use std :: fmt :: Debug ; pub use std :: ops :: Deref ; pub use std :: rc :: Rc ; pub use hydroflow_lang :: graph :: FlatGraphBuilder ; pub use hydroflow_lang :: parse :: Pipeline ; pub use proc_macro2 :: { Span , TokenStream } ; pub use quote :: ToTokens ; pub use syn :: parse_quote ; pub use crate :: __staged :: deploy :: { Deploy , RegisterPort } ; pub use crate :: __staged :: location :: LocationId ; pub use crate :: ir :: DebugExpr ; # [cfg (stageleft_macro)] impl From < syn :: Expr > for DebugExpr { fn from (expr : syn :: Expr) -> DebugExpr { DebugExpr (expr) } } # [cfg (stageleft_macro)] impl Deref for DebugExpr { type Target = syn :: Expr ; fn deref (& self) -> & Self :: Target { & self . 0 } } # [cfg (stageleft_macro)] impl ToTokens for DebugExpr { fn to_tokens (& self , tokens : & mut TokenStream) { self . 0 . to_tokens (tokens) ; } } # [cfg (stageleft_macro)] impl Debug for DebugExpr { fn fmt (& self , f : & mut std :: fmt :: Formatter < '_ >) -> std :: fmt :: Result { write ! (f , "{}" , self . 0 . to_token_stream ()) } } pub use crate :: ir :: DebugInstantiate ; # [cfg (stageleft_macro)] impl Debug for DebugInstantiate { fn fmt (& self , f : & mut std :: fmt :: Formatter < '_ >) -> std :: fmt :: Result { write ! (f , "") } } pub use crate :: ir :: DebugPipelineFn ; # [cfg (stageleft_macro)] impl Debug for DebugPipelineFn { fn fmt (& self , f : & mut std :: fmt :: Formatter < '_ >) -> std :: fmt :: Result { write ! (f , "") } } pub use crate :: ir :: HfPlusSource ; pub use crate :: ir :: HfPlusLeaf ; # [cfg (stageleft_macro)] impl HfPlusLeaf { pub fn compile_network < 'a , D : Deploy < 'a > + 'a > (self , compile_env : & D :: CompileEnv , seen_tees : & mut SeenTees , nodes : & HashMap < usize , D :: Process > , clusters : & HashMap < usize , D :: Cluster > , externals : & HashMap < usize , D :: ExternalProcess > ,) -> HfPlusLeaf { self . transform_children (| n , s | { n . compile_network :: < D > (compile_env , s , nodes , clusters , externals) ; } , seen_tees ,) } pub fn connect_network (self , seen_tees : & mut SeenTees) -> HfPlusLeaf { self . transform_children (| n , s | { n . connect_network (s) ; } , seen_tees ,) } pub fn transform_children (self , mut transform : impl FnMut (& mut HfPlusNode , & mut SeenTees) , seen_tees : & mut SeenTees ,) -> HfPlusLeaf { match self { HfPlusLeaf :: ForEach { f , mut input } => { transform (& mut input , seen_tees) ; HfPlusLeaf :: ForEach { f , input } } HfPlusLeaf :: DestSink { sink , mut input } => { transform (& mut input , seen_tees) ; HfPlusLeaf :: DestSink { sink , input } } HfPlusLeaf :: CycleSink { ident , location_kind , mut input , } => { transform (& mut input , seen_tees) ; HfPlusLeaf :: CycleSink { ident , location_kind , input , } } } } pub fn emit (& self , graph_builders : & mut BTreeMap < usize , FlatGraphBuilder > , built_tees : & mut HashMap < * const RefCell < HfPlusNode > , (syn :: Ident , usize) > , next_stmt_id : & mut usize ,) { match self { HfPlusLeaf :: ForEach { f , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; graph_builders . entry (input_location_id) . or_default () . add_statement (parse_quote ! { # input_ident -> for_each (# f) ; }) ; } HfPlusLeaf :: DestSink { sink , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; graph_builders . entry (input_location_id) . or_default () . add_statement (parse_quote ! { # input_ident -> dest_sink (# sink) ; }) ; } HfPlusLeaf :: CycleSink { ident , location_kind , input , } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let location_id = match location_kind . root () { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (_) => panic ! () , } ; assert_eq ! (input_location_id , * location_id , "cycle_sink location mismatch") ; graph_builders . entry (* location_id) . or_default () . add_statement (parse_quote ! { # ident = # input_ident ; }) ; } } } } type PrintedTees = RefCell < Option < (usize , HashMap < * const RefCell < HfPlusNode > , usize >) > > ; thread_local ! { static PRINTED_TEES : PrintedTees = const { RefCell :: new (None) } ; } pub fn dbg_dedup_tee < T > (f : impl FnOnce () -> T) -> T { PRINTED_TEES . with (| printed_tees | { let mut printed_tees_mut = printed_tees . borrow_mut () ; * printed_tees_mut = Some ((0 , HashMap :: new ())) ; drop (printed_tees_mut) ; let ret = f () ; let mut printed_tees_mut = printed_tees . borrow_mut () ; * printed_tees_mut = None ; ret }) } pub use crate :: ir :: TeeNode ; # [cfg (stageleft_macro)] impl Debug for TeeNode { fn fmt (& self , f : & mut std :: fmt :: Formatter < '_ >) -> std :: fmt :: Result { PRINTED_TEES . with (| printed_tees | { let mut printed_tees_mut_borrow = printed_tees . borrow_mut () ; let printed_tees_mut = printed_tees_mut_borrow . as_mut () ; if let Some (printed_tees_mut) = printed_tees_mut { if let Some (existing) = printed_tees_mut . 1 . get (& (self . 0 . as_ref () as * const RefCell < HfPlusNode >)) { write ! (f , "" , existing) } else { let next_id = printed_tees_mut . 0 ; printed_tees_mut . 0 += 1 ; printed_tees_mut . 1 . insert (self . 0 . as_ref () as * const RefCell < HfPlusNode > , next_id) ; drop (printed_tees_mut_borrow) ; write ! (f , ": " , next_id) ? ; Debug :: fmt (& self . 0 . borrow () , f) } } else { drop (printed_tees_mut_borrow) ; write ! (f , ": ") ? ; Debug :: fmt (& self . 0 . borrow () , f) } }) } } pub use crate :: ir :: HfPlusNode ; pub type SeenTees = HashMap < * const RefCell < HfPlusNode > , Rc < RefCell < HfPlusNode > > > ; # [cfg (stageleft_macro)] impl < 'a > HfPlusNode { pub fn compile_network < D : Deploy < 'a > + 'a > (& mut self , compile_env : & D :: CompileEnv , seen_tees : & mut SeenTees , nodes : & HashMap < usize , D :: Process > , clusters : & HashMap < usize , D :: Cluster > , externals : & HashMap < usize , D :: ExternalProcess > ,) { self . transform_children (| n , s | n . compile_network :: < D > (compile_env , s , nodes , clusters , externals) , seen_tees ,) ; if let HfPlusNode :: Network { from_location , from_key , to_location , to_key , instantiate_fn , .. } = self { let (sink_expr , source_expr , connect_fn) = match instantiate_fn { DebugInstantiate :: Building () => instantiate_network :: < D > (from_location , * from_key , to_location , * to_key , nodes , clusters , externals , compile_env ,) , DebugInstantiate :: Finalized (_ , _ , _) => panic ! ("network already finalized") , } ; * instantiate_fn = DebugInstantiate :: Finalized (sink_expr , source_expr , Some (connect_fn)) ; } } pub fn connect_network (& mut self , seen_tees : & mut SeenTees) { self . transform_children (| n , s | n . connect_network (s) , seen_tees) ; if let HfPlusNode :: Network { instantiate_fn , .. } = self { match instantiate_fn { DebugInstantiate :: Building () => panic ! ("network not built") , DebugInstantiate :: Finalized (_ , _ , connect_fn) => { connect_fn . take () . unwrap () () ; } } } } pub fn transform_bottom_up < C > (& mut self , mut transform : impl FnMut (& mut HfPlusNode , & mut C) + Copy , seen_tees : & mut SeenTees , ctx : & mut C ,) { self . transform_children (| n , s | n . transform_bottom_up (transform , s , ctx) , seen_tees) ; transform (self , ctx) } # [inline (always)] pub fn transform_children (& mut self , mut transform : impl FnMut (& mut HfPlusNode , & mut SeenTees) , seen_tees : & mut SeenTees ,) { match self { HfPlusNode :: Placeholder => { panic ! () ; } HfPlusNode :: Source { .. } => { } HfPlusNode :: CycleSource { .. } => { } HfPlusNode :: Tee { inner } => { if let Some (transformed) = seen_tees . get (& (inner . 0 . as_ref () as * const RefCell < HfPlusNode >)) { * inner = TeeNode (transformed . clone ()) ; } else { let transformed_cell = Rc :: new (RefCell :: new (HfPlusNode :: Placeholder)) ; seen_tees . insert (inner . 0 . as_ref () as * const RefCell < HfPlusNode > , transformed_cell . clone () ,) ; let mut orig = inner . 0 . replace (HfPlusNode :: Placeholder) ; transform (& mut orig , seen_tees) ; * transformed_cell . borrow_mut () = orig ; * inner = TeeNode (transformed_cell) ; } } HfPlusNode :: Persist (inner) => transform (inner . as_mut () , seen_tees) , HfPlusNode :: Unpersist (inner) => transform (inner . as_mut () , seen_tees) , HfPlusNode :: Delta (inner) => transform (inner . as_mut () , seen_tees) , HfPlusNode :: Union (left , right) => { transform (left . as_mut () , seen_tees) ; transform (right . as_mut () , seen_tees) ; } HfPlusNode :: CrossProduct (left , right) => { transform (left . as_mut () , seen_tees) ; transform (right . as_mut () , seen_tees) ; } HfPlusNode :: CrossSingleton (left , right) => { transform (left . as_mut () , seen_tees) ; transform (right . as_mut () , seen_tees) ; } HfPlusNode :: Join (left , right) => { transform (left . as_mut () , seen_tees) ; transform (right . as_mut () , seen_tees) ; } HfPlusNode :: Difference (left , right) => { transform (left . as_mut () , seen_tees) ; transform (right . as_mut () , seen_tees) ; } HfPlusNode :: AntiJoin (left , right) => { transform (left . as_mut () , seen_tees) ; transform (right . as_mut () , seen_tees) ; } HfPlusNode :: Map { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: FlatMap { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Filter { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: FilterMap { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Sort (input) => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: DeferTick (input) => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Enumerate (input) => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Inspect { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Unique (input) => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Fold { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: FoldKeyed { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Reduce { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: ReduceKeyed { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Network { input , .. } => { transform (input . as_mut () , seen_tees) ; } } } pub fn emit (& self , graph_builders : & mut BTreeMap < usize , FlatGraphBuilder > , built_tees : & mut HashMap < * const RefCell < HfPlusNode > , (syn :: Ident , usize) > , next_stmt_id : & mut usize ,) -> (syn :: Ident , usize) { match self { HfPlusNode :: Placeholder => { panic ! () } HfPlusNode :: Persist (inner) => { let (inner_ident , location) = inner . emit (graph_builders , built_tees , next_stmt_id) ; let persist_id = * next_stmt_id ; * next_stmt_id += 1 ; let persist_ident = syn :: Ident :: new (& format ! ("stream_{}" , persist_id) , Span :: call_site ()) ; let builder = graph_builders . entry (location) . or_default () ; builder . add_statement (parse_quote ! { # persist_ident = # inner_ident -> persist ::<'static > () ; }) ; (persist_ident , location) } HfPlusNode :: Unpersist (_) => { panic ! ("Unpersist is a marker node and should have been optimized away. This is likely a compiler bug.") } HfPlusNode :: Delta (inner) => { let (inner_ident , location) = inner . emit (graph_builders , built_tees , next_stmt_id) ; let delta_id = * next_stmt_id ; * next_stmt_id += 1 ; let delta_ident = syn :: Ident :: new (& format ! ("stream_{}" , delta_id) , Span :: call_site ()) ; let builder = graph_builders . entry (location) . or_default () ; builder . add_statement (parse_quote ! { # delta_ident = # inner_ident -> multiset_delta () ; }) ; (delta_ident , location) } HfPlusNode :: Source { source , location_kind , } => { let location_id = match location_kind { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (id) => id , } ; if let HfPlusSource :: ExternalNetwork () = source { (syn :: Ident :: new ("DUMMY" , Span :: call_site ()) , * location_id) } else { let source_id = * next_stmt_id ; * next_stmt_id += 1 ; let source_ident = syn :: Ident :: new (& format ! ("stream_{}" , source_id) , Span :: call_site ()) ; let source_stmt = match source { HfPlusSource :: Stream (expr) => { parse_quote ! { # source_ident = source_stream (# expr) ; } } HfPlusSource :: ExternalNetwork () => { unreachable ! () } HfPlusSource :: Iter (expr) => { parse_quote ! { # source_ident = source_iter (# expr) ; } } HfPlusSource :: Spin () => { parse_quote ! { # source_ident = spin () ; } } } ; graph_builders . entry (* location_id) . or_default () . add_statement (source_stmt) ; (source_ident , * location_id) } } HfPlusNode :: CycleSource { ident , location_kind , } => { let location_id = match location_kind . root () { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (_) => panic ! () , } ; (ident . clone () , * location_id) } HfPlusNode :: Tee { inner } => { if let Some (ret) = built_tees . get (& (inner . 0 . as_ref () as * const RefCell < HfPlusNode >)) { ret . clone () } else { let (inner_ident , inner_location_id) = inner . 0 . borrow () . emit (graph_builders , built_tees , next_stmt_id) ; let tee_id = * next_stmt_id ; * next_stmt_id += 1 ; let tee_ident = syn :: Ident :: new (& format ! ("stream_{}" , tee_id) , Span :: call_site ()) ; let builder = graph_builders . entry (inner_location_id) . or_default () ; builder . add_statement (parse_quote ! { # tee_ident = # inner_ident -> tee () ; }) ; built_tees . insert (inner . 0 . as_ref () as * const RefCell < HfPlusNode > , (tee_ident . clone () , inner_location_id) ,) ; (tee_ident , inner_location_id) } } HfPlusNode :: Union (left , right) => { let (left_ident , left_location_id) = left . emit (graph_builders , built_tees , next_stmt_id) ; let (right_ident , right_location_id) = right . emit (graph_builders , built_tees , next_stmt_id) ; assert_eq ! (left_location_id , right_location_id , "union inputs must be in the same location") ; let union_id = * next_stmt_id ; * next_stmt_id += 1 ; let union_ident = syn :: Ident :: new (& format ! ("stream_{}" , union_id) , Span :: call_site ()) ; let builder = graph_builders . entry (left_location_id) . or_default () ; builder . add_statement (parse_quote ! { # union_ident = union () ; }) ; builder . add_statement (parse_quote ! { # left_ident -> [0] # union_ident ; }) ; builder . add_statement (parse_quote ! { # right_ident -> [1] # union_ident ; }) ; (union_ident , left_location_id) } HfPlusNode :: CrossSingleton (left , right) => { let (left_ident , left_location_id) = left . emit (graph_builders , built_tees , next_stmt_id) ; let (right_ident , right_location_id) = right . emit (graph_builders , built_tees , next_stmt_id) ; assert_eq ! (left_location_id , right_location_id , "cross_singleton inputs must be in the same location") ; let union_id = * next_stmt_id ; * next_stmt_id += 1 ; let cross_ident = syn :: Ident :: new (& format ! ("stream_{}" , union_id) , Span :: call_site ()) ; let builder = graph_builders . entry (left_location_id) . or_default () ; builder . add_statement (parse_quote ! { # cross_ident = cross_singleton () ; }) ; builder . add_statement (parse_quote ! { # left_ident -> [input] # cross_ident ; }) ; builder . add_statement (parse_quote ! { # right_ident -> [single] # cross_ident ; }) ; (cross_ident , left_location_id) } HfPlusNode :: CrossProduct (..) | HfPlusNode :: Join (..) => { let operator : syn :: Ident = if matches ! (self , HfPlusNode :: CrossProduct (..)) { parse_quote ! (cross_join_multiset) } else { parse_quote ! (join_multiset) } ; let (HfPlusNode :: CrossProduct (left , right) | HfPlusNode :: Join (left , right)) = self else { unreachable ! () } ; let (left_inner , left_was_persist) = if let HfPlusNode :: Persist (left) = left . as_ref () { (left , true) } else { (left , false) } ; let (right_inner , right_was_persist) = if let HfPlusNode :: Persist (right) = right . as_ref () { (right , true) } else { (right , false) } ; let (left_ident , left_location_id) = left_inner . emit (graph_builders , built_tees , next_stmt_id) ; let (right_ident , right_location_id) = right_inner . emit (graph_builders , built_tees , next_stmt_id) ; assert_eq ! (left_location_id , right_location_id , "join / cross product inputs must be in the same location") ; let stream_id = * next_stmt_id ; * next_stmt_id += 1 ; let stream_ident = syn :: Ident :: new (& format ! ("stream_{}" , stream_id) , Span :: call_site ()) ; let builder = graph_builders . entry (left_location_id) . or_default () ; match (left_was_persist , right_was_persist) { (true , true) => { builder . add_statement (parse_quote ! { # stream_ident = # operator ::<'static , 'static > () ; }) ; } (true , false) => { builder . add_statement (parse_quote ! { # stream_ident = # operator ::<'static , 'tick > () ; }) ; } (false , true) => { builder . add_statement (parse_quote ! { # stream_ident = # operator ::<'tick , 'static > () ; }) ; } (false , false) => { builder . add_statement (parse_quote ! { # stream_ident = # operator ::<'tick , 'tick > () ; }) ; } } ; builder . add_statement (parse_quote ! { # left_ident -> [0] # stream_ident ; }) ; builder . add_statement (parse_quote ! { # right_ident -> [1] # stream_ident ; }) ; (stream_ident , left_location_id) } HfPlusNode :: Difference (..) | HfPlusNode :: AntiJoin (..) => { let operator : syn :: Ident = if matches ! (self , HfPlusNode :: Difference (..)) { parse_quote ! (difference_multiset) } else { parse_quote ! (anti_join_multiset) } ; let (HfPlusNode :: Difference (left , right) | HfPlusNode :: AntiJoin (left , right)) = self else { unreachable ! () } ; let (right , right_was_persist) = if let HfPlusNode :: Persist (right) = right . as_ref () { (right , true) } else { (right , false) } ; let (left_ident , left_location_id) = left . emit (graph_builders , built_tees , next_stmt_id) ; let (right_ident , right_location_id) = right . emit (graph_builders , built_tees , next_stmt_id) ; assert_eq ! (left_location_id , right_location_id , "difference / anti join inputs must be in the same location") ; let stream_id = * next_stmt_id ; * next_stmt_id += 1 ; let stream_ident = syn :: Ident :: new (& format ! ("stream_{}" , stream_id) , Span :: call_site ()) ; let builder = graph_builders . entry (left_location_id) . or_default () ; if right_was_persist { builder . add_statement (parse_quote ! { # stream_ident = # operator ::<'tick , 'static > () ; }) ; } else { builder . add_statement (parse_quote ! { # stream_ident = # operator ::<'tick , 'tick > () ; }) ; } builder . add_statement (parse_quote ! { # left_ident -> [pos] # stream_ident ; }) ; builder . add_statement (parse_quote ! { # right_ident -> [neg] # stream_ident ; }) ; (stream_ident , left_location_id) } HfPlusNode :: Map { f , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let map_id = * next_stmt_id ; * next_stmt_id += 1 ; let map_ident = syn :: Ident :: new (& format ! ("stream_{}" , map_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # map_ident = # input_ident -> map (# f) ; }) ; (map_ident , input_location_id) } HfPlusNode :: FlatMap { f , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let flat_map_id = * next_stmt_id ; * next_stmt_id += 1 ; let flat_map_ident = syn :: Ident :: new (& format ! ("stream_{}" , flat_map_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # flat_map_ident = # input_ident -> flat_map (# f) ; }) ; (flat_map_ident , input_location_id) } HfPlusNode :: Filter { f , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let filter_id = * next_stmt_id ; * next_stmt_id += 1 ; let filter_ident = syn :: Ident :: new (& format ! ("stream_{}" , filter_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # filter_ident = # input_ident -> filter (# f) ; }) ; (filter_ident , input_location_id) } HfPlusNode :: FilterMap { f , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let filter_map_id = * next_stmt_id ; * next_stmt_id += 1 ; let filter_map_ident = syn :: Ident :: new (& format ! ("stream_{}" , filter_map_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # filter_map_ident = # input_ident -> filter_map (# f) ; }) ; (filter_map_ident , input_location_id) } HfPlusNode :: Sort (input) => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let sort_id = * next_stmt_id ; * next_stmt_id += 1 ; let sort_ident = syn :: Ident :: new (& format ! ("stream_{}" , sort_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # sort_ident = # input_ident -> sort () ; }) ; (sort_ident , input_location_id) } HfPlusNode :: DeferTick (input) => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let defer_tick_id = * next_stmt_id ; * next_stmt_id += 1 ; let defer_tick_ident = syn :: Ident :: new (& format ! ("stream_{}" , defer_tick_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # defer_tick_ident = # input_ident -> defer_tick_lazy () ; }) ; (defer_tick_ident , input_location_id) } HfPlusNode :: Enumerate (input) => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let enumerate_id = * next_stmt_id ; * next_stmt_id += 1 ; let enumerate_ident = syn :: Ident :: new (& format ! ("stream_{}" , enumerate_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # enumerate_ident = # input_ident -> enumerate () ; }) ; (enumerate_ident , input_location_id) } HfPlusNode :: Inspect { f , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let inspect_id = * next_stmt_id ; * next_stmt_id += 1 ; let inspect_ident = syn :: Ident :: new (& format ! ("stream_{}" , inspect_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # inspect_ident = # input_ident -> inspect (# f) ; }) ; (inspect_ident , input_location_id) } HfPlusNode :: Unique (input) => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let unique_id = * next_stmt_id ; * next_stmt_id += 1 ; let unique_ident = syn :: Ident :: new (& format ! ("stream_{}" , unique_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # unique_ident = # input_ident -> unique ::<'tick > () ; }) ; (unique_ident , input_location_id) } HfPlusNode :: Fold { .. } | HfPlusNode :: FoldKeyed { .. } => { let operator : syn :: Ident = if matches ! (self , HfPlusNode :: Fold { .. }) { parse_quote ! (fold) } else { parse_quote ! (fold_keyed) } ; let (HfPlusNode :: Fold { init , acc , input } | HfPlusNode :: FoldKeyed { init , acc , input }) = self else { unreachable ! () } ; let (input , input_was_persist) = if let HfPlusNode :: Persist (input) = input . as_ref () { (input , true) } else { (input , false) } ; let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let reduce_id = * next_stmt_id ; * next_stmt_id += 1 ; let fold_ident = syn :: Ident :: new (& format ! ("stream_{}" , reduce_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; if input_was_persist { builder . add_statement (parse_quote ! { # fold_ident = # input_ident -> # operator ::<'static > (# init , # acc) ; }) ; } else { builder . add_statement (parse_quote ! { # fold_ident = # input_ident -> # operator ::<'tick > (# init , # acc) ; }) ; } (fold_ident , input_location_id) } HfPlusNode :: Reduce { .. } | HfPlusNode :: ReduceKeyed { .. } => { let operator : syn :: Ident = if matches ! (self , HfPlusNode :: Reduce { .. }) { parse_quote ! (reduce) } else { parse_quote ! (reduce_keyed) } ; let (HfPlusNode :: Reduce { f , input } | HfPlusNode :: ReduceKeyed { f , input }) = self else { unreachable ! () } ; let (input , input_was_persist) = if let HfPlusNode :: Persist (input) = input . as_ref () { (input , true) } else { (input , false) } ; let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let reduce_id = * next_stmt_id ; * next_stmt_id += 1 ; let reduce_ident = syn :: Ident :: new (& format ! ("stream_{}" , reduce_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; if input_was_persist { builder . add_statement (parse_quote ! { # reduce_ident = # input_ident -> # operator ::<'static > (# f) ; }) ; } else { builder . add_statement (parse_quote ! { # reduce_ident = # input_ident -> # operator ::<'tick > (# f) ; }) ; } (reduce_ident , input_location_id) } HfPlusNode :: Network { from_location : _ , from_key : _ , to_location , to_key : _ , serialize_pipeline , instantiate_fn , deserialize_pipeline , input , } => { let (sink_expr , source_expr , _connect_fn) = match instantiate_fn { DebugInstantiate :: Building () => { panic ! ("Expected the network to be finalized") } DebugInstantiate :: Finalized (sink , source , connect_fn) => { (sink , source , connect_fn) } } ; let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let sender_builder = graph_builders . entry (input_location_id) . or_default () ; if let Some (serialize_pipeline) = serialize_pipeline { sender_builder . add_statement (parse_quote ! { # input_ident -> # serialize_pipeline -> dest_sink (# sink_expr) ; }) ; } else { sender_builder . add_statement (parse_quote ! { # input_ident -> dest_sink (# sink_expr) ; }) ; } let to_id = match to_location { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (id) => id , } ; let receiver_builder = graph_builders . entry (* to_id) . or_default () ; let receiver_stream_id = * next_stmt_id ; * next_stmt_id += 1 ; let receiver_stream_ident = syn :: Ident :: new (& format ! ("stream_{}" , receiver_stream_id) , Span :: call_site ()) ; if let Some (deserialize_pipeline) = deserialize_pipeline { receiver_builder . add_statement (parse_quote ! { # receiver_stream_ident = source_stream (# source_expr) -> # deserialize_pipeline ; }) ; } else { receiver_builder . add_statement (parse_quote ! { # receiver_stream_ident = source_stream (# source_expr) ; }) ; } (receiver_stream_ident , * to_id) } } } } # [expect (clippy :: too_many_arguments , reason = "networking internals")] pub fn instantiate_network < 'a , D : Deploy < 'a > + 'a > (from_location : & mut LocationId , from_key : Option < usize > , to_location : & mut LocationId , to_key : Option < usize > , nodes : & HashMap < usize , D :: Process > , clusters : & HashMap < usize , D :: Cluster > , externals : & HashMap < usize , D :: ExternalProcess > , compile_env : & D :: CompileEnv ,) -> (syn :: Expr , syn :: Expr , Box < dyn FnOnce () >) { let ((sink , source) , connect_fn) = match (from_location , to_location) { (LocationId :: Process (from) , LocationId :: Process (to)) => { let from_node = nodes . get (from) . unwrap_or_else (| | { panic ! ("A process used in the graph was not instantiated: {}" , from) }) . clone () ; let to_node = nodes . get (to) . unwrap_or_else (| | { panic ! ("A process used in the graph was not instantiated: {}" , to) }) . clone () ; let sink_port = D :: allocate_process_port (& from_node) ; let source_port = D :: allocate_process_port (& to_node) ; (D :: o2o_sink_source (compile_env , & from_node , & sink_port , & to_node , & source_port) , D :: o2o_connect (& from_node , & sink_port , & to_node , & source_port) ,) } (LocationId :: Process (from) , LocationId :: Cluster (to)) => { let from_node = nodes . get (from) . unwrap_or_else (| | { panic ! ("A process used in the graph was not instantiated: {}" , from) }) . clone () ; let to_node = clusters . get (to) . unwrap_or_else (| | { panic ! ("A cluster used in the graph was not instantiated: {}" , to) }) . clone () ; let sink_port = D :: allocate_process_port (& from_node) ; let source_port = D :: allocate_cluster_port (& to_node) ; (D :: o2m_sink_source (compile_env , & from_node , & sink_port , & to_node , & source_port) , D :: o2m_connect (& from_node , & sink_port , & to_node , & source_port) ,) } (LocationId :: Cluster (from) , LocationId :: Process (to)) => { let from_node = clusters . get (from) . unwrap_or_else (| | { panic ! ("A cluster used in the graph was not instantiated: {}" , from) }) . clone () ; let to_node = nodes . get (to) . unwrap_or_else (| | { panic ! ("A process used in the graph was not instantiated: {}" , to) }) . clone () ; let sink_port = D :: allocate_cluster_port (& from_node) ; let source_port = D :: allocate_process_port (& to_node) ; (D :: m2o_sink_source (compile_env , & from_node , & sink_port , & to_node , & source_port) , D :: m2o_connect (& from_node , & sink_port , & to_node , & source_port) ,) } (LocationId :: Cluster (from) , LocationId :: Cluster (to)) => { let from_node = clusters . get (from) . unwrap_or_else (| | { panic ! ("A cluster used in the graph was not instantiated: {}" , from) }) . clone () ; let to_node = clusters . get (to) . unwrap_or_else (| | { panic ! ("A cluster used in the graph was not instantiated: {}" , to) }) . clone () ; let sink_port = D :: allocate_cluster_port (& from_node) ; let source_port = D :: allocate_cluster_port (& to_node) ; (D :: m2m_sink_source (compile_env , & from_node , & sink_port , & to_node , & source_port) , D :: m2m_connect (& from_node , & sink_port , & to_node , & source_port) ,) } (LocationId :: ExternalProcess (from) , LocationId :: Process (to)) => { let from_node = externals . get (from) . unwrap_or_else (| | { panic ! ("A external used in the graph was not instantiated: {}" , from) }) . clone () ; let to_node = nodes . get (to) . unwrap_or_else (| | { panic ! ("A process used in the graph was not instantiated: {}" , to) }) . clone () ; let sink_port = D :: allocate_external_port (& from_node) ; let source_port = D :: allocate_process_port (& to_node) ; from_node . register (from_key . unwrap () , sink_port . clone ()) ; ((parse_quote ! (DUMMY) , D :: e2o_source (compile_env , & from_node , & sink_port , & to_node , & source_port) ,) , D :: e2o_connect (& from_node , & sink_port , & to_node , & source_port) ,) } (LocationId :: ExternalProcess (_from) , LocationId :: Cluster (_to)) => { todo ! ("NYI") } (LocationId :: ExternalProcess (_) , LocationId :: ExternalProcess (_)) => { panic ! ("Cannot send from external to external") } (LocationId :: Process (from) , LocationId :: ExternalProcess (to)) => { let from_node = nodes . get (from) . unwrap_or_else (| | { panic ! ("A process used in the graph was not instantiated: {}" , from) }) . clone () ; let to_node = externals . get (to) . unwrap_or_else (| | { panic ! ("A external used in the graph was not instantiated: {}" , to) }) . clone () ; let sink_port = D :: allocate_process_port (& from_node) ; let source_port = D :: allocate_external_port (& to_node) ; to_node . register (to_key . unwrap () , source_port . clone ()) ; ((D :: o2e_sink (compile_env , & from_node , & sink_port , & to_node , & source_port) , parse_quote ! (DUMMY) ,) , D :: o2e_connect (& from_node , & sink_port , & to_node , & source_port) ,) } (LocationId :: Cluster (_from) , LocationId :: ExternalProcess (_to)) => { todo ! ("NYI") } (LocationId :: Tick (_ , _) , _) => panic ! () , (_ , LocationId :: Tick (_ , _)) => panic ! () , } ; (sink , source , connect_fn) } } pub mod rewrites { pub mod persist_pullup { pub use std :: cell :: RefCell ; pub use std :: collections :: HashSet ; pub use crate :: __staged :: ir :: * ; pub fn persist_pullup_node (node : & mut HfPlusNode , persist_pulled_tees : & mut HashSet < * const RefCell < HfPlusNode > > ,) { * node = match std :: mem :: replace (node , HfPlusNode :: Placeholder) { HfPlusNode :: Unpersist (box HfPlusNode :: Persist (box behind_persist)) => behind_persist , HfPlusNode :: Delta (box HfPlusNode :: Persist (box behind_persist)) => behind_persist , HfPlusNode :: Tee { inner } => { if persist_pulled_tees . contains (& (inner . 0 . as_ref () as * const RefCell < HfPlusNode >)) { HfPlusNode :: Persist (Box :: new (HfPlusNode :: Tee { inner : TeeNode (inner . 0 . clone ()) , })) } else if matches ! (* inner . 0 . borrow () , HfPlusNode :: Persist (_)) { persist_pulled_tees . insert (inner . 0 . as_ref () as * const RefCell < HfPlusNode >) ; if let HfPlusNode :: Persist (box behind_persist) = inner . 0 . replace (HfPlusNode :: Placeholder) { * inner . 0 . borrow_mut () = behind_persist ; } else { unreachable ! () } HfPlusNode :: Persist (Box :: new (HfPlusNode :: Tee { inner : TeeNode (inner . 0 . clone ()) , })) } else { HfPlusNode :: Tee { inner } } } HfPlusNode :: Map { f , input : box HfPlusNode :: Persist (behind_persist) , } => HfPlusNode :: Persist (Box :: new (HfPlusNode :: Map { f , input : behind_persist , })) , HfPlusNode :: FilterMap { f , input : box HfPlusNode :: Persist (behind_persist) , } => HfPlusNode :: Persist (Box :: new (HfPlusNode :: FilterMap { f , input : behind_persist , })) , HfPlusNode :: FlatMap { f , input : box HfPlusNode :: Persist (behind_persist) , } => HfPlusNode :: Persist (Box :: new (HfPlusNode :: FlatMap { f , input : behind_persist , })) , HfPlusNode :: Filter { f , input : box HfPlusNode :: Persist (behind_persist) , } => HfPlusNode :: Persist (Box :: new (HfPlusNode :: Filter { f , input : behind_persist , })) , HfPlusNode :: Network { from_location , from_key , to_location , to_key , serialize_pipeline , instantiate_fn , deserialize_pipeline , input : box HfPlusNode :: Persist (behind_persist) , .. } => HfPlusNode :: Persist (Box :: new (HfPlusNode :: Network { from_location , from_key , to_location , to_key , serialize_pipeline , instantiate_fn , deserialize_pipeline , input : behind_persist , })) , HfPlusNode :: Union (box HfPlusNode :: Persist (left) , box HfPlusNode :: Persist (right)) => { HfPlusNode :: Persist (Box :: new (HfPlusNode :: Union (left , right))) } HfPlusNode :: CrossProduct (box HfPlusNode :: Persist (left) , box HfPlusNode :: Persist (right)) => { HfPlusNode :: Persist (Box :: new (HfPlusNode :: Delta (Box :: new (HfPlusNode :: CrossProduct (Box :: new (HfPlusNode :: Persist (left)) , Box :: new (HfPlusNode :: Persist (right)) ,) ,)))) } HfPlusNode :: Join (box HfPlusNode :: Persist (left) , box HfPlusNode :: Persist (right)) => { HfPlusNode :: Persist (Box :: new (HfPlusNode :: Delta (Box :: new (HfPlusNode :: Join (Box :: new (HfPlusNode :: Persist (left)) , Box :: new (HfPlusNode :: Persist (right)) ,))))) } HfPlusNode :: Unique (box HfPlusNode :: Persist (inner)) => { HfPlusNode :: Persist (Box :: new (HfPlusNode :: Delta (Box :: new (HfPlusNode :: Unique (Box :: new (HfPlusNode :: Persist (inner)) ,))))) } node => node , } ; } pub fn persist_pullup (ir : Vec < HfPlusLeaf >) -> Vec < HfPlusLeaf > { let mut seen_tees = Default :: default () ; let mut persist_pulled_tees = Default :: default () ; ir . into_iter () . map (| l | { l . transform_children (| n , s | n . transform_bottom_up (persist_pullup_node , s , & mut persist_pulled_tees) , & mut seen_tees ,) }) . collect () } # [cfg (stageleft_macro)] pub mod tests { pub use stageleft :: * ; pub use crate :: __staged :: deploy :: MultiGraph ; pub use crate :: __staged :: location :: Location ; # [test] pub fn persist_pullup_through_map () { let flow = crate :: builder :: FlowBuilder :: new () ; let process = flow . process :: < () > () ; process . source_iter (q ! (0 .. 10)) . map (q ! (| v | v + 1)) . for_each (q ! (| n | println ! ("{}" , n))) ; let built = flow . finalize () ; insta :: assert_debug_snapshot ! (built . ir ()) ; let optimized = built . optimize_with (super :: persist_pullup) ; insta :: assert_debug_snapshot ! (optimized . ir ()) ; for (id , graph) in optimized . compile_no_network :: < MultiGraph > () . hydroflow_ir () { insta :: with_settings ! ({ snapshot_suffix => format ! ("surface_graph_{id}") } , { insta :: assert_snapshot ! (graph . surface_syntax_string ()) ; }) ; } } # [test] pub fn persist_pullup_behind_tee () { let flow = crate :: builder :: FlowBuilder :: new () ; let process = flow . process :: < () > () ; let tick = process . tick () ; let before_tee = process . source_iter (q ! (0 .. 10)) . tick_batch (& tick) . persist () ; before_tee . clone () . map (q ! (| v | v + 1)) . all_ticks () . for_each (q ! (| n | println ! ("{}" , n))) ; before_tee . clone () . map (q ! (| v | v + 1)) . all_ticks () . for_each (q ! (| n | println ! ("{}" , n))) ; let built = flow . finalize () ; insta :: assert_debug_snapshot ! (built . ir ()) ; let optimized = built . optimize_with (super :: persist_pullup) ; insta :: assert_debug_snapshot ! (optimized . ir ()) ; for (id , graph) in optimized . compile_no_network :: < MultiGraph > () . hydroflow_ir () { insta :: with_settings ! ({ snapshot_suffix => format ! ("surface_graph_{id}") } , { insta :: assert_snapshot ! (graph . surface_syntax_string ()) ; }) ; } } } } pub mod profiler { pub use std :: cell :: RefCell ; pub use hydroflow :: futures :: channel :: mpsc :: UnboundedSender ; pub use stageleft :: * ; pub use super :: profiler as myself ; pub use crate :: __staged :: ir :: * ; pub use crate :: __staged :: RuntimeContext ; pub fn increment_counter (count : & mut u64) { * count += 1 ; } pub fn quoted_any_fn < 'a , F : Fn (& usize) + 'a , Q : IntoQuotedMut < 'a , F > > (q : Q) -> Q { q } # [doc = " Add a profiling node before each node to count the cardinality of its input"] pub fn add_profiling_node < 'a > (node : & mut HfPlusNode , _context : RuntimeContext < 'a > , counters : RuntimeData < & 'a RefCell < Vec < u64 > > > , counter_queue : RuntimeData < & 'a RefCell < UnboundedSender < (usize , u64) > > > , id : & mut u32 , seen_tees : & mut SeenTees ,) { let my_id = * id ; * id += 1 ; node . transform_children (| node , seen_tees | { add_profiling_node (node , _context , counters , counter_queue , id , seen_tees) } , seen_tees ,) ; let orig_node = std :: mem :: replace (node , HfPlusNode :: Placeholder) ; * node = HfPlusNode :: Inspect { f : quoted_any_fn (q ! ({ counter_queue . borrow () . unbounded_send ((my_id as usize , counters . borrow () [my_id as usize])) . unwrap () ; counters . borrow_mut () [my_id as usize] = 0 ; move | _ | { myself :: increment_counter (& mut counters . borrow_mut () [my_id as usize]) ; } })) . splice_untyped () . into () , input : Box :: new (orig_node) , } } # [doc = " Count the cardinality of each input and periodically output to a file"] pub fn profiling < 'a > (ir : Vec < HfPlusLeaf > , context : RuntimeContext < 'a > , counters : RuntimeData < & 'a RefCell < Vec < u64 > > > , counter_queue : RuntimeData < & 'a RefCell < UnboundedSender < (usize , u64) > > > ,) -> Vec < HfPlusLeaf > { let mut id = 0 ; let mut seen_tees = Default :: default () ; ir . into_iter () . map (| l | { l . transform_children (| node , seen_tees | { add_profiling_node (node , context , counters , counter_queue , & mut id , seen_tees) } , & mut seen_tees ,) }) . collect () } # [cfg (stageleft_macro)] pub mod tests { pub use stageleft :: * ; pub use crate :: __staged :: deploy :: MultiGraph ; pub use crate :: __staged :: location :: Location ; # [test] pub fn profiler_wrapping_all_operators () { let flow = crate :: builder :: FlowBuilder :: new () ; let process = flow . process :: < () > () ; process . source_iter (q ! (0 .. 10)) . map (q ! (| v | v + 1)) . for_each (q ! (| n | println ! ("{}" , n))) ; let runtime_context = flow . runtime_context () ; let built = flow . finalize () ; insta :: assert_debug_snapshot ! (& built . ir ()) ; let counters = RuntimeData :: new ("Fake") ; let counter_queue = RuntimeData :: new ("Fake") ; let pushed_down = built . optimize_with (crate :: rewrites :: persist_pullup :: persist_pullup) . optimize_with (| ir | super :: profiling (ir , runtime_context , counters , counter_queue)) ; insta :: assert_debug_snapshot ! (& pushed_down . ir ()) ; let _ = pushed_down . compile_no_network :: < MultiGraph > () ; } } } pub mod properties { pub use std :: collections :: HashSet ; pub use stageleft :: * ; pub use crate :: __staged :: ir :: { HfPlusLeaf , HfPlusNode , SeenTees } ; pub use crate :: rewrites :: properties :: PropertyDatabase ; # [doc = " Allows us to convert the hydroflow datatype for folds to a binary operation for the algebra"] # [doc = " property tests."] # [allow (clippy :: allow_attributes , dead_code , reason = "staged programming")] pub fn convert_hf_to_binary < I , A : Default , F : Fn (& mut A , I) > (f : F) -> impl Fn (I , I) -> A { move | a , b | { let mut acc = Default :: default () ; f (& mut acc , a) ; f (& mut acc , b) ; acc } } # [cfg (stageleft_macro)] impl PropertyDatabase { # [doc = " Tags the expression as commutative."] pub fn add_commutative_tag < 'a , I , A , F : Fn (& mut A , I) , Q : Quoted < 'a , F > + Clone > (& mut self , expr : Q ,) -> Q { let expr_clone = expr . clone () ; self . commutative . insert (expr_clone . splice_untyped ()) ; expr } pub fn is_tagged_commutative (& self , expr : & syn :: Expr) -> bool { self . commutative . contains (expr) } } pub fn properties_optimize_node (node : & mut HfPlusNode , db : & PropertyDatabase , seen_tees : & mut SeenTees ,) { node . transform_children (| node , seen_tees | properties_optimize_node (node , db , seen_tees) , seen_tees ,) ; match node { HfPlusNode :: ReduceKeyed { f , .. } if db . is_tagged_commutative (& f . 0) => { dbg ! ("IDENTIFIED COMMUTATIVE OPTIMIZATION for {:?}" , & f) ; } _ => { } } } pub fn properties_optimize (ir : Vec < HfPlusLeaf > , db : & PropertyDatabase) -> Vec < HfPlusLeaf > { let mut seen_tees = Default :: default () ; ir . into_iter () . map (| l | { l . transform_children (| node , seen_tees | properties_optimize_node (node , db , seen_tees) , & mut seen_tees ,) }) . collect () } # [cfg (stageleft_macro)] pub mod tests { pub use super :: * ; pub use crate :: __staged :: deploy :: SingleProcessGraph ; pub use crate :: __staged :: location :: Location ; pub use crate :: __staged :: FlowBuilder ; # [test] pub fn test_property_database () { let mut db = PropertyDatabase :: default () ; assert ! (! db . is_tagged_commutative (& (q ! (| a : & mut i32 , b : i32 | * a += b) . splice_untyped ()))) ; let _ = db . add_commutative_tag (q ! (| a : & mut i32 , b : i32 | * a += b)) ; assert ! (db . is_tagged_commutative (& (q ! (| a : & mut i32 , b : i32 | * a += b) . splice_untyped ()))) ; } # [test] pub fn test_property_optimized () { let flow = FlowBuilder :: new () ; let mut database = PropertyDatabase :: default () ; let process = flow . process :: < () > () ; let tick = process . tick () ; let counter_func = q ! (| count : & mut i32 , _ | * count += 1) ; let _ = database . add_commutative_tag (counter_func) ; process . source_iter (q ! (vec ! [])) . map (q ! (| string : String | (string , ()))) . tick_batch (& tick) . fold_keyed (q ! (|| 0) , counter_func) . all_ticks () . for_each (q ! (| (string , count) | println ! ("{}: {}" , string , count))) ; let built = flow . optimize_with (| ir | properties_optimize (ir , & database)) . with_default_optimize :: < SingleProcessGraph > () ; insta :: assert_debug_snapshot ! (built . ir ()) ; let _ = built . compile_no_network () ; } } } } pub mod staging_util { pub use proc_macro2 :: { Span , TokenStream } ; pub use quote :: quote ; pub fn get_this_crate () -> TokenStream { let hydroflow_crate = proc_macro_crate :: crate_name ("hydroflow_plus") . expect ("hydroflow_plus should be present in `Cargo.toml`") ; match hydroflow_crate { proc_macro_crate :: FoundCrate :: Itself => quote ! { hydroflow_plus } , proc_macro_crate :: FoundCrate :: Name (name) => { let ident = syn :: Ident :: new (& name , Span :: call_site ()) ; quote ! { # ident } } } } } # [cfg (stageleft_macro)] # [stageleft :: runtime] # [cfg (test)] mod tests { # [ctor :: ctor] fn init () { crate :: deploy :: init_test () ; } } [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> /opt/rustwide/target/debug/build/hydroflow_plus-41cd71e800d48fc4/out/lib_pub.rs:1:76265 [INFO] [stdout] | [INFO] [stdout] 1 | ... RuntimeData < & DeployPorts < HydroflowPlusMeta > > ,) -> impl Quoted < u32 > + Copy { q ! (cli . meta . cluster_id . expect ("Tried ... [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -------------------------- the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 1 | pub use hydroflow :: scheduled :: graph :: Hydroflow ; pub use hydroflow :: * ; pub use stageleft :: * ; pub mod runtime_support { pub use bincode ; } pub mod runtime_context { pub use std :: marker :: PhantomData ; pub use hydroflow :: scheduled :: context :: Context ; pub use proc_macro2 :: TokenStream ; pub use quote :: quote ; pub use stageleft :: runtime_support :: FreeVariable ; pub use crate :: runtime_context :: RuntimeContext ; # [cfg (stageleft_macro)] impl RuntimeContext < '_ > { pub fn new () -> Self { Self { _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl Copy for RuntimeContext < '_ > { } # [cfg (stageleft_macro)] impl Default for RuntimeContext < '_ > { fn default () -> Self { Self :: new () } } # [cfg (stageleft_macro)] impl < 'a > FreeVariable < & 'a Context > for RuntimeContext < 'a > { fn to_tokens (self) -> (Option < TokenStream > , Option < TokenStream >) { (None , Some (quote ! (& context))) } } } pub use runtime_context :: RuntimeContext ; pub mod stream { pub use std :: cell :: RefCell ; pub use std :: hash :: Hash ; pub use std :: marker :: PhantomData ; pub use std :: ops :: Deref ; pub use std :: rc :: Rc ; pub use hydroflow :: bytes :: Bytes ; pub use hydroflow :: futures :: Sink ; pub use hydroflow_lang :: parse :: Pipeline ; pub use serde :: de :: DeserializeOwned ; pub use serde :: Serialize ; pub use stageleft :: { q , IntoQuotedMut , Quoted } ; pub use syn :: parse_quote ; pub use crate :: __staged :: builder :: FLOW_USED_MESSAGE ; pub use crate :: __staged :: cycle :: { CycleCollection , CycleComplete , DeferTick , ForwardRefMarker , TickCycleMarker } ; pub use crate :: __staged :: ir :: { DebugInstantiate , HfPlusLeaf , HfPlusNode , TeeNode } ; pub use crate :: __staged :: location :: cluster :: ClusterSelfId ; pub use crate :: __staged :: location :: external_process :: { ExternalBincodeStream , ExternalBytesPort } ; pub use crate :: __staged :: location :: { check_matching_location , CanSend , ExternalProcess , Location , LocationId , NoTick , Tick , } ; pub use crate :: __staged :: staging_util :: get_this_crate ; pub use crate :: __staged :: { Cluster , ClusterId , Optional , Process , Singleton } ; pub use crate :: stream :: Unbounded ; pub use crate :: stream :: Bounded ; pub use crate :: stream :: Stream ; # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Stream < T , L , B > { fn location_kind (& self) -> LocationId { self . location . id () } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > DeferTick for Stream < T , Tick < L > , Bounded > { fn defer_tick (self) -> Self { Stream :: defer_tick (self) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleCollection < 'a , TickCycleMarker > for Stream < T , Tick < L > , Bounded > { type Location = Tick < L > ; fn create_source (ident : syn :: Ident , location : Tick < L >) -> Self { let location_id = location . id () ; Stream :: new (location , HfPlusNode :: CycleSource { ident , location_kind : location_id , } ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleComplete < 'a , TickCycleMarker > for Stream < T , Tick < L > , Bounded > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (self . ir_node . into_inner ()) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > CycleCollection < 'a , ForwardRefMarker > for Stream < T , L , B > { type Location = L ; fn create_source (ident : syn :: Ident , location : L) -> Self { let location_id = location . id () ; Stream :: new (location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: CycleSource { ident , location_kind : location_id , })) ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > CycleComplete < 'a , ForwardRefMarker > for Stream < T , L , B > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Stream < T , L , B > { pub (crate) fn new (location : L , ir_node : HfPlusNode) -> Self { Stream { location , ir_node : RefCell :: new (ir_node) , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < 'a , T : Clone , L : Location < 'a > , B > Clone for Stream < T , L , B > { fn clone (& self) -> Self { if ! matches ! (self . ir_node . borrow () . deref () , HfPlusNode :: Tee { .. }) { let orig_ir_node = self . ir_node . replace (HfPlusNode :: Placeholder) ; * self . ir_node . borrow_mut () = HfPlusNode :: Tee { inner : TeeNode (Rc :: new (RefCell :: new (orig_ir_node))) , } ; } if let HfPlusNode :: Tee { inner } = self . ir_node . borrow () . deref () { Stream { location : self . location . clone () , ir_node : HfPlusNode :: Tee { inner : TeeNode (inner . 0 . clone ()) , } . into () , _phantom : PhantomData , } } else { unreachable ! () } } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Stream < T , L , B > { pub fn map < U , F : Fn (T) -> U + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Stream < U , L , B > { Stream :: new (self . location , HfPlusNode :: Map { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn cloned (self) -> Stream < T , L , B > where T : Clone , { self . map (q ! (| d | d . clone ())) } pub fn flat_map < U , I : IntoIterator < Item = U > , F : Fn (T) -> I + 'a > (self , f : impl IntoQuotedMut < 'a , F > ,) -> Stream < U , L , B > { Stream :: new (self . location , HfPlusNode :: FlatMap { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn flatten < U > (self) -> Stream < U , L , B > where T : IntoIterator < Item = U > , { self . flat_map (q ! (| d | d)) } pub fn filter < F : Fn (& T) -> bool + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Stream < T , L , B > { Stream :: new (self . location , HfPlusNode :: Filter { f : f . splice_fn1_borrow () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn filter_map < U , F : Fn (T) -> Option < U > + 'a > (self , f : impl IntoQuotedMut < 'a , F > ,) -> Stream < U , L , B > { Stream :: new (self . location , HfPlusNode :: FilterMap { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn cross_singleton < O > (self , other : impl Into < Optional < O , L , Bounded > > ,) -> Stream < (T , O) , L , B > where O : Clone , { let other : Optional < O , L , Bounded > = other . into () ; check_matching_location (& self . location , & other . location) ; Stream :: new (self . location , HfPlusNode :: CrossSingleton (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } # [doc = " Allow this stream through if the other stream has elements, otherwise the output is empty."] pub fn continue_if < U > (self , signal : Optional < U , L , Bounded >) -> Stream < T , L , B > { self . cross_singleton (signal . map (q ! (| _u | ()))) . map (q ! (| (d , _signal) | d)) } # [doc = " Allow this stream through if the other stream is empty, otherwise the output is empty."] pub fn continue_unless < U > (self , other : Optional < U , L , Bounded >) -> Stream < T , L , B > { self . continue_if (other . into_stream () . count () . filter (q ! (| c | * c == 0))) } pub fn cross_product < O > (self , other : Stream < O , L , B >) -> Stream < (T , O) , L , B > where T : Clone , O : Clone , { check_matching_location (& self . location , & other . location) ; Stream :: new (self . location , HfPlusNode :: CrossProduct (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } pub fn union (self , other : Stream < T , L , B >) -> Stream < T , L , B > { check_matching_location (& self . location , & other . location) ; Stream :: new (self . location , HfPlusNode :: Union (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } pub fn enumerate (self) -> Stream < (usize , T) , L , B > { Stream :: new (self . location , HfPlusNode :: Enumerate (Box :: new (self . ir_node . into_inner ())) ,) } pub fn unique (self) -> Stream < T , L , B > where T : Eq + Hash , { Stream :: new (self . location , HfPlusNode :: Unique (Box :: new (self . ir_node . into_inner ())) ,) } pub fn filter_not_in (self , other : Stream < T , L , Bounded >) -> Stream < T , L , Bounded > where T : Eq + Hash , { check_matching_location (& self . location , & other . location) ; Stream :: new (self . location , HfPlusNode :: Difference (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } pub fn first (self) -> Optional < T , L , Bounded > { Optional :: new (self . location , self . ir_node . into_inner ()) } pub fn inspect < F : Fn (& T) + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Stream < T , L , B > { if L :: is_top_level () { Stream :: new (self . location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: Inspect { f : f . splice_fn1_borrow () . into () , input : Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , })) ,) } else { Stream :: new (self . location , HfPlusNode :: Inspect { f : f . splice_fn1_borrow () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } } pub fn fold < A , I : Fn () -> A + 'a , F : Fn (& mut A , T) > (self , init : impl IntoQuotedMut < 'a , I > , comb : impl IntoQuotedMut < 'a , F > ,) -> Singleton < A , L , B > { let mut core = HfPlusNode :: Fold { init : init . splice_fn0 () . into () , acc : comb . splice_fn2_borrow_mut () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ; if L :: is_top_level () { core = HfPlusNode :: Persist (Box :: new (core)) ; } Singleton :: new (self . location , core) } pub fn reduce < F : Fn (& mut T , T) + 'a > (self , comb : impl IntoQuotedMut < 'a , F > ,) -> Optional < T , L , B > { let mut core = HfPlusNode :: Reduce { f : comb . splice_fn2_borrow_mut () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ; if L :: is_top_level () { core = HfPlusNode :: Persist (Box :: new (core)) ; } Optional :: new (self . location , core) } pub fn max (self) -> Optional < T , L , B > where T : Ord , { self . reduce (q ! (| curr , new | { if new > * curr { * curr = new ; } })) } pub fn min (self) -> Optional < T , L , B > where T : Ord , { self . reduce (q ! (| curr , new | { if new < * curr { * curr = new ; } })) } pub fn count (self) -> Singleton < usize , L , B > { self . fold (q ! (|| 0usize) , q ! (| count , _ | * count += 1)) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > Stream < T , L , Bounded > { pub fn sort (self) -> Stream < T , L , Bounded > where T : Ord , { Stream :: new (self . location , HfPlusNode :: Sort (Box :: new (self . ir_node . into_inner ())) ,) } } # [cfg (stageleft_macro)] impl < 'a , K , V1 , L : Location < 'a > , B > Stream < (K , V1) , L , B > { pub fn join < V2 > (self , n : Stream < (K , V2) , L , B >) -> Stream < (K , (V1 , V2)) , L , B > where K : Eq + Hash , { check_matching_location (& self . location , & n . location) ; Stream :: new (self . location , HfPlusNode :: Join (Box :: new (self . ir_node . into_inner ()) , Box :: new (n . ir_node . into_inner ()) ,) ,) } pub fn anti_join (self , n : Stream < K , L , Bounded >) -> Stream < (K , V1) , L , B > where K : Eq + Hash , { check_matching_location (& self . location , & n . location) ; Stream :: new (self . location , HfPlusNode :: AntiJoin (Box :: new (self . ir_node . into_inner ()) , Box :: new (n . ir_node . into_inner ()) ,) ,) } } # [cfg (stageleft_macro)] impl < 'a , K : Eq + Hash , V , L : Location < 'a > > Stream < (K , V) , Tick < L > , Bounded > { pub fn fold_keyed < A , I : Fn () -> A + 'a , F : Fn (& mut A , V) + 'a > (self , init : impl IntoQuotedMut < 'a , I > , comb : impl IntoQuotedMut < 'a , F > ,) -> Stream < (K , A) , Tick < L > , Bounded > { Stream :: new (self . location , HfPlusNode :: FoldKeyed { init : init . splice_fn0 () . into () , acc : comb . splice_fn2_borrow_mut () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn reduce_keyed < F : Fn (& mut V , V) + 'a > (self , comb : impl IntoQuotedMut < 'a , F > ,) -> Stream < (K , V) , Tick < L > , Bounded > { Stream :: new (self . location , HfPlusNode :: ReduceKeyed { f : comb . splice_fn2_borrow_mut () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > Stream < T , L , B > { pub fn tick_batch (self , tick : & Tick < L >) -> Stream < T , Tick < L > , Bounded > { Stream :: new (tick . clone () , HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn tick_prefix (self , tick : & Tick < L >) -> Stream < T , Tick < L > , Bounded > where T : Clone , { self . tick_batch (tick) . persist () } pub fn sample_every (self , interval : impl Quoted < 'a , std :: time :: Duration > + Copy + 'a ,) -> Stream < T , L , Unbounded > { let samples = self . location . source_interval (interval) ; let tick = self . location . tick () ; self . tick_batch (& tick) . continue_if (samples . tick_batch (& tick) . first ()) . all_ticks () } pub fn for_each < F : Fn (T) + 'a > (self , f : impl IntoQuotedMut < 'a , F >) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: ForEach { input : Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , f : f . splice_fn1 () . into () , }) ; } pub fn dest_sink < S : Unpin + Sink < T > + 'a > (self , sink : impl Quoted < 'a , S >) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: DestSink { sink : sink . splice_typed () . into () , input : Box :: new (self . ir_node . into_inner ()) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > Stream < T , Tick < L > , Bounded > { pub fn all_ticks (self) -> Stream < T , L , Unbounded > { Stream :: new (self . location . outer () . clone () , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn persist (self) -> Stream < T , Tick < L > , Bounded > where T : Clone , { Stream :: new (self . location , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn defer_tick (self) -> Stream < T , Tick < L > , Bounded > { Stream :: new (self . location , HfPlusNode :: DeferTick (Box :: new (self . ir_node . into_inner ())) ,) } pub fn delta (self) -> Stream < T , Tick < L > , Bounded > { Stream :: new (self . location , HfPlusNode :: Delta (Box :: new (self . ir_node . into_inner ())) ,) } } pub fn serialize_bincode < T : Serialize > (is_demux : bool) -> Pipeline { let root = get_this_crate () ; let t_type : syn :: Type = stageleft :: quote_type :: < T > () ; if is_demux { parse_quote ! { map (| (id , data) : (# root :: ClusterId < _ >, # t_type) | { (id . raw_id , # root :: runtime_support :: bincode :: serialize ::<# t_type > (& data) . unwrap () . into ()) }) } } else { parse_quote ! { map (| data | { # root :: runtime_support :: bincode :: serialize ::<# t_type > (& data) . unwrap () . into () }) } } } pub fn deserialize_bincode < T : DeserializeOwned > (tagged : Option < syn :: Type >) -> Pipeline { let root = get_this_crate () ; let t_type : syn :: Type = stageleft :: quote_type :: < T > () ; if let Some (c_type) = tagged { parse_quote ! { map (| res | { let (id , b) = res . unwrap () ; (# root :: ClusterId ::<# c_type >:: from_raw (id) , # root :: runtime_support :: bincode :: deserialize ::<# t_type > (& b) . unwrap ()) }) } } else { parse_quote ! { map (| res | { # root :: runtime_support :: bincode :: deserialize ::<# t_type > (& res . unwrap ()) . unwrap () }) } } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > Stream < T , L , B > { pub fn decouple_process < P2 > (self , other : & Process < 'a , P2 > ,) -> Stream < T , Process < 'a , P2 > , Unbounded > where L : CanSend < 'a , Process < 'a , P2 > , In < T > = T , Out < T > = T > , T : Clone + Serialize + DeserializeOwned , { self . send_bincode :: < Process < 'a , P2 > , T > (other) } pub fn decouple_cluster < C2 , Tag > (self , other : & Cluster < 'a , C2 > ,) -> Stream < T , Cluster < 'a , C2 > , Unbounded > where L : CanSend < 'a , Cluster < 'a , C2 > , In < T > = (ClusterId < C2 > , T) , Out < T > = (Tag , T) > , T : Clone + Serialize + DeserializeOwned , { let self_node_id = match self . location_kind () { LocationId :: Cluster (cluster_id) => ClusterSelfId { id : cluster_id , _phantom : PhantomData , } , _ => panic ! ("decouple_cluster must be called on a cluster") , } ; self . map (q ! (move | b | (self_node_id , b . clone ()))) . send_bincode_interleaved (other) } pub fn send_bincode < L2 : Location < 'a > , CoreType > (self , other : & L2 ,) -> Stream < L :: Out < CoreType > , L2 , Unbounded > where L : CanSend < 'a , L2 , In < CoreType > = T > , CoreType : Serialize + DeserializeOwned , { let serialize_pipeline = Some (serialize_bincode :: < CoreType > (L :: is_demux ())) ; let deserialize_pipeline = Some (deserialize_bincode :: < CoreType > (L :: tagged_type ())) ; Stream :: new (other . clone () , HfPlusNode :: Network { from_location : self . location_kind () , from_key : None , to_location : other . id () , to_key : None , serialize_pipeline , instantiate_fn : DebugInstantiate :: Building () , deserialize_pipeline , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn send_bincode_external < L2 : 'a , CoreType > (self , other : & ExternalProcess < L2 > ,) -> ExternalBincodeStream < L :: Out < CoreType > > where L : CanSend < 'a , ExternalProcess < 'a , L2 > , In < CoreType > = T , Out < CoreType > = CoreType > , CoreType : Serialize + DeserializeOwned , { let serialize_pipeline = Some (serialize_bincode :: < CoreType > (L :: is_demux ())) ; let mut flow_state_borrow = self . location . flow_state () . borrow_mut () ; let external_key = flow_state_borrow . next_external_out ; flow_state_borrow . next_external_out += 1 ; let leaves = flow_state_borrow . leaves . as_mut () . expect ("Attempted to add a leaf to a flow that has already been finalized. No leaves can be added after the flow has been compiled()") ; let dummy_f : syn :: Expr = syn :: parse_quote ! (()) ; leaves . push (HfPlusLeaf :: ForEach { f : dummy_f . into () , input : Box :: new (HfPlusNode :: Network { from_location : self . location_kind () , from_key : None , to_location : other . id () , to_key : Some (external_key) , serialize_pipeline , instantiate_fn : DebugInstantiate :: Building () , deserialize_pipeline : None , input : Box :: new (self . ir_node . into_inner ()) , }) , }) ; ExternalBincodeStream { process_id : other . id , port_id : external_key , _phantom : PhantomData , } } pub fn send_bytes < L2 : Location < 'a > > (self , other : & L2) -> Stream < L :: Out < Bytes > , L2 , Unbounded > where L : CanSend < 'a , L2 , In < Bytes > = T > , { let root = get_this_crate () ; Stream :: new (other . clone () , HfPlusNode :: Network { from_location : self . location_kind () , from_key : None , to_location : other . id () , to_key : None , serialize_pipeline : None , instantiate_fn : DebugInstantiate :: Building () , deserialize_pipeline : if let Some (c_type) = L :: tagged_type () { Some (parse_quote ! (map (| (id , b) | (# root :: ClusterId <# c_type >:: from_raw (id) , b . unwrap () . freeze ()))) ,) } else { Some (parse_quote ! (map (| b | b . unwrap () . freeze ()))) } , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn send_bytes_external < L2 : 'a > (self , other : & ExternalProcess < L2 >) -> ExternalBytesPort where L : CanSend < 'a , ExternalProcess < 'a , L2 > , In < Bytes > = T , Out < Bytes > = Bytes > , { let mut flow_state_borrow = self . location . flow_state () . borrow_mut () ; let external_key = flow_state_borrow . next_external_out ; flow_state_borrow . next_external_out += 1 ; let leaves = flow_state_borrow . leaves . as_mut () . expect ("Attempted to add a leaf to a flow that has already been finalized. No leaves can be added after the flow has been compiled()") ; let dummy_f : syn :: Expr = syn :: parse_quote ! (()) ; leaves . push (HfPlusLeaf :: ForEach { f : dummy_f . into () , input : Box :: new (HfPlusNode :: Network { from_location : self . location_kind () , from_key : None , to_location : other . id () , to_key : Some (external_key) , serialize_pipeline : None , instantiate_fn : DebugInstantiate :: Building () , deserialize_pipeline : None , input : Box :: new (self . ir_node . into_inner ()) , }) , }) ; ExternalBytesPort { process_id : other . id , port_id : external_key , } } pub fn send_bincode_interleaved < L2 : Location < 'a > , Tag , CoreType > (self , other : & L2 ,) -> Stream < CoreType , L2 , Unbounded > where L : CanSend < 'a , L2 , In < CoreType > = T , Out < CoreType > = (Tag , CoreType) > , CoreType : Serialize + DeserializeOwned , { self . send_bincode :: < L2 , CoreType > (other) . map (q ! (| (_ , b) | b)) } pub fn send_bytes_interleaved < L2 : Location < 'a > , Tag > (self , other : & L2 ,) -> Stream < Bytes , L2 , Unbounded > where L : CanSend < 'a , L2 , In < Bytes > = T , Out < Bytes > = (Tag , Bytes) > , { self . send_bytes :: < L2 > (other) . map (q ! (| (_ , b) | b)) } pub fn broadcast_bincode < C2 > (self , other : & Cluster < 'a , C2 > ,) -> Stream < L :: Out < T > , Cluster < 'a , C2 > , Unbounded > where L : CanSend < 'a , Cluster < 'a , C2 > , In < T > = (ClusterId < C2 > , T) > , T : Clone + Serialize + DeserializeOwned , { let ids = other . members () ; self . flat_map (q ! (| b | ids . iter () . map (move | id | (:: std :: clone :: Clone :: clone (id) , :: std :: clone :: Clone :: clone (& b))))) . send_bincode (other) } pub fn broadcast_bincode_interleaved < C2 , Tag > (self , other : & Cluster < 'a , C2 > ,) -> Stream < T , Cluster < 'a , C2 > , Unbounded > where L : CanSend < 'a , Cluster < 'a , C2 > , In < T > = (ClusterId < C2 > , T) , Out < T > = (Tag , T) > + 'a , T : Clone + Serialize + DeserializeOwned , { self . broadcast_bincode (other) . map (q ! (| (_ , b) | b)) } pub fn broadcast_bytes < C2 > (self , other : & Cluster < 'a , C2 > ,) -> Stream < L :: Out < Bytes > , Cluster < 'a , C2 > , Unbounded > where L : CanSend < 'a , Cluster < 'a , C2 > , In < Bytes > = (ClusterId < C2 > , T) > + 'a , T : Clone , { let ids = other . members () ; self . flat_map (q ! (| b | ids . iter () . map (move | id | (:: std :: clone :: Clone :: clone (id) , :: std :: clone :: Clone :: clone (& b))))) . send_bytes (other) } pub fn broadcast_bytes_interleaved < C2 , Tag > (self , other : & Cluster < 'a , C2 > ,) -> Stream < Bytes , Cluster < 'a , C2 > , Unbounded > where L : CanSend < 'a , Cluster < 'a , C2 > , In < Bytes > = (ClusterId < C2 > , T) , Out < Bytes > = (Tag , Bytes) > + 'a , T : Clone , { self . broadcast_bytes (other) . map (q ! (| (_ , b) | b)) } } # [cfg (stageleft_macro)] pub mod tests { pub use hydro_deploy :: Deployment ; pub use hydroflow :: futures :: StreamExt ; pub use serde :: { Deserialize , Serialize } ; pub use stageleft :: q ; pub use crate :: __staged :: location :: Location ; pub use crate :: __staged :: FlowBuilder ; pub struct P1 { } pub struct P2 { } # [derive (Serialize , Deserialize , Debug)] pub struct SendOverNetwork { pub n : u32 , } # [tokio :: test] pub async fn first_ten_distributed () { let mut deployment = Deployment :: new () ; let flow = FlowBuilder :: new () ; let first_node = flow . process :: < P1 > () ; let second_node = flow . process :: < P2 > () ; let external = flow . external_process :: < P2 > () ; let numbers = first_node . source_iter (q ! (0 .. 10)) ; let out_port = numbers . map (q ! (| n | SendOverNetwork { n })) . send_bincode (& second_node) . send_bincode_external (& external) ; let nodes = flow . with_process (& first_node , deployment . Localhost ()) . with_process (& second_node , deployment . Localhost ()) . with_external (& external , deployment . Localhost ()) . deploy (& mut deployment) ; deployment . deploy () . await . unwrap () ; let mut external_out = nodes . connect_source_bincode (out_port) . await ; deployment . start () . await . unwrap () ; for i in 0 .. 10 { assert_eq ! (external_out . next () . await . unwrap () . n , i) ; } } } } pub use stream :: { Bounded , Stream , Unbounded } ; pub mod singleton { pub use std :: cell :: RefCell ; pub use std :: marker :: PhantomData ; pub use std :: ops :: Deref ; pub use std :: rc :: Rc ; pub use stageleft :: { q , IntoQuotedMut , Quoted } ; pub use crate :: __staged :: builder :: FLOW_USED_MESSAGE ; pub use crate :: __staged :: cycle :: { CycleCollection , CycleCollectionWithInitial , CycleComplete , DeferTick , ForwardRefMarker , TickCycleMarker , } ; pub use crate :: __staged :: ir :: { HfPlusLeaf , HfPlusNode , TeeNode } ; pub use crate :: __staged :: location :: { check_matching_location , Location , LocationId , NoTick , Tick } ; pub use crate :: __staged :: stream :: { Bounded , Unbounded } ; pub use crate :: __staged :: { Optional , Stream } ; pub use crate :: singleton :: Singleton ; # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Singleton < T , L , B > { pub (crate) fn new (location : L , ir_node : HfPlusNode) -> Self { Singleton { location , ir_node : RefCell :: new (ir_node) , _phantom : PhantomData , } } fn location_kind (& self) -> LocationId { self . location . id () } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > From < Singleton < T , L , Bounded > > for Singleton < T , L , Unbounded > { fn from (singleton : Singleton < T , L , Bounded >) -> Self { Singleton :: new (singleton . location , singleton . ir_node . into_inner ()) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > DeferTick for Singleton < T , Tick < L > , Bounded > { fn defer_tick (self) -> Self { Singleton :: defer_tick (self) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleCollectionWithInitial < 'a , TickCycleMarker > for Singleton < T , Tick < L > , Bounded > { type Location = Tick < L > ; fn create_source (ident : syn :: Ident , initial : Self , location : Tick < L >) -> Self { let location_id = location . id () ; Singleton :: new (location , HfPlusNode :: Union (Box :: new (HfPlusNode :: CycleSource { ident , location_kind : location_id , }) , initial . ir_node . into_inner () . into () ,) ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleComplete < 'a , TickCycleMarker > for Singleton < T , Tick < L > , Bounded > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (self . ir_node . into_inner ()) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleCollection < 'a , ForwardRefMarker > for Singleton < T , Tick < L > , Bounded > { type Location = Tick < L > ; fn create_source (ident : syn :: Ident , location : Tick < L >) -> Self { let location_id = location . id () ; Singleton :: new (location , HfPlusNode :: CycleSource { ident , location_kind : location_id , } ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleComplete < 'a , ForwardRefMarker > for Singleton < T , Tick < L > , Bounded > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (self . ir_node . into_inner ()) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T : Clone , L : Location < 'a > , B > Clone for Singleton < T , L , B > { fn clone (& self) -> Self { if ! matches ! (self . ir_node . borrow () . deref () , HfPlusNode :: Tee { .. }) { let orig_ir_node = self . ir_node . replace (HfPlusNode :: Placeholder) ; * self . ir_node . borrow_mut () = HfPlusNode :: Tee { inner : TeeNode (Rc :: new (RefCell :: new (orig_ir_node))) , } ; } if let HfPlusNode :: Tee { inner } = self . ir_node . borrow () . deref () { Singleton { location : self . location . clone () , ir_node : HfPlusNode :: Tee { inner : TeeNode (inner . 0 . clone ()) , } . into () , _phantom : PhantomData , } } else { unreachable ! () } } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Singleton < T , L , B > { pub fn into_stream (self) -> Stream < T , L , Bounded > { Stream :: new (self . location , self . ir_node . into_inner ()) } pub fn map < U , F : Fn (T) -> U + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Singleton < U , L , B > { Singleton :: new (self . location , HfPlusNode :: Map { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn flat_map < U , I : IntoIterator < Item = U > , F : Fn (T) -> I + 'a > (self , f : impl IntoQuotedMut < 'a , F > ,) -> Stream < U , L , B > { Stream :: new (self . location , HfPlusNode :: FlatMap { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn filter < F : Fn (& T) -> bool + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Optional < T , L , B > { Optional :: new (self . location , HfPlusNode :: Filter { f : f . splice_fn1_borrow () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn filter_map < U , F : Fn (T) -> Option < U > + 'a > (self , f : impl IntoQuotedMut < 'a , F > ,) -> Optional < U , L , B > { Optional :: new (self . location , HfPlusNode :: FilterMap { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn zip < Other > (self , other : Other) -> < Self as ZipResult < 'a , Other > > :: Out where Self : ZipResult < 'a , Other , Location = L > , { check_matching_location (& self . location , & Self :: other_location (& other)) ; if L :: is_top_level () { Self :: make (self . location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: CrossSingleton (Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , Box :: new (HfPlusNode :: Unpersist (Box :: new (Self :: other_ir_node (other)))) ,))) ,) } else { Self :: make (self . location , HfPlusNode :: CrossSingleton (Box :: new (self . ir_node . into_inner ()) , Box :: new (Self :: other_ir_node (other)) ,) ,) } } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > Singleton < T , L , Bounded > { pub fn continue_if < U > (self , signal : Optional < U , L , Bounded >) -> Optional < T , L , Bounded > { self . zip (signal . map (q ! (| _u | ()))) . map (q ! (| (d , _signal) | d)) } pub fn continue_unless < U > (self , other : Optional < U , L , Bounded >) -> Optional < T , L , Bounded > { self . continue_if (other . into_stream () . count () . filter (q ! (| c | * c == 0))) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > Singleton < T , L , B > { pub fn latest_tick (self , tick : & Tick < L >) -> Singleton < T , Tick < L > , Bounded > { Singleton :: new (tick . clone () , HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn tick_samples (self) -> Stream < T , L , Unbounded > { let tick = self . location . tick () ; self . latest_tick (& tick) . all_ticks () } pub fn sample_every (self , interval : impl Quoted < 'a , std :: time :: Duration > + Copy + 'a ,) -> Stream < T , L , Unbounded > { let samples = self . location . source_interval (interval) ; let tick = self . location . tick () ; self . latest_tick (& tick) . continue_if (samples . tick_batch (& tick) . first ()) . all_ticks () } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > Singleton < T , Tick < L > , Bounded > { pub fn all_ticks (self) -> Stream < T , L , Unbounded > { Stream :: new (self . location . outer () . clone () , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn latest (self) -> Singleton < T , L , Unbounded > { Singleton :: new (self . location . outer () . clone () , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn defer_tick (self) -> Singleton < T , Tick < L > , Bounded > { Singleton :: new (self . location , HfPlusNode :: DeferTick (Box :: new (self . ir_node . into_inner ())) ,) } pub fn persist (self) -> Stream < T , Tick < L > , Bounded > { Stream :: new (self . location , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn delta (self) -> Optional < T , Tick < L > , Bounded > { Optional :: new (self . location , HfPlusNode :: Delta (Box :: new (self . ir_node . into_inner ())) ,) } } pub use crate :: singleton :: ZipResult ; # [cfg (stageleft_macro)] impl < 'a , T , U : Clone , L : Location < 'a > , B > ZipResult < 'a , Singleton < U , L , B > > for Singleton < T , L , B > { type Out = Singleton < (T , U) , L , B > ; type Location = L ; fn other_location (other : & Singleton < U , L , B >) -> L { other . location . clone () } fn other_ir_node (other : Singleton < U , L , B >) -> HfPlusNode { other . ir_node . into_inner () } fn make (location : L , ir_node : HfPlusNode) -> Self :: Out { Singleton :: new (location , ir_node) } } # [cfg (stageleft_macro)] impl < 'a , T , U : Clone , L : Location < 'a > , B > ZipResult < 'a , Optional < U , L , B > > for Singleton < T , L , B > { type Out = Optional < (T , U) , L , B > ; type Location = L ; fn other_location (other : & Optional < U , L , B >) -> L { other . location . clone () } fn other_ir_node (other : Optional < U , L , B >) -> HfPlusNode { other . ir_node . into_inner () } fn make (location : L , ir_node : HfPlusNode) -> Self :: Out { Optional :: new (location , ir_node) } } } pub use singleton :: Singleton ; pub mod optional { pub use std :: cell :: RefCell ; pub use std :: marker :: PhantomData ; pub use std :: ops :: Deref ; pub use std :: rc :: Rc ; pub use stageleft :: { q , IntoQuotedMut , Quoted } ; pub use syn :: parse_quote ; pub use crate :: __staged :: builder :: FLOW_USED_MESSAGE ; pub use crate :: __staged :: cycle :: { CycleCollection , CycleComplete , DeferTick , ForwardRefMarker , TickCycleMarker } ; pub use crate :: __staged :: ir :: { HfPlusLeaf , HfPlusNode , HfPlusSource , TeeNode } ; pub use crate :: __staged :: location :: { check_matching_location , LocationId , NoTick } ; pub use crate :: __staged :: { Bounded , Location , Singleton , Stream , Tick , Unbounded } ; pub use crate :: optional :: Optional ; # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Optional < T , L , B > { pub (crate) fn new (location : L , ir_node : HfPlusNode) -> Self { Optional { location , ir_node : RefCell :: new (ir_node) , _phantom : PhantomData , } } pub fn some (singleton : Singleton < T , L , B >) -> Self { Optional :: new (singleton . location , singleton . ir_node . into_inner ()) } fn location_kind (& self) -> LocationId { self . location . id () } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > DeferTick for Optional < T , Tick < L > , Bounded > { fn defer_tick (self) -> Self { Optional :: defer_tick (self) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleCollection < 'a , TickCycleMarker > for Optional < T , Tick < L > , Bounded > { type Location = Tick < L > ; fn create_source (ident : syn :: Ident , location : Tick < L >) -> Self { let location_id = location . id () ; Optional :: new (location , HfPlusNode :: CycleSource { ident , location_kind : location_id , } ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleComplete < 'a , TickCycleMarker > for Optional < T , Tick < L > , Bounded > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (self . ir_node . into_inner ()) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleCollection < 'a , ForwardRefMarker > for Optional < T , Tick < L > , Bounded > { type Location = Tick < L > ; fn create_source (ident : syn :: Ident , location : Tick < L >) -> Self { let location_id = location . id () ; Optional :: new (location , HfPlusNode :: CycleSource { ident , location_kind : location_id , } ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > CycleComplete < 'a , ForwardRefMarker > for Optional < T , Tick < L > , Bounded > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (self . ir_node . into_inner ()) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > CycleCollection < 'a , ForwardRefMarker > for Optional < T , L , B > { type Location = L ; fn create_source (ident : syn :: Ident , location : L) -> Self { let location_id = location . id () ; Optional :: new (location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: CycleSource { ident , location_kind : location_id , })) ,) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > CycleComplete < 'a , ForwardRefMarker > for Optional < T , L , B > { fn complete (self , ident : syn :: Ident) { self . location . flow_state () . borrow_mut () . leaves . as_mut () . expect (FLOW_USED_MESSAGE) . push (HfPlusLeaf :: CycleSink { ident , location_kind : self . location_kind () , input : Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , }) ; } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > From < Optional < T , L , Bounded > > for Optional < T , L , Unbounded > { fn from (singleton : Optional < T , L , Bounded >) -> Self { Optional :: new (singleton . location , singleton . ir_node . into_inner ()) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > From < Singleton < T , L , B > > for Optional < T , L , B > { fn from (singleton : Singleton < T , L , B >) -> Self { Optional :: some (singleton) } } # [cfg (stageleft_macro)] impl < 'a , T : Clone , L : Location < 'a > , B > Clone for Optional < T , L , B > { fn clone (& self) -> Self { if ! matches ! (self . ir_node . borrow () . deref () , HfPlusNode :: Tee { .. }) { let orig_ir_node = self . ir_node . replace (HfPlusNode :: Placeholder) ; * self . ir_node . borrow_mut () = HfPlusNode :: Tee { inner : TeeNode (Rc :: new (RefCell :: new (orig_ir_node))) , } ; } if let HfPlusNode :: Tee { inner } = self . ir_node . borrow () . deref () { Optional { location : self . location . clone () , ir_node : HfPlusNode :: Tee { inner : TeeNode (inner . 0 . clone ()) , } . into () , _phantom : PhantomData , } } else { unreachable ! () } } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > , B > Optional < T , L , B > { pub fn into_stream (self) -> Stream < T , L , B > { if L :: is_top_level () { panic ! ("Converting an optional to a stream is not yet supported at the top level") ; } Stream :: new (self . location , self . ir_node . into_inner ()) } pub fn map < U , F : Fn (T) -> U + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Optional < U , L , B > { Optional :: new (self . location , HfPlusNode :: Map { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn flat_map < U , I : IntoIterator < Item = U > , F : Fn (T) -> I + 'a > (self , f : impl IntoQuotedMut < 'a , F > ,) -> Stream < U , L , B > { Stream :: new (self . location , HfPlusNode :: FlatMap { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn filter < F : Fn (& T) -> bool + 'a > (self , f : impl IntoQuotedMut < 'a , F >) -> Optional < T , L , B > { Optional :: new (self . location , HfPlusNode :: Filter { f : f . splice_fn1_borrow () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn filter_map < U , F : Fn (T) -> Option < U > + 'a > (self , f : impl IntoQuotedMut < 'a , F > ,) -> Optional < U , L , B > { Optional :: new (self . location , HfPlusNode :: FilterMap { f : f . splice_fn1 () . into () , input : Box :: new (self . ir_node . into_inner ()) , } ,) } pub fn union (self , other : Optional < T , L , B >) -> Optional < T , L , B > { check_matching_location (& self . location , & other . location) ; if L :: is_top_level () { Optional :: new (self . location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: Union (Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , Box :: new (HfPlusNode :: Unpersist (Box :: new (other . ir_node . into_inner ()))) ,))) ,) } else { Optional :: new (self . location , HfPlusNode :: Union (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } } pub fn zip < O > (self , other : impl Into < Optional < O , L , B > >) -> Optional < (T , O) , L , B > where O : Clone , { let other : Optional < O , L , B > = other . into () ; check_matching_location (& self . location , & other . location) ; if L :: is_top_level () { Optional :: new (self . location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: CrossSingleton (Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , Box :: new (HfPlusNode :: Unpersist (Box :: new (other . ir_node . into_inner ()))) ,))) ,) } else { Optional :: new (self . location , HfPlusNode :: CrossSingleton (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } } pub fn unwrap_or (self , other : Singleton < T , L , B >) -> Singleton < T , L , B > { check_matching_location (& self . location , & other . location) ; if L :: is_top_level () { Singleton :: new (self . location , HfPlusNode :: Persist (Box :: new (HfPlusNode :: Union (Box :: new (HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ()))) , Box :: new (HfPlusNode :: Unpersist (Box :: new (other . ir_node . into_inner ()))) ,))) ,) } else { Singleton :: new (self . location , HfPlusNode :: Union (Box :: new (self . ir_node . into_inner ()) , Box :: new (other . ir_node . into_inner ()) ,) ,) } } pub fn into_singleton (self) -> Singleton < Option < T > , L , B > where T : Clone , { let none : syn :: Expr = parse_quote ! ([:: std :: option :: Option :: None]) ; let core_ir = HfPlusNode :: Persist (Box :: new (HfPlusNode :: Source { source : HfPlusSource :: Iter (none . into ()) , location_kind : self . location . id () . root () . clone () , })) ; let none_singleton = if L :: is_top_level () { Singleton :: new (self . location . clone () , HfPlusNode :: Persist (Box :: new (core_ir)) ,) } else { Singleton :: new (self . location . clone () , core_ir) } ; self . map (q ! (| v | Some (v))) . unwrap_or (none_singleton) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > Optional < T , L , Bounded > { pub fn continue_if < U > (self , signal : Optional < U , L , Bounded >) -> Optional < T , L , Bounded > { self . zip (signal . map (q ! (| _u | ()))) . map (q ! (| (d , _signal) | d)) } pub fn continue_unless < U > (self , other : Optional < U , L , Bounded >) -> Optional < T , L , Bounded > { self . continue_if (other . into_stream () . count () . filter (q ! (| c | * c == 0))) } pub fn then < U > (self , value : Singleton < U , L , Bounded >) -> Optional < U , L , Bounded > { value . continue_if (self) } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > + NoTick , B > Optional < T , L , B > { pub fn latest_tick (self , tick : & Tick < L >) -> Optional < T , Tick < L > , Bounded > { Optional :: new (tick . clone () , HfPlusNode :: Unpersist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn tick_samples (self) -> Stream < T , L , Unbounded > { let tick = self . location . tick () ; self . latest_tick (& tick) . all_ticks () } pub fn sample_every (self , interval : impl Quoted < 'a , std :: time :: Duration > + Copy + 'a ,) -> Stream < T , L , Unbounded > { let samples = self . location . source_interval (interval) ; let tick = self . location . tick () ; self . latest_tick (& tick) . continue_if (samples . tick_batch (& tick) . first ()) . all_ticks () } } # [cfg (stageleft_macro)] impl < 'a , T , L : Location < 'a > > Optional < T , Tick < L > , Bounded > { pub fn all_ticks (self) -> Stream < T , L , Unbounded > { Stream :: new (self . location . outer () . clone () , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn latest (self) -> Optional < T , L , Unbounded > { Optional :: new (self . location . outer () . clone () , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn defer_tick (self) -> Optional < T , Tick < L > , Bounded > { Optional :: new (self . location , HfPlusNode :: DeferTick (Box :: new (self . ir_node . into_inner ())) ,) } pub fn persist (self) -> Stream < T , Tick < L > , Bounded > { Stream :: new (self . location , HfPlusNode :: Persist (Box :: new (self . ir_node . into_inner ())) ,) } pub fn delta (self) -> Optional < T , Tick < L > , Bounded > { Optional :: new (self . location , HfPlusNode :: Delta (Box :: new (self . ir_node . into_inner ())) ,) } } } pub use optional :: Optional ; pub mod location { pub use std :: fmt :: Debug ; pub use std :: marker :: PhantomData ; pub use std :: time :: Duration ; pub use hydroflow :: futures :: stream :: Stream as FuturesStream ; pub use hydroflow :: { tokio , tokio_stream } ; pub use proc_macro2 :: Span ; pub use stageleft :: { q , Quoted } ; pub use super :: builder :: FlowState ; pub use crate :: __staged :: cycle :: { CycleCollection , ForwardRef , ForwardRefMarker } ; pub use crate :: __staged :: ir :: { HfPlusNode , HfPlusSource } ; pub use crate :: __staged :: { Singleton , Stream , Unbounded } ; pub mod external_process { pub use std :: marker :: PhantomData ; pub use hydroflow :: bytes :: Bytes ; pub use serde :: de :: DeserializeOwned ; pub use serde :: Serialize ; pub use super :: { Location , LocationId , NoTick } ; pub use crate :: __staged :: builder :: FlowState ; pub use crate :: __staged :: ir :: { HfPlusNode , HfPlusSource } ; pub use crate :: __staged :: { Stream , Unbounded } ; pub use crate :: location :: external_process :: ExternalBytesPort ; pub use crate :: location :: external_process :: ExternalBincodeSink ; pub use crate :: location :: external_process :: ExternalBincodeStream ; pub use crate :: location :: external_process :: ExternalProcess ; # [cfg (stageleft_macro)] impl < P > Clone for ExternalProcess < '_ , P > { fn clone (& self) -> Self { ExternalProcess { id : self . id , flow_state : self . flow_state . clone () , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < 'a , P > Location < 'a > for ExternalProcess < 'a , P > { fn id (& self) -> LocationId { LocationId :: ExternalProcess (self . id) } fn flow_state (& self) -> & FlowState { & self . flow_state } fn is_top_level () -> bool { true } } # [cfg (stageleft_macro)] impl < 'a , P > ExternalProcess < 'a , P > { pub fn source_external_bytes < L : Location < 'a > + NoTick > (& self , to : & L ,) -> (ExternalBytesPort , Stream < Bytes , L , Unbounded >) { let next_external_port_id = { let mut flow_state = self . flow_state . borrow_mut () ; let id = flow_state . next_external_out ; flow_state . next_external_out += 1 ; id } ; (ExternalBytesPort { process_id : self . id , port_id : next_external_port_id , } , Stream :: new (to . clone () , HfPlusNode :: Persist (Box :: new (HfPlusNode :: Network { from_location : LocationId :: ExternalProcess (self . id) , from_key : Some (next_external_port_id) , to_location : to . id () , to_key : None , serialize_pipeline : None , instantiate_fn : crate :: ir :: DebugInstantiate :: Building () , deserialize_pipeline : Some (syn :: parse_quote ! (map (| b | b . unwrap () . freeze ()))) , input : Box :: new (HfPlusNode :: Source { source : HfPlusSource :: ExternalNetwork () , location_kind : LocationId :: ExternalProcess (self . id) , }) , })) ,) ,) } pub fn source_external_bincode < L : Location < 'a > + NoTick , T : Serialize + DeserializeOwned > (& self , to : & L ,) -> (ExternalBincodeSink < T > , Stream < T , L , Unbounded >) { let next_external_port_id = { let mut flow_state = self . flow_state . borrow_mut () ; let id = flow_state . next_external_out ; flow_state . next_external_out += 1 ; id } ; (ExternalBincodeSink { process_id : self . id , port_id : next_external_port_id , _phantom : PhantomData , } , Stream :: new (to . clone () , HfPlusNode :: Persist (Box :: new (HfPlusNode :: Network { from_location : LocationId :: ExternalProcess (self . id) , from_key : Some (next_external_port_id) , to_location : to . id () , to_key : None , serialize_pipeline : None , instantiate_fn : crate :: ir :: DebugInstantiate :: Building () , deserialize_pipeline : Some (crate :: stream :: deserialize_bincode :: < T > (None)) , input : Box :: new (HfPlusNode :: Source { source : HfPlusSource :: ExternalNetwork () , location_kind : LocationId :: ExternalProcess (self . id) , }) , })) ,) ,) } } } pub use external_process :: ExternalProcess ; pub mod process { pub use std :: marker :: PhantomData ; pub use super :: { Location , LocationId } ; pub use crate :: __staged :: builder :: FlowState ; pub use crate :: location :: process :: Process ; # [cfg (stageleft_macro)] impl < P > Clone for Process < '_ , P > { fn clone (& self) -> Self { Process { id : self . id , flow_state : self . flow_state . clone () , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < 'a , P > Location < 'a > for Process < 'a , P > { fn id (& self) -> LocationId { LocationId :: Process (self . id) } fn flow_state (& self) -> & FlowState { & self . flow_state } fn is_top_level () -> bool { true } } } pub use process :: Process ; pub mod cluster { pub use std :: fmt :: { Debug , Display } ; pub use std :: hash :: Hash ; pub use std :: marker :: PhantomData ; pub use proc_macro2 :: { Span , TokenStream } ; pub use quote :: quote ; pub use serde :: { Deserialize , Serialize } ; pub use stageleft :: runtime_support :: FreeVariable ; pub use stageleft :: { quote_type , Quoted } ; pub use super :: { Location , LocationId } ; pub use crate :: __staged :: builder :: FlowState ; pub use crate :: __staged :: staging_util :: get_this_crate ; pub use crate :: location :: cluster :: Cluster ; # [cfg (stageleft_macro)] impl < 'a , C > Cluster < 'a , C > { pub fn self_id (& self) -> impl Quoted < 'a , ClusterId < C > > + Copy + 'a { ClusterSelfId { id : self . id , _phantom : PhantomData , } } pub fn members (& self) -> impl Quoted < 'a , & 'a Vec < ClusterId < C > > > + Copy + 'a { ClusterIds { id : self . id , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < C > Clone for Cluster < '_ , C > { fn clone (& self) -> Self { Cluster { id : self . id , flow_state : self . flow_state . clone () , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < 'a , C > Location < 'a > for Cluster < 'a , C > { fn id (& self) -> LocationId { LocationId :: Cluster (self . id) } fn flow_state (& self) -> & FlowState { & self . flow_state } fn is_top_level () -> bool { true } } pub use crate :: location :: cluster :: ClusterId ; # [cfg (stageleft_macro)] impl < C > Debug for ClusterId < C > { fn fmt (& self , f : & mut std :: fmt :: Formatter < '_ >) -> std :: fmt :: Result { write ! (f , "ClusterId::<{}>({})" , std :: any :: type_name ::< C > () , self . raw_id) } } # [cfg (stageleft_macro)] impl < C > Display for ClusterId < C > { fn fmt (& self , f : & mut std :: fmt :: Formatter < '_ >) -> std :: fmt :: Result { write ! (f , "ClusterId::<{}>({})" , std :: any :: type_name ::< C > () , self . raw_id) } } # [cfg (stageleft_macro)] impl < C > Clone for ClusterId < C > { fn clone (& self) -> Self { * self } } # [cfg (stageleft_macro)] impl < C > Copy for ClusterId < C > { } # [cfg (stageleft_macro)] impl < C > Serialize for ClusterId < C > { fn serialize < S > (& self , serializer : S) -> Result < S :: Ok , S :: Error > where S : serde :: ser :: Serializer , { self . raw_id . serialize (serializer) } } # [cfg (stageleft_macro)] impl < 'de , C > Deserialize < 'de > for ClusterId < C > { fn deserialize < D > (deserializer : D) -> Result < Self , D :: Error > where D : serde :: de :: Deserializer < 'de > , { u32 :: deserialize (deserializer) . map (| id | ClusterId { raw_id : id , _phantom : PhantomData , }) } } # [cfg (stageleft_macro)] impl < C > PartialEq for ClusterId < C > { fn eq (& self , other : & Self) -> bool { self . raw_id == other . raw_id } } # [cfg (stageleft_macro)] impl < C > Eq for ClusterId < C > { } # [cfg (stageleft_macro)] impl < C > PartialOrd for ClusterId < C > { fn partial_cmp (& self , other : & Self) -> Option < std :: cmp :: Ordering > { Some (self . cmp (other)) } } # [cfg (stageleft_macro)] impl < C > Ord for ClusterId < C > { fn cmp (& self , other : & Self) -> std :: cmp :: Ordering { self . raw_id . cmp (& other . raw_id) } } # [cfg (stageleft_macro)] impl < C > Hash for ClusterId < C > { fn hash < H : std :: hash :: Hasher > (& self , state : & mut H) { self . raw_id . hash (state) } } # [cfg (stageleft_macro)] impl < C > ClusterId < C > { pub fn from_raw (id : u32) -> Self { ClusterId { raw_id : id , _phantom : PhantomData , } } } pub use crate :: location :: cluster :: ClusterIds ; # [cfg (stageleft_macro)] impl < C > Clone for ClusterIds < '_ , C > { fn clone (& self) -> Self { * self } } # [cfg (stageleft_macro)] impl < C > Copy for ClusterIds < '_ , C > { } # [cfg (stageleft_macro)] impl < 'a , C > FreeVariable < & 'a Vec < ClusterId < C > > > for ClusterIds < 'a , C > { fn to_tokens (self) -> (Option < TokenStream > , Option < TokenStream >) where Self : Sized , { let ident = syn :: Ident :: new (& format ! ("__hydroflow_plus_cluster_ids_{}" , self . id) , Span :: call_site () ,) ; let root = get_this_crate () ; let c_type = quote_type :: < C > () ; (None , Some (quote ! { unsafe { :: std :: mem :: transmute ::< _ , &:: std :: vec :: Vec <# root :: ClusterId <# c_type >>> (# ident) } } ,) ,) } } # [cfg (stageleft_macro)] impl < 'a , C > Quoted < 'a , & 'a Vec < ClusterId < C > > > for ClusterIds < 'a , C > { } pub use crate :: location :: cluster :: ClusterSelfId ; # [cfg (stageleft_macro)] impl < C > Clone for ClusterSelfId < '_ , C > { fn clone (& self) -> Self { * self } } # [cfg (stageleft_macro)] impl < C > Copy for ClusterSelfId < '_ , C > { } # [cfg (stageleft_macro)] impl < C > FreeVariable < ClusterId < C > > for ClusterSelfId < '_ , C > { fn to_tokens (self) -> (Option < TokenStream > , Option < TokenStream >) where Self : Sized , { let ident = syn :: Ident :: new (& format ! ("__hydroflow_plus_cluster_self_id_{}" , self . id) , Span :: call_site () ,) ; let root = get_this_crate () ; let c_type : syn :: Type = quote_type :: < C > () ; (None , Some (quote ! { # root :: ClusterId ::<# c_type >:: from_raw (# ident) }) ,) } } # [cfg (stageleft_macro)] impl < 'a , C > Quoted < 'a , ClusterId < C > > for ClusterSelfId < 'a , C > { } } pub use cluster :: { Cluster , ClusterId } ; pub mod can_send { pub use stageleft :: quote_type ; pub use super :: { Cluster , ClusterId , ExternalProcess , Location , Process } ; pub use crate :: location :: can_send :: CanSend ; # [cfg (stageleft_macro)] impl < 'a , P1 , P2 > CanSend < 'a , Process < 'a , P2 > > for Process < 'a , P1 > { type In < T > = T ; type Out < T > = T ; fn is_demux () -> bool { false } fn tagged_type () -> Option < syn :: Type > { None } } # [cfg (stageleft_macro)] impl < 'a , P1 , C2 > CanSend < 'a , Cluster < 'a , C2 > > for Process < 'a , P1 > { type In < T > = (ClusterId < C2 > , T) ; type Out < T > = T ; fn is_demux () -> bool { true } fn tagged_type () -> Option < syn :: Type > { None } } # [cfg (stageleft_macro)] impl < 'a , C1 , P2 > CanSend < 'a , Process < 'a , P2 > > for Cluster < 'a , C1 > { type In < T > = T ; type Out < T > = (ClusterId < C1 > , T) ; fn is_demux () -> bool { false } fn tagged_type () -> Option < syn :: Type > { Some (quote_type :: < C1 > ()) } } # [cfg (stageleft_macro)] impl < 'a , C1 , C2 > CanSend < 'a , Cluster < 'a , C2 > > for Cluster < 'a , C1 > { type In < T > = (ClusterId < C2 > , T) ; type Out < T > = (ClusterId < C1 > , T) ; fn is_demux () -> bool { true } fn tagged_type () -> Option < syn :: Type > { Some (quote_type :: < C1 > ()) } } # [cfg (stageleft_macro)] impl < 'a , P1 , E2 > CanSend < 'a , ExternalProcess < 'a , E2 > > for Process < 'a , P1 > { type In < T > = T ; type Out < T > = T ; fn is_demux () -> bool { false } fn tagged_type () -> Option < syn :: Type > { None } } } pub use can_send :: CanSend ; pub mod tick { pub use std :: marker :: PhantomData ; pub use proc_macro2 :: Span ; pub use stageleft :: { q , Quoted } ; pub use super :: { Cluster , Location , LocationId , Process } ; pub use crate :: __staged :: builder :: FlowState ; pub use crate :: __staged :: cycle :: { CycleCollection , CycleCollectionWithInitial , DeferTick , ForwardRef , ForwardRefMarker , TickCycle , TickCycleMarker , } ; pub use crate :: __staged :: ir :: { HfPlusNode , HfPlusSource } ; pub use crate :: __staged :: { Bounded , Optional , Singleton , Stream } ; pub use crate :: location :: tick :: NoTick ; # [cfg (stageleft_macro)] impl < T > NoTick for Process < '_ , T > { } # [cfg (stageleft_macro)] impl < T > NoTick for Cluster < '_ , T > { } pub use crate :: location :: tick :: Tick ; # [cfg (stageleft_macro)] impl < 'a , L : Location < 'a > > Location < 'a > for Tick < L > { fn id (& self) -> LocationId { LocationId :: Tick (self . id , Box :: new (self . l . id ())) } fn flow_state (& self) -> & FlowState { self . l . flow_state () } fn is_top_level () -> bool { false } } # [cfg (stageleft_macro)] impl < 'a , L : Location < 'a > > Tick < L > { pub fn outer (& self) -> & L { & self . l } pub fn spin_batch (& self , batch_size : impl Quoted < 'a , usize > + Copy + 'a ,) -> Stream < () , Self , Bounded > where L : NoTick , { self . l . spin () . flat_map (q ! (move | _ | 0 .. batch_size)) . map (q ! (| _ | ())) . tick_batch (self) } pub fn singleton < T : Clone > (& self , e : impl Quoted < 'a , T >) -> Singleton < T , Self , Bounded > where L : NoTick , { self . outer () . singleton (e) . latest_tick (self) } pub fn singleton_first_tick < T : Clone > (& self , e : impl Quoted < 'a , T > ,) -> Optional < T , Self , Bounded > where L : NoTick , { let e_arr = q ! ([e]) ; let e = e_arr . splice_untyped () ; Optional :: new (self . clone () , HfPlusNode :: Source { source : HfPlusSource :: Iter (e . into ()) , location_kind : self . l . id () , } ,) } pub fn forward_ref < S : CycleCollection < 'a , ForwardRefMarker , Location = Self > > (& self ,) -> (ForwardRef < 'a , S > , S) where L : NoTick , { let next_id = { let on_id = match self . l . id () { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (_) => panic ! () , } ; let mut flow_state = self . flow_state () . borrow_mut () ; let next_id_entry = flow_state . cycle_counts . entry (on_id) . or_default () ; let id = * next_id_entry ; * next_id_entry += 1 ; id } ; let ident = syn :: Ident :: new (& format ! ("cycle_{}" , next_id) , Span :: call_site ()) ; (ForwardRef { ident : ident . clone () , _phantom : PhantomData , } , S :: create_source (ident , self . clone ()) ,) } pub fn cycle < S : CycleCollection < 'a , TickCycleMarker , Location = Self > + DeferTick > (& self ,) -> (TickCycle < 'a , S > , S) where L : NoTick , { let next_id = { let on_id = match self . l . id () { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (_) => panic ! () , } ; let mut flow_state = self . flow_state () . borrow_mut () ; let next_id_entry = flow_state . cycle_counts . entry (on_id) . or_default () ; let id = * next_id_entry ; * next_id_entry += 1 ; id } ; let ident = syn :: Ident :: new (& format ! ("cycle_{}" , next_id) , Span :: call_site ()) ; (TickCycle { ident : ident . clone () , _phantom : PhantomData , } , S :: create_source (ident , self . clone ()) ,) } pub fn cycle_with_initial < S : CycleCollectionWithInitial < 'a , TickCycleMarker , Location = Self > + DeferTick , > (& self , initial : S ,) -> (TickCycle < 'a , S > , S) where L : NoTick , { let next_id = { let on_id = match self . l . id () { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (_) => panic ! () , } ; let mut flow_state = self . flow_state () . borrow_mut () ; let next_id_entry = flow_state . cycle_counts . entry (on_id) . or_default () ; let id = * next_id_entry ; * next_id_entry += 1 ; id } ; let ident = syn :: Ident :: new (& format ! ("cycle_{}" , next_id) , Span :: call_site ()) ; (TickCycle { ident : ident . clone () , _phantom : PhantomData , } , S :: create_source (ident , initial , self . clone ()) ,) } } } pub use tick :: { NoTick , Tick } ; pub use crate :: location :: LocationId ; # [cfg (stageleft_macro)] impl LocationId { pub fn root (& self) -> & LocationId { match self { LocationId :: Process (_) => self , LocationId :: Cluster (_) => self , LocationId :: Tick (_ , id) => id . root () , LocationId :: ExternalProcess (_) => self , } } pub fn raw_id (& self) -> usize { match self { LocationId :: Process (id) => * id , LocationId :: Cluster (id) => * id , LocationId :: Tick (_ , _) => panic ! ("cannot get raw id for tick") , LocationId :: ExternalProcess (id) => * id , } } } pub fn check_matching_location < 'a , L : Location < 'a > > (l1 : & L , l2 : & L) { assert_eq ! (l1 . id () , l2 . id () , "locations do not match") ; } pub use crate :: location :: Location ; } pub use location :: { Cluster , ClusterId , Location , Process , Tick } ; pub mod deploy { pub use std :: future :: Future ; pub use std :: io :: Error ; pub use std :: pin :: Pin ; pub use hydroflow :: bytes :: Bytes ; pub use hydroflow :: futures :: { Sink , Stream } ; pub use hydroflow_lang :: graph :: HydroflowGraph ; pub use serde :: de :: DeserializeOwned ; pub use serde :: Serialize ; pub use stageleft :: Quoted ; # [cfg (feature = "deploy_runtime")] pub mod macro_runtime { pub use std :: cell :: RefCell ; pub use std :: pin :: Pin ; pub use std :: rc :: Rc ; pub use hydroflow :: util :: deploy :: DeployPorts ; pub use stageleft :: { Quoted , RuntimeData } ; pub use super :: HydroflowPlusMeta ; pub use crate :: __staged :: deploy :: { ClusterSpec , Deploy , ExternalSpec , Node , ProcessSpec , RegisterPort } ; pub use crate :: __staged :: lang :: graph :: HydroflowGraph ; pub use crate :: deploy :: macro_runtime :: DeployRuntime ; # [cfg (stageleft_macro)] impl < 'a > Deploy < 'a > for DeployRuntime { type InstantiateEnv = () ; type CompileEnv = RuntimeData < & 'a DeployPorts < HydroflowPlusMeta > > ; type Process = DeployRuntimeNode ; type Cluster = DeployRuntimeCluster ; type ExternalProcess = DeployRuntimeNode ; type Port = String ; type ExternalRawPort = () ; type Meta = () ; type GraphId = usize ; fn has_trivial_node () -> bool { true } fn trivial_process (_id : usize) -> Self :: Process { DeployRuntimeNode { next_port : Rc :: new (RefCell :: new (0)) , } } fn trivail_cluster (_id : usize) -> Self :: Cluster { DeployRuntimeCluster { next_port : Rc :: new (RefCell :: new (0)) , } } fn allocate_process_port (process : & Self :: Process) -> Self :: Port { process . next_port () } fn allocate_cluster_port (cluster : & Self :: Cluster) -> Self :: Port { cluster . next_port () } fn allocate_external_port (_external : & Self :: ExternalProcess) -> Self :: Port { panic ! () ; } fn o2o_sink_source (env : & Self :: CompileEnv , _p1 : & Self :: Process , p1_port : & Self :: Port , _p2 : & Self :: Process , p2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { super :: deploy_runtime :: deploy_o2o (* env , p1_port . as_str () , p2_port . as_str ()) } fn o2o_connect (_p1 : & Self :: Process , _p1_port : & Self :: Port , _p2 : & Self :: Process , _p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { Box :: new (| | panic ! ()) } fn o2m_sink_source (env : & Self :: CompileEnv , _p1 : & Self :: Process , p1_port : & Self :: Port , _c2 : & Self :: Cluster , c2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { super :: deploy_runtime :: deploy_o2m (* env , p1_port . as_str () , c2_port . as_str ()) } fn o2m_connect (_p1 : & Self :: Process , _p1_port : & Self :: Port , _c2 : & Self :: Cluster , _c2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { Box :: new (| | panic ! ()) } fn m2o_sink_source (env : & Self :: CompileEnv , _c1 : & Self :: Cluster , c1_port : & Self :: Port , _p2 : & Self :: Process , p2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { super :: deploy_runtime :: deploy_m2o (* env , c1_port . as_str () , p2_port . as_str ()) } fn m2o_connect (_c1 : & Self :: Cluster , _c1_port : & Self :: Port , _p2 : & Self :: Process , _p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { Box :: new (| | panic ! ()) } fn m2m_sink_source (env : & Self :: CompileEnv , _c1 : & Self :: Cluster , c1_port : & Self :: Port , _c2 : & Self :: Cluster , c2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { super :: deploy_runtime :: deploy_m2m (* env , c1_port . as_str () , c2_port . as_str ()) } fn m2m_connect (_c1 : & Self :: Cluster , _c1_port : & Self :: Port , _c2 : & Self :: Cluster , _c2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { Box :: new (| | panic ! ()) } fn e2o_source (_compile_env : & Self :: CompileEnv , _p1 : & Self :: ExternalProcess , _p1_port : & Self :: Port , _p2 : & Self :: Process , _p2_port : & Self :: Port ,) -> syn :: Expr { panic ! () } fn e2o_connect (_p1 : & Self :: ExternalProcess , _p1_port : & Self :: Port , _p2 : & Self :: Process , _p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { panic ! () } fn o2e_sink (_compile_env : & Self :: CompileEnv , _p1 : & Self :: Process , _p1_port : & Self :: Port , _p2 : & Self :: ExternalProcess , _p2_port : & Self :: Port ,) -> syn :: Expr { panic ! () } fn o2e_connect (_p1 : & Self :: Process , _p1_port : & Self :: Port , _p2 : & Self :: ExternalProcess , _p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { panic ! () } fn cluster_ids (env : & Self :: CompileEnv , of_cluster : usize ,) -> impl Quoted < 'a , & 'a Vec < u32 > > + Copy + 'a { super :: deploy_runtime :: cluster_members (* env , of_cluster) } fn cluster_self_id (env : & Self :: CompileEnv) -> impl Quoted < 'a , u32 > + Copy + 'a { super :: deploy_runtime :: cluster_self_id (* env) } } pub use crate :: deploy :: macro_runtime :: DeployRuntimeNode ; # [cfg (stageleft_macro)] impl < 'a > RegisterPort < 'a , DeployRuntime > for DeployRuntimeNode { fn register (& self , _key : usize , _port : < DeployRuntime as Deploy > :: Port) { panic ! () } fn raw_port (& self , _key : usize) -> < DeployRuntime as Deploy > :: ExternalRawPort { panic ! () } # [expect (clippy :: manual_async_fn , reason = "buggy Clippy lint for lifetime bounds")] fn as_bytes_sink (& self , _key : usize ,) -> impl std :: future :: Future < Output = Pin < Box < dyn crate :: futures :: Sink < crate :: bytes :: Bytes , Error = std :: io :: Error > > > , > + 'a { async { panic ! () } } # [expect (clippy :: manual_async_fn , reason = "buggy Clippy lint for lifetime bounds")] fn as_bincode_sink < T : serde :: Serialize + 'static > (& self , _key : usize ,) -> impl std :: future :: Future < Output = Pin < Box < dyn crate :: futures :: Sink < T , Error = std :: io :: Error > > > , > + 'a { async { panic ! () } } # [expect (clippy :: manual_async_fn , reason = "buggy Clippy lint for lifetime bounds")] fn as_bytes_source (& self , _key : usize ,) -> impl std :: future :: Future < Output = Pin < Box < dyn hydroflow :: futures :: Stream < Item = hydroflow :: bytes :: Bytes > > > , > + 'a { async { panic ! () } } # [expect (clippy :: manual_async_fn , reason = "buggy Clippy lint for lifetime bounds")] fn as_bincode_source < T : serde :: de :: DeserializeOwned + 'static > (& self , _key : usize ,) -> impl std :: future :: Future < Output = Pin < Box < dyn hydroflow :: futures :: Stream < Item = T > > > > + 'a { async { panic ! () } } } # [cfg (stageleft_macro)] impl Node for DeployRuntimeNode { type Port = String ; type Meta = () ; type InstantiateEnv = () ; fn next_port (& self) -> String { let next_send_port = * self . next_port . borrow () ; * self . next_port . borrow_mut () += 1 ; format ! ("port_{}" , next_send_port) } fn update_meta (& mut self , _meta : & Self :: Meta) { } fn instantiate (& self , _env : & mut Self :: InstantiateEnv , _meta : & mut Self :: Meta , _graph : HydroflowGraph , _extra_stmts : Vec < syn :: Stmt > ,) { panic ! (".deploy() cannot be called on a DeployRuntimeNode") ; } } pub use crate :: deploy :: macro_runtime :: DeployRuntimeCluster ; # [cfg (stageleft_macro)] impl Node for DeployRuntimeCluster { type Port = String ; type Meta = () ; type InstantiateEnv = () ; fn next_port (& self) -> String { let next_send_port = * self . next_port . borrow () ; * self . next_port . borrow_mut () += 1 ; format ! ("port_{}" , next_send_port) } fn update_meta (& mut self , _meta : & Self :: Meta) { } fn instantiate (& self , _env : & mut Self :: InstantiateEnv , _meta : & mut Self :: Meta , _graph : HydroflowGraph , _extra_stmts : Vec < syn :: Stmt > ,) { panic ! (".deploy() cannot be called on a DeployRuntimeCluster") ; } } # [cfg (stageleft_macro)] impl ProcessSpec < '_ , DeployRuntime > for () { fn build (self , _id : usize , _name_hint : & str) -> DeployRuntimeNode { DeployRuntimeNode { next_port : Rc :: new (RefCell :: new (0)) , } } } # [cfg (stageleft_macro)] impl ClusterSpec < '_ , DeployRuntime > for () { fn build (self , _id : usize , _name_hint : & str) -> DeployRuntimeCluster { DeployRuntimeCluster { next_port : Rc :: new (RefCell :: new (0)) , } } } # [cfg (stageleft_macro)] impl ExternalSpec < '_ , DeployRuntime > for () { fn build (self , _id : usize , _name_hint : & str) -> DeployRuntimeNode { panic ! () } } } # [cfg (feature = "deploy")] pub mod trybuild { pub use std :: fs ; pub use std :: path :: PathBuf ; pub use stageleft :: internal :: quote ; pub use trybuild_internals_api :: cargo :: { self , Metadata } ; pub use trybuild_internals_api :: env :: Update ; pub use trybuild_internals_api :: run :: { PathDependency , Project } ; pub use trybuild_internals_api :: { dependencies , features , path , Runner } ; pub use crate :: __staged :: lang :: graph :: { partition_graph , HydroflowGraph } ; pub static IS_TEST : std :: sync :: atomic :: AtomicBool = std :: sync :: atomic :: AtomicBool :: new (false) ; pub fn init_test () { IS_TEST . store (true , std :: sync :: atomic :: Ordering :: Relaxed) ; } pub fn compile_graph_trybuild (graph : HydroflowGraph , extra_stmts : Vec < syn :: Stmt >) -> syn :: File { let partitioned_graph = partition_graph (graph) . expect ("Failed to partition (cycle detected).") ; let mut diagnostics = Vec :: new () ; let tokens = partitioned_graph . as_code (& quote ! { hydroflow_plus } , true , quote ! () , & mut diagnostics) ; let source_ast : syn :: File = syn :: parse_quote ! { #! [feature (box_patterns)] #! [allow (unused_crate_dependencies , missing_docs)] use hydroflow_plus ::*; # [allow (unused)] fn __hfplus_runtime <'a > (__hydroflow_plus_trybuild_cli : &'a hydroflow_plus :: util :: deploy :: DeployPorts < hydroflow_plus :: deploy :: HydroflowPlusMeta >) -> hydroflow_plus :: Hydroflow <'a > { # (# extra_stmts) * # tokens } # [tokio :: main] async fn main () { let ports = hydroflow_plus :: util :: deploy :: init_no_ack_start () . await ; let flow = __hfplus_runtime (& ports) ; println ! ("ack start") ; hydroflow_plus :: util :: deploy :: launch_flow (flow) . await ; } } ; source_ast } pub fn create_trybuild (source : & str , bin : & str , is_test : bool ,) -> Result < (PathBuf , PathBuf , Option < Vec < String > >) , trybuild_internals_api :: error :: Error > { let Metadata { target_directory : target_dir , workspace_root : workspace , packages , } = cargo :: metadata () ? ; let source_dir = cargo :: manifest_dir () ? ; let mut source_manifest = dependencies :: get_manifest (& source_dir) ? ; if ! is_test { source_manifest . dev_dependencies . clear () ; } let mut features = features :: find () ; let path_dependencies = source_manifest . dependencies . iter () . filter_map (| (name , dep) | { let path = dep . path . as_ref () ? ; if packages . iter () . any (| p | & p . name == name) { None } else { Some (PathDependency { name : name . clone () , normalized_path : path . canonicalize () . ok () ? , }) } }) . collect () ; let crate_name = source_manifest . package . name . clone () ; let project_dir = path ! (target_dir / "hfplus_trybuild" / crate_name /) ; fs :: create_dir_all (& project_dir) ? ; let project_name = format ! ("{}-hfplus-trybuild" , crate_name) ; let mut manifest = Runner :: make_manifest (& workspace , & project_name , & source_dir , & packages , & [] , source_manifest ,) ? ; manifest . features . remove ("stageleft_devel") ; if let Some (enabled_features) = & mut features { enabled_features . retain (| feature | manifest . features . contains_key (feature) || feature == "default") ; manifest . features . get_mut ("default") . iter_mut () . for_each (| v | { v . retain (| f | f != "stageleft_devel") ; }) ; } let project = Project { dir : project_dir , source_dir , target_dir , name : project_name , update : Update :: env () ? , has_pass : false , has_compile_fail : false , features , workspace , path_dependencies , manifest , keep_going : false , } ; let manifest_toml = toml :: to_string (& project . manifest) ? ; fs :: write (path ! (project . dir / "Cargo.toml") , manifest_toml) ? ; fs :: create_dir_all (path ! (project . dir / "src" / "bin")) ? ; let out_path = path ! (project . dir / "src" / "bin" / format ! ("{bin}.rs")) ; if ! out_path . exists () || fs :: read_to_string (& out_path) ? != source { fs :: write (path ! (project . dir / "src" / "bin" / format ! ("{bin}.rs")) , source ,) ? ; } let workspace_cargo_lock = path ! (project . workspace / "Cargo.lock") ; if workspace_cargo_lock . exists () { let _ = fs :: copy (workspace_cargo_lock , path ! (project . dir / "Cargo.lock")) ; } else { let _ = cargo :: cargo (& project) . arg ("generate-lockfile") . status () ; } let workspace_dot_cargo_config_toml = path ! (project . workspace / ".cargo" / "config.toml") ; if workspace_dot_cargo_config_toml . exists () { let dot_cargo_folder = path ! (project . dir / ".cargo") ; fs :: create_dir_all (& dot_cargo_folder) ? ; let _ = fs :: copy (workspace_dot_cargo_config_toml , path ! (dot_cargo_folder / "config.toml") ,) ; } Ok ((project . dir . as_ref () . into () , path ! (project . target_dir / "hfplus_trybuild") , project . features ,)) } } pub use macro_runtime :: * ; # [cfg (feature = "deploy")] pub use trybuild :: init_test ; # [cfg (feature = "deploy_runtime")] pub mod deploy_runtime { pub use std :: collections :: HashMap ; pub use hydroflow :: util :: deploy :: { ConnectedDemux , ConnectedDirect , ConnectedSink , ConnectedSource , ConnectedTagged , DeployPorts , } ; pub use serde :: { Deserialize , Serialize } ; pub use stageleft :: { q , Quoted , RuntimeData } ; pub use crate :: deploy :: deploy_runtime :: HydroflowPlusMeta ; pub fn cluster_members (cli : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , of_cluster : usize ,) -> impl Quoted < & Vec < u32 > > + Copy { q ! (cli . meta . clusters . get (& of_cluster) . unwrap ()) } pub fn cluster_self_id (cli : RuntimeData < & DeployPorts < HydroflowPlusMeta > > ,) -> impl Quoted <'_, u32 > + Copy { q ! (cli . meta . cluster_id . expect ("Tried to read Cluster ID on a non-cluster node")) } pub fn deploy_o2o (env : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , p1_port : & str , p2_port : & str ,) -> (syn :: Expr , syn :: Expr) { ({ q ! ({ env . port (p1_port) . connect_local_blocking ::< ConnectedDirect > () . into_sink () }) . splice_untyped () } , { q ! ({ env . port (p2_port) . connect_local_blocking ::< ConnectedDirect > () . into_source () }) . splice_untyped () } ,) } pub fn deploy_o2m (env : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , p1_port : & str , c2_port : & str ,) -> (syn :: Expr , syn :: Expr) { ({ q ! ({ env . port (p1_port) . connect_local_blocking ::< ConnectedDemux < ConnectedDirect >> () . into_sink () }) . splice_untyped () } , { q ! ({ env . port (c2_port) . connect_local_blocking ::< ConnectedDirect > () . into_source () }) . splice_untyped () } ,) } pub fn deploy_m2o (env : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , c1_port : & str , p2_port : & str ,) -> (syn :: Expr , syn :: Expr) { ({ q ! ({ env . port (c1_port) . connect_local_blocking ::< ConnectedDirect > () . into_sink () }) . splice_untyped () } , { q ! ({ env . port (p2_port) . connect_local_blocking ::< ConnectedTagged < ConnectedDirect >> () . into_source () }) . splice_untyped () } ,) } pub fn deploy_m2m (env : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , c1_port : & str , c2_port : & str ,) -> (syn :: Expr , syn :: Expr) { ({ q ! ({ env . port (c1_port) . connect_local_blocking ::< ConnectedDemux < ConnectedDirect >> () . into_sink () }) . splice_untyped () } , { q ! ({ env . port (c2_port) . connect_local_blocking ::< ConnectedTagged < ConnectedDirect >> () . into_source () }) . splice_untyped () } ,) } pub fn deploy_e2o (env : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , _e1_port : & str , p2_port : & str ,) -> syn :: Expr { q ! ({ env . port (p2_port) . connect_local_blocking ::< ConnectedDirect > () . into_source () }) . splice_untyped () } pub fn deploy_o2e (env : RuntimeData < & DeployPorts < HydroflowPlusMeta > > , p1_port : & str , _e2_port : & str ,) -> syn :: Expr { q ! ({ env . port (p1_port) . connect_local_blocking ::< ConnectedDirect > () . into_sink () }) . splice_untyped () } } # [cfg (feature = "deploy_runtime")] pub use deploy_runtime :: HydroflowPlusMeta ; # [cfg (feature = "deploy")] pub mod deploy_graph { pub use std :: cell :: RefCell ; pub use std :: collections :: HashMap ; pub use std :: future :: Future ; pub use std :: io :: Error ; pub use std :: pin :: Pin ; pub use std :: rc :: Rc ; pub use std :: sync :: Arc ; pub use hydro_deploy :: custom_service :: CustomClientPort ; pub use hydro_deploy :: hydroflow_crate :: ports :: { DemuxSink , HydroflowSink , HydroflowSource , TaggedSource , } ; pub use hydro_deploy :: hydroflow_crate :: tracing_options :: TracingOptions ; pub use hydro_deploy :: hydroflow_crate :: HydroflowCrateService ; pub use hydro_deploy :: { CustomService , Deployment , Host , HydroflowCrate } ; pub use hydroflow :: futures :: StreamExt ; pub use hydroflow :: util :: deploy :: { ConnectedSink , ConnectedSource } ; pub use nameof :: name_of ; pub use serde :: de :: DeserializeOwned ; pub use serde :: Serialize ; pub use sha2 :: { Digest , Sha256 } ; pub use stageleft :: { Quoted , RuntimeData } ; pub use syn :: visit_mut :: VisitMut ; pub use tokio :: sync :: RwLock ; pub use trybuild_internals_api :: path ; pub use super :: deploy_runtime :: * ; pub use super :: trybuild :: { compile_graph_trybuild , create_trybuild } ; pub use super :: { ClusterSpec , Deploy , ExternalSpec , IntoProcessSpec , Node , ProcessSpec , RegisterPort } ; pub use crate :: __staged :: futures :: SinkExt ; pub use crate :: __staged :: lang :: graph :: HydroflowGraph ; pub use crate :: deploy :: deploy_graph :: HydroDeploy ; # [cfg (stageleft_macro)] impl < 'a > Deploy < 'a > for HydroDeploy { type InstantiateEnv = Deployment ; type CompileEnv = () ; type Process = DeployNode ; type Cluster = DeployCluster ; type ExternalProcess = DeployExternal ; type Meta = HashMap < usize , Vec < u32 > > ; type GraphId = () ; type Port = String ; type ExternalRawPort = CustomClientPort ; fn allocate_process_port (process : & Self :: Process) -> Self :: Port { process . next_port () } fn allocate_cluster_port (cluster : & Self :: Cluster) -> Self :: Port { cluster . next_port () } fn allocate_external_port (external : & Self :: ExternalProcess) -> Self :: Port { external . next_port () } fn o2o_sink_source (_env : & () , _p1 : & Self :: Process , p1_port : & Self :: Port , _p2 : & Self :: Process , p2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { let p1_port = p1_port . as_str () ; let p2_port = p2_port . as_str () ; deploy_o2o (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , p1_port , p2_port ,) } fn o2o_connect (p1 : & Self :: Process , p1_port : & Self :: Port , p2 : & Self :: Process , p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { let p1 = p1 . clone () ; let p1_port = p1_port . clone () ; let p2 = p2 . clone () ; let p2_port = p2_port . clone () ; Box :: new (move | | { let self_underlying_borrow = p1 . underlying . borrow () ; let self_underlying = self_underlying_borrow . as_ref () . unwrap () ; let source_port = self_underlying . try_read () . unwrap () . get_port (p1_port . clone () , self_underlying) ; let other_underlying_borrow = p2 . underlying . borrow () ; let other_underlying = other_underlying_borrow . as_ref () . unwrap () ; let recipient_port = other_underlying . try_read () . unwrap () . get_port (p2_port . clone () , other_underlying) ; source_port . send_to (& recipient_port) }) } fn o2m_sink_source (_env : & () , _p1 : & Self :: Process , p1_port : & Self :: Port , _c2 : & Self :: Cluster , c2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { let p1_port = p1_port . as_str () ; let c2_port = c2_port . as_str () ; deploy_o2m (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , p1_port , c2_port ,) } fn o2m_connect (p1 : & Self :: Process , p1_port : & Self :: Port , c2 : & Self :: Cluster , c2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { let p1 = p1 . clone () ; let p1_port = p1_port . clone () ; let c2 = c2 . clone () ; let c2_port = c2_port . clone () ; Box :: new (move | | { let self_underlying_borrow = p1 . underlying . borrow () ; let self_underlying = self_underlying_borrow . as_ref () . unwrap () ; let source_port = self_underlying . try_read () . unwrap () . get_port (p1_port . clone () , self_underlying) ; let recipient_port = DemuxSink { demux : c2 . members . borrow () . iter () . enumerate () . map (| (id , c) | { let n = c . underlying . try_read () . unwrap () ; (id as u32 , Arc :: new (n . get_port (c2_port . clone () , & c . underlying)) as Arc < dyn HydroflowSink + 'static > ,) }) . collect () , } ; source_port . send_to (& recipient_port) }) } fn m2o_sink_source (_env : & () , _c1 : & Self :: Cluster , c1_port : & Self :: Port , _p2 : & Self :: Process , p2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { let c1_port = c1_port . as_str () ; let p2_port = p2_port . as_str () ; deploy_m2o (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , c1_port , p2_port ,) } fn m2o_connect (c1 : & Self :: Cluster , c1_port : & Self :: Port , p2 : & Self :: Process , p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { let c1 = c1 . clone () ; let c1_port = c1_port . clone () ; let p2 = p2 . clone () ; let p2_port = p2_port . clone () ; Box :: new (move | | { let other_underlying_borrow = p2 . underlying . borrow () ; let other_underlying = other_underlying_borrow . as_ref () . unwrap () ; let recipient_port = other_underlying . try_read () . unwrap () . get_port (p2_port . clone () , other_underlying) . merge () ; for (i , node) in c1 . members . borrow () . iter () . enumerate () { let source_port = node . underlying . try_read () . unwrap () . get_port (c1_port . clone () , & node . underlying) ; TaggedSource { source : Arc :: new (source_port) , tag : i as u32 , } . send_to (& recipient_port) ; } }) } fn m2m_sink_source (_env : & () , _c1 : & Self :: Cluster , c1_port : & Self :: Port , _c2 : & Self :: Cluster , c2_port : & Self :: Port ,) -> (syn :: Expr , syn :: Expr) { let c1_port = c1_port . as_str () ; let c2_port = c2_port . as_str () ; deploy_m2m (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , c1_port , c2_port ,) } fn m2m_connect (c1 : & Self :: Cluster , c1_port : & Self :: Port , c2 : & Self :: Cluster , c2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { let c1 = c1 . clone () ; let c1_port = c1_port . clone () ; let c2 = c2 . clone () ; let c2_port = c2_port . clone () ; Box :: new (move | | { for (i , sender) in c1 . members . borrow () . iter () . enumerate () { let source_port = sender . underlying . try_read () . unwrap () . get_port (c1_port . clone () , & sender . underlying) ; let recipient_port = DemuxSink { demux : c2 . members . borrow () . iter () . enumerate () . map (| (id , c) | { let n = c . underlying . try_read () . unwrap () ; (id as u32 , Arc :: new (n . get_port (c2_port . clone () , & c . underlying) . merge ()) as Arc < dyn HydroflowSink + 'static > ,) }) . collect () , } ; TaggedSource { source : Arc :: new (source_port) , tag : i as u32 , } . send_to (& recipient_port) ; } }) } fn e2o_source (_compile_env : & Self :: CompileEnv , _p1 : & Self :: ExternalProcess , p1_port : & Self :: Port , _p2 : & Self :: Process , p2_port : & Self :: Port ,) -> syn :: Expr { let p1_port = p1_port . as_str () ; let p2_port = p2_port . as_str () ; deploy_e2o (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , p1_port , p2_port ,) } fn e2o_connect (p1 : & Self :: ExternalProcess , p1_port : & Self :: Port , p2 : & Self :: Process , p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { let p1 = p1 . clone () ; let p1_port = p1_port . clone () ; let p2 = p2 . clone () ; let p2_port = p2_port . clone () ; Box :: new (move | | { let self_underlying_borrow = p1 . underlying . borrow () ; let self_underlying = self_underlying_borrow . as_ref () . unwrap () ; let source_port = self_underlying . try_read () . unwrap () . declare_client (self_underlying) ; let other_underlying_borrow = p2 . underlying . borrow () ; let other_underlying = other_underlying_borrow . as_ref () . unwrap () ; let recipient_port = other_underlying . try_read () . unwrap () . get_port (p2_port . clone () , other_underlying) ; source_port . send_to (& recipient_port) ; p1 . client_ports . borrow_mut () . insert (p1_port . clone () , source_port) ; }) } fn o2e_sink (_compile_env : & Self :: CompileEnv , _p1 : & Self :: Process , p1_port : & Self :: Port , _p2 : & Self :: ExternalProcess , p2_port : & Self :: Port ,) -> syn :: Expr { let p1_port = p1_port . as_str () ; let p2_port = p2_port . as_str () ; deploy_o2e (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , p1_port , p2_port ,) } fn o2e_connect (p1 : & Self :: Process , p1_port : & Self :: Port , p2 : & Self :: ExternalProcess , p2_port : & Self :: Port ,) -> Box < dyn FnOnce () > { let p1 = p1 . clone () ; let p1_port = p1_port . clone () ; let p2 = p2 . clone () ; let p2_port = p2_port . clone () ; Box :: new (move | | { let self_underlying_borrow = p1 . underlying . borrow () ; let self_underlying = self_underlying_borrow . as_ref () . unwrap () ; let source_port = self_underlying . try_read () . unwrap () . get_port (p1_port . clone () , self_underlying) ; let other_underlying_borrow = p2 . underlying . borrow () ; let other_underlying = other_underlying_borrow . as_ref () . unwrap () ; let recipient_port = other_underlying . try_read () . unwrap () . declare_client (other_underlying) ; source_port . send_to (& recipient_port) ; p2 . client_ports . borrow_mut () . insert (p2_port . clone () , recipient_port) ; }) } fn cluster_ids (_env : & Self :: CompileEnv , of_cluster : usize ,) -> impl Quoted < 'a , & 'a Vec < u32 > > + Copy + 'a { cluster_members (RuntimeData :: new ("__hydroflow_plus_trybuild_cli") , of_cluster ,) } fn cluster_self_id (_env : & Self :: CompileEnv) -> impl Quoted < 'a , u32 > + Copy + 'a { cluster_self_id (RuntimeData :: new ("__hydroflow_plus_trybuild_cli")) } } pub use crate :: deploy :: deploy_graph :: DeployCrateWrapper ; pub use crate :: deploy :: deploy_graph :: TrybuildHost ; # [cfg (stageleft_macro)] impl From < Arc < dyn Host > > for TrybuildHost { fn from (host : Arc < dyn Host >) -> Self { Self { host , display_name : None , rustflags : None , tracing : None , name_hint : None , cluster_idx : None , } } } # [cfg (stageleft_macro)] impl < H : Host + 'static > From < Arc < H > > for TrybuildHost { fn from (host : Arc < H >) -> Self { Self { host , display_name : None , rustflags : None , tracing : None , name_hint : None , cluster_idx : None , } } } # [cfg (stageleft_macro)] impl TrybuildHost { pub fn new (host : Arc < dyn Host >) -> Self { Self { host , display_name : None , rustflags : None , tracing : None , name_hint : None , cluster_idx : None , } } pub fn display_name (self , display_name : impl Into < String >) -> Self { if self . display_name . is_some () { panic ! ("{} already set" , name_of ! (display_name in Self)) ; } Self { display_name : Some (display_name . into ()) , .. self } } pub fn rustflags (self , rustflags : impl Into < String >) -> Self { if self . rustflags . is_some () { panic ! ("{} already set" , name_of ! (rustflags in Self)) ; } Self { rustflags : Some (rustflags . into ()) , .. self } } pub fn tracing (self , tracing : TracingOptions) -> Self { if self . tracing . is_some () { panic ! ("{} already set" , name_of ! (tracing in Self)) ; } Self { tracing : Some (tracing) , .. self } } } # [cfg (stageleft_macro)] impl IntoProcessSpec < '_ , HydroDeploy > for Arc < dyn Host > { type ProcessSpec = TrybuildHost ; fn into_process_spec (self) -> TrybuildHost { TrybuildHost { host : self , display_name : None , rustflags : None , tracing : None , name_hint : None , cluster_idx : None , } } } # [cfg (stageleft_macro)] impl < H : Host + 'static > IntoProcessSpec < '_ , HydroDeploy > for Arc < H > { type ProcessSpec = TrybuildHost ; fn into_process_spec (self) -> TrybuildHost { TrybuildHost { host : self , display_name : None , rustflags : None , tracing : None , name_hint : None , cluster_idx : None , } } } pub use crate :: deploy :: deploy_graph :: DeployExternal ; # [cfg (stageleft_macro)] impl DeployExternal { pub fn take_port (& self , key : usize) -> CustomClientPort { self . client_ports . borrow_mut () . remove (self . allocated_ports . borrow () . get (& key) . unwrap ()) . unwrap () } } # [cfg (stageleft_macro)] impl < 'a > RegisterPort < 'a , HydroDeploy > for DeployExternal { fn register (& self , key : usize , port : < HydroDeploy as Deploy > :: Port) { self . allocated_ports . borrow_mut () . insert (key , port) ; } fn raw_port (& self , key : usize) -> < HydroDeploy as Deploy > :: ExternalRawPort { self . client_ports . borrow_mut () . remove (self . allocated_ports . borrow () . get (& key) . unwrap ()) . unwrap () } fn as_bytes_sink (& self , key : usize ,) -> impl Future < Output = Pin < Box < dyn crate :: futures :: Sink < crate :: bytes :: Bytes , Error = Error > > > > + 'a { let port = self . raw_port (key) ; async move { let sink = port . connect () . await . into_sink () ; sink as Pin < Box < dyn crate :: futures :: Sink < crate :: bytes :: Bytes , Error = Error > > > } } fn as_bincode_sink < T : Serialize + 'static > (& self , key : usize ,) -> impl Future < Output = Pin < Box < dyn crate :: futures :: Sink < T , Error = Error > > > > + 'a { let port = self . raw_port (key) ; async move { let sink = port . connect () . await . into_sink () ; Box :: pin (sink . with (| item | async move { Ok (bincode :: serialize (& item) . unwrap () . into ()) })) as Pin < Box < dyn crate :: futures :: Sink < T , Error = Error > > > } } fn as_bytes_source (& self , key : usize ,) -> impl Future < Output = Pin < Box < dyn crate :: futures :: Stream < Item = crate :: bytes :: Bytes > > > > + 'a { let port = self . raw_port (key) ; async move { let source = port . connect () . await . into_source () ; Box :: pin (source . map (| r | r . unwrap () . freeze ())) as Pin < Box < dyn crate :: futures :: Stream < Item = crate :: bytes :: Bytes > > > } } fn as_bincode_source < T : DeserializeOwned + 'static > (& self , key : usize ,) -> impl Future < Output = Pin < Box < dyn crate :: futures :: Stream < Item = T > > > > + 'a { let port = self . raw_port (key) ; async move { let source = port . connect () . await . into_source () ; Box :: pin (source . map (| item | bincode :: deserialize (& item . unwrap ()) . unwrap ())) as Pin < Box < dyn crate :: futures :: Stream < Item = T > > > } } } # [cfg (stageleft_macro)] impl Node for DeployExternal { type Port = String ; type Meta = HashMap < usize , Vec < u32 > > ; type InstantiateEnv = Deployment ; fn next_port (& self) -> Self :: Port { let next_port = * self . next_port . borrow () ; * self . next_port . borrow_mut () += 1 ; format ! ("port_{}" , next_port) } fn instantiate (& self , env : & mut Self :: InstantiateEnv , _meta : & mut Self :: Meta , _graph : HydroflowGraph , _extra_stmts : Vec < syn :: Stmt > ,) { let service = env . CustomService (self . host . clone () , vec ! []) ; * self . underlying . borrow_mut () = Some (service) ; } fn update_meta (& mut self , _meta : & Self :: Meta) { } } # [cfg (stageleft_macro)] impl ExternalSpec < '_ , HydroDeploy > for Arc < dyn Host > { fn build (self , _id : usize , _name_hint : & str) -> DeployExternal { DeployExternal { next_port : Rc :: new (RefCell :: new (0)) , host : self , underlying : Rc :: new (RefCell :: new (None)) , allocated_ports : Rc :: new (RefCell :: new (HashMap :: new ())) , client_ports : Rc :: new (RefCell :: new (HashMap :: new ())) , } } } # [cfg (stageleft_macro)] impl < H : Host + 'static > ExternalSpec < '_ , HydroDeploy > for Arc < H > { fn build (self , _id : usize , _name_hint : & str) -> DeployExternal { DeployExternal { next_port : Rc :: new (RefCell :: new (0)) , host : self , underlying : Rc :: new (RefCell :: new (None)) , allocated_ports : Rc :: new (RefCell :: new (HashMap :: new ())) , client_ports : Rc :: new (RefCell :: new (HashMap :: new ())) , } } } pub use crate :: deploy :: deploy_graph :: CrateOrTrybuild ; pub use crate :: deploy :: deploy_graph :: DeployNode ; # [cfg (stageleft_macro)] impl DeployCrateWrapper for DeployNode { fn underlying (& self) -> Arc < RwLock < HydroflowCrateService > > { self . underlying . borrow () . as_ref () . unwrap () . clone () } } # [cfg (stageleft_macro)] impl Node for DeployNode { type Port = String ; type Meta = HashMap < usize , Vec < u32 > > ; type InstantiateEnv = Deployment ; fn next_port (& self) -> String { let next_port = * self . next_port . borrow () ; * self . next_port . borrow_mut () += 1 ; format ! ("port_{}" , next_port) } fn update_meta (& mut self , meta : & Self :: Meta) { let underlying_node = self . underlying . borrow () ; let mut n = underlying_node . as_ref () . unwrap () . try_write () . unwrap () ; n . update_meta (HydroflowPlusMeta { clusters : meta . clone () , cluster_id : None , subgraph_id : self . id , }) ; } fn instantiate (& self , env : & mut Self :: InstantiateEnv , _meta : & mut Self :: Meta , graph : HydroflowGraph , extra_stmts : Vec < syn :: Stmt > ,) { let service = match self . service_spec . borrow_mut () . take () . unwrap () { CrateOrTrybuild :: Crate (c) => c , CrateOrTrybuild :: Trybuild (trybuild) => { let (bin_name , (dir , target_dir , features)) = create_graph_trybuild (graph , extra_stmts , & trybuild . name_hint) ; create_trybuild_service (trybuild , & dir , & target_dir , & features , & bin_name) } } ; * self . underlying . borrow_mut () = Some (env . add_service (service)) ; } } pub use crate :: deploy :: deploy_graph :: DeployClusterNode ; # [cfg (stageleft_macro)] impl DeployCrateWrapper for DeployClusterNode { fn underlying (& self) -> Arc < RwLock < HydroflowCrateService > > { self . underlying . clone () } } pub use crate :: deploy :: deploy_graph :: DeployCluster ; # [cfg (stageleft_macro)] impl DeployCluster { pub fn members (& self) -> Vec < DeployClusterNode > { self . members . borrow () . clone () } } # [cfg (stageleft_macro)] impl Node for DeployCluster { type Port = String ; type Meta = HashMap < usize , Vec < u32 > > ; type InstantiateEnv = Deployment ; fn next_port (& self) -> String { let next_port = * self . next_port . borrow () ; * self . next_port . borrow_mut () += 1 ; format ! ("port_{}" , next_port) } fn instantiate (& self , env : & mut Self :: InstantiateEnv , meta : & mut Self :: Meta , graph : HydroflowGraph , extra_stmts : Vec < syn :: Stmt > ,) { let has_trybuild = self . cluster_spec . borrow () . as_ref () . unwrap () . iter () . any (| spec | matches ! (spec , CrateOrTrybuild :: Trybuild { .. })) ; let maybe_trybuild = if has_trybuild { Some (create_graph_trybuild (graph , extra_stmts , & self . name_hint)) } else { None } ; let cluster_nodes = self . cluster_spec . borrow_mut () . take () . unwrap () . into_iter () . map (| spec | { let service = match spec { CrateOrTrybuild :: Crate (c) => c , CrateOrTrybuild :: Trybuild (trybuild) => { let (bin_name , (dir , target_dir , features)) = maybe_trybuild . as_ref () . unwrap () ; create_trybuild_service (trybuild , dir , target_dir , features , bin_name) } } ; env . add_service (service) }) . collect :: < Vec < _ > > () ; meta . insert (self . id , (0 .. (cluster_nodes . len () as u32)) . collect ()) ; * self . members . borrow_mut () = cluster_nodes . into_iter () . map (| n | DeployClusterNode { underlying : n }) . collect () ; } fn update_meta (& mut self , meta : & Self :: Meta) { for (cluster_id , node) in self . members . borrow () . iter () . enumerate () { let mut n = node . underlying . try_write () . unwrap () ; n . update_meta (HydroflowPlusMeta { clusters : meta . clone () , cluster_id : Some (cluster_id as u32) , subgraph_id : self . id , }) ; } } } pub use crate :: deploy :: deploy_graph :: DeployProcessSpec ; # [cfg (stageleft_macro)] impl DeployProcessSpec { pub fn new (t : HydroflowCrate) -> Self { Self (t) } } # [cfg (stageleft_macro)] impl ProcessSpec < '_ , HydroDeploy > for DeployProcessSpec { fn build (self , id : usize , _name_hint : & str) -> DeployNode { DeployNode { id , next_port : Rc :: new (RefCell :: new (0)) , service_spec : Rc :: new (RefCell :: new (Some (CrateOrTrybuild :: Crate (self . 0)))) , underlying : Rc :: new (RefCell :: new (None)) , } } } # [cfg (stageleft_macro)] impl ProcessSpec < '_ , HydroDeploy > for TrybuildHost { fn build (mut self , id : usize , name_hint : & str) -> DeployNode { self . name_hint = Some (format ! ("{} (process {id})" , name_hint)) ; DeployNode { id , next_port : Rc :: new (RefCell :: new (0)) , service_spec : Rc :: new (RefCell :: new (Some (CrateOrTrybuild :: Trybuild (self)))) , underlying : Rc :: new (RefCell :: new (None)) , } } } pub use crate :: deploy :: deploy_graph :: DeployClusterSpec ; # [cfg (stageleft_macro)] impl DeployClusterSpec { pub fn new (crates : Vec < HydroflowCrate >) -> Self { Self (crates) } } # [cfg (stageleft_macro)] impl ClusterSpec < '_ , HydroDeploy > for DeployClusterSpec { fn build (self , id : usize , _name_hint : & str) -> DeployCluster { DeployCluster { id , next_port : Rc :: new (RefCell :: new (0)) , cluster_spec : Rc :: new (RefCell :: new (Some (self . 0 . into_iter () . map (CrateOrTrybuild :: Crate) . collect () ,))) , members : Rc :: new (RefCell :: new (vec ! [])) , name_hint : None , } } } # [cfg (stageleft_macro)] impl < T : Into < TrybuildHost > , I : IntoIterator < Item = T > > ClusterSpec < '_ , HydroDeploy > for I { fn build (self , id : usize , name_hint : & str) -> DeployCluster { let name_hint = format ! ("{} (cluster {id})" , name_hint) ; DeployCluster { id , next_port : Rc :: new (RefCell :: new (0)) , cluster_spec : Rc :: new (RefCell :: new (Some (self . into_iter () . enumerate () . map (| (idx , b) | { let mut b = b . into () ; b . name_hint = Some (name_hint . clone ()) ; b . cluster_idx = Some (idx) ; CrateOrTrybuild :: Trybuild (b) }) . collect () ,))) , members : Rc :: new (RefCell :: new (vec ! [])) , name_hint : Some (name_hint) , } } } pub fn clean_name_hint (name_hint : & str) -> String { name_hint . replace ("::" , "__") . replace (" " , "_") . replace ("," , "_") . replace ("<" , "_") . replace (">" , "") . replace ("(" , "") . replace (")" , "") } pub use crate :: deploy :: deploy_graph :: ReplaceCrateNameWithStaged ; # [cfg (stageleft_macro)] impl VisitMut for ReplaceCrateNameWithStaged { fn visit_type_path_mut (& mut self , i : & mut syn :: TypePath) { if let Some (first) = i . path . segments . first () { if first . ident == self . crate_name { let tail = i . path . segments . iter () . skip (1) . collect :: < Vec < _ > > () ; * i = syn :: parse_quote ! (crate :: __staged # (::# tail) *) ; } } syn :: visit_mut :: visit_type_path_mut (self , i) ; } } pub use crate :: deploy :: deploy_graph :: ReplaceCrateWithOrig ; # [cfg (stageleft_macro)] impl VisitMut for ReplaceCrateWithOrig { fn visit_item_use_mut (& mut self , i : & mut syn :: ItemUse) { if let syn :: UseTree :: Path (p) = & mut i . tree { if p . ident == "crate" { p . ident = syn :: Ident :: new (& self . crate_name , p . ident . span ()) ; i . leading_colon = Some (Default :: default ()) ; } } syn :: visit_mut :: visit_item_use_mut (self , i) ; } } pub fn create_graph_trybuild (graph : HydroflowGraph , extra_stmts : Vec < syn :: Stmt > , name_hint : & Option < String > ,) -> (String , (std :: path :: PathBuf , std :: path :: PathBuf , Option < Vec < String > >) ,) { let source_dir = trybuild_internals_api :: cargo :: manifest_dir () . unwrap () ; let source_manifest = trybuild_internals_api :: dependencies :: get_manifest (& source_dir) . unwrap () ; let crate_name = & source_manifest . package . name . to_string () . replace ("-" , "_") ; let is_test = super :: trybuild :: IS_TEST . load (std :: sync :: atomic :: Ordering :: Relaxed) ; let mut generated_code = compile_graph_trybuild (graph , extra_stmts) ; ReplaceCrateNameWithStaged { crate_name : crate_name . clone () , } . visit_file_mut (& mut generated_code) ; let mut inlined_staged = stageleft_tool :: gen_staged_trybuild (& path ! (source_dir / "src" / "lib.rs") , crate_name . clone () , is_test ,) ; ReplaceCrateWithOrig { crate_name : crate_name . clone () , } . visit_file_mut (& mut inlined_staged) ; let source = prettyplease :: unparse (& syn :: parse_quote ! { # generated_code # [allow (unused , ambiguous_glob_reexports , clippy :: suspicious_else_formatting , unexpected_cfgs , reason = "generated code")] pub mod __staged { # inlined_staged } }) ; let mut hasher = Sha256 :: new () ; hasher . update (& source) ; let hash = format ! ("{:X}" , hasher . finalize ()) . chars () . take (8) . collect :: < String > () ; let bin_name = if let Some (name_hint) = & name_hint { format ! ("{}_{}" , clean_name_hint (name_hint) , & hash) } else { hash } ; let trybuild_created = create_trybuild (& source , & bin_name , is_test) . unwrap () ; (bin_name , trybuild_created) } pub fn create_trybuild_service (trybuild : TrybuildHost , dir : & std :: path :: PathBuf , target_dir : & std :: path :: PathBuf , features : & Option < Vec < String > > , bin_name : & str ,) -> HydroflowCrate { let mut ret = HydroflowCrate :: new (dir , trybuild . host) . target_dir (target_dir) . bin (bin_name) . no_default_features () ; if let Some (display_name) = trybuild . display_name { ret = ret . display_name (display_name) ; } else if let Some (name_hint) = trybuild . name_hint { if let Some (cluster_idx) = trybuild . cluster_idx { ret = ret . display_name (format ! ("{} / {}" , name_hint , cluster_idx)) ; } else { ret = ret . display_name (name_hint) ; } } if let Some (rustflags) = trybuild . rustflags { ret = ret . rustflags (rustflags) ; } if let Some (tracing) = trybuild . tracing { ret = ret . tracing (tracing) ; } if let Some (features) = features { ret = ret . features (features . clone ()) ; } ret } } # [cfg (feature = "deploy")] pub use deploy_graph :: * ; pub mod in_memory_graph { pub use hydroflow_lang :: graph :: HydroflowGraph ; pub use super :: { LocalDeploy , Node , ProcessSpec } ; pub use crate :: deploy :: in_memory_graph :: SingleProcessGraph ; # [cfg (stageleft_macro)] impl LocalDeploy < '_ > for SingleProcessGraph { type Process = SingleNode ; type Cluster = SingleNode ; type ExternalProcess = SingleNode ; type Meta = () ; type GraphId = () ; fn has_trivial_node () -> bool { true } fn trivial_process (_id : usize) -> Self :: Process { SingleNode { } } fn trivial_cluster (_id : usize) -> Self :: Cluster { SingleNode { } } } # [cfg (stageleft_macro)] impl ProcessSpec < '_ , SingleProcessGraph > for () { fn build (self , _id : usize , _name_hint : & str) -> SingleNode { SingleNode { } } } pub use crate :: deploy :: in_memory_graph :: SingleNode ; # [cfg (stageleft_macro)] impl Node for SingleNode { type Port = () ; type Meta = () ; type InstantiateEnv = () ; fn next_port (& self) { panic ! () ; } fn update_meta (& mut self , _meta : & Self :: Meta) { } fn instantiate (& self , _env : & mut Self :: InstantiateEnv , _meta : & mut Self :: Meta , _graph : HydroflowGraph , _extra_stmts : Vec < syn :: Stmt > ,) { } } pub use crate :: deploy :: in_memory_graph :: MultiGraph ; # [cfg (stageleft_macro)] impl LocalDeploy < '_ > for MultiGraph { type Process = MultiNode ; type Cluster = MultiNode ; type ExternalProcess = MultiNode ; type Meta = () ; type GraphId = usize ; fn has_trivial_node () -> bool { true } fn trivial_process (_id : usize) -> Self :: Process { MultiNode { } } fn trivial_cluster (_id : usize) -> Self :: Cluster { MultiNode { } } } # [cfg (stageleft_macro)] impl ProcessSpec < '_ , MultiGraph > for () { fn build (self , _id : usize , _name_hint : & str) -> MultiNode { MultiNode { } } } pub use crate :: deploy :: in_memory_graph :: MultiNode ; # [cfg (stageleft_macro)] impl Node for MultiNode { type Port = () ; type Meta = () ; type InstantiateEnv = () ; fn next_port (& self) { panic ! () ; } fn update_meta (& mut self , _meta : & Self :: Meta) { } fn instantiate (& self , _env : & mut Self :: InstantiateEnv , _meta : & mut Self :: Meta , _graph : HydroflowGraph , _extra_stmts : Vec < syn :: Stmt > ,) { } } } pub use in_memory_graph :: * ; pub use crate :: deploy :: LocalDeploy ; pub use crate :: deploy :: Deploy ; # [cfg (stageleft_macro)] impl < 'a , T : Deploy < 'a , Process = N , Cluster = C , ExternalProcess = E , Meta = M , GraphId = R > , N : Node < Meta = M > , C : Node < Meta = M > , E : Node < Meta = M > , M : Default , R , > LocalDeploy < 'a > for T { type Process = N ; type Cluster = C ; type ExternalProcess = E ; type Meta = M ; type GraphId = R ; fn has_trivial_node () -> bool { < T as Deploy < 'a > > :: has_trivial_node () } fn trivial_process (id : usize) -> Self :: Process { < T as Deploy < 'a > > :: trivial_process (id) } fn trivial_cluster (id : usize) -> Self :: Cluster { < T as Deploy < 'a > > :: trivail_cluster (id) } } pub use crate :: deploy :: ProcessSpec ; pub use crate :: deploy :: IntoProcessSpec ; # [cfg (stageleft_macro)] impl < 'a , D : LocalDeploy < 'a > + ? Sized , T : ProcessSpec < 'a , D > > IntoProcessSpec < 'a , D > for T { type ProcessSpec = T ; fn into_process_spec (self) -> Self :: ProcessSpec { self } } pub use crate :: deploy :: ClusterSpec ; pub use crate :: deploy :: ExternalSpec ; pub use crate :: deploy :: Node ; pub use crate :: deploy :: RegisterPort ; } pub use deploy :: { ClusterSpec , Deploy , ProcessSpec } ; pub mod cycle { pub use std :: marker :: PhantomData ; pub use crate :: __staged :: location :: Location ; pub use crate :: cycle :: ForwardRefMarker ; pub use crate :: cycle :: TickCycleMarker ; pub use crate :: cycle :: DeferTick ; pub use crate :: cycle :: CycleComplete ; pub use crate :: cycle :: CycleCollection ; pub use crate :: cycle :: CycleCollectionWithInitial ; pub use crate :: cycle :: ForwardRef ; # [cfg (stageleft_macro)] impl < 'a , S : CycleComplete < 'a , ForwardRefMarker > > ForwardRef < 'a , S > { pub fn complete (self , stream : S) { let ident = self . ident ; S :: complete (stream , ident) } } pub use crate :: cycle :: TickCycle ; # [cfg (stageleft_macro)] impl < 'a , S : CycleComplete < 'a , TickCycleMarker > + DeferTick > TickCycle < 'a , S > { pub fn complete_next_tick (self , stream : S) { let ident = self . ident ; S :: complete (stream . defer_tick () , ident) } } } pub mod builder { pub use std :: cell :: RefCell ; pub use std :: collections :: HashMap ; pub use std :: marker :: PhantomData ; pub use std :: rc :: Rc ; pub use compiled :: CompiledFlow ; pub use deploy :: { DeployFlow , DeployResult } ; pub use stageleft :: * ; pub use crate :: __staged :: deploy :: { ExternalSpec , IntoProcessSpec , LocalDeploy } ; pub use crate :: __staged :: ir :: HfPlusLeaf ; pub use crate :: __staged :: location :: { Cluster , ExternalProcess , Process } ; pub use crate :: __staged :: { ClusterSpec , Deploy , RuntimeContext } ; pub mod built { pub use std :: collections :: { BTreeMap , HashMap } ; pub use std :: marker :: PhantomData ; pub use hydroflow_lang :: graph :: { eliminate_extra_unions_tees , HydroflowGraph } ; pub use super :: compiled :: CompiledFlow ; pub use super :: deploy :: { DeployFlow , DeployResult } ; pub use crate :: __staged :: deploy :: { ClusterSpec , Deploy , ExternalSpec , IntoProcessSpec , LocalDeploy } ; pub use crate :: __staged :: ir :: HfPlusLeaf ; pub use crate :: __staged :: location :: { Cluster , ExternalProcess , Process } ; pub use crate :: builder :: built :: BuiltFlow ; # [cfg (stageleft_macro)] impl Drop for BuiltFlow < '_ > { fn drop (& mut self) { if ! self . used { panic ! ("Dropped BuiltFlow without instantiating, you may have forgotten to call `compile` or `deploy`.") ; } } } pub fn build_inner (ir : & mut Vec < HfPlusLeaf >) -> BTreeMap < usize , HydroflowGraph > { let mut builders = BTreeMap :: new () ; let mut built_tees = HashMap :: new () ; let mut next_stmt_id = 0 ; for leaf in ir { leaf . emit (& mut builders , & mut built_tees , & mut next_stmt_id) ; } builders . into_iter () . map (| (k , v) | { let (mut flat_graph , _ , _) = v . build () ; eliminate_extra_unions_tees (& mut flat_graph) ; (k , flat_graph) }) . collect () } # [cfg (stageleft_macro)] impl < 'a > BuiltFlow < 'a > { pub fn ir (& self) -> & Vec < HfPlusLeaf > { & self . ir } pub fn optimize_with (mut self , f : impl FnOnce (Vec < HfPlusLeaf >) -> Vec < HfPlusLeaf >) -> Self { self . used = true ; BuiltFlow { ir : f (std :: mem :: take (& mut self . ir)) , processes : std :: mem :: take (& mut self . processes) , clusters : std :: mem :: take (& mut self . clusters) , used : false , _phantom : PhantomData , } } pub fn with_default_optimize < D : LocalDeploy < 'a > > (self) -> DeployFlow < 'a , D > { self . optimize_with (crate :: rewrites :: persist_pullup :: persist_pullup) . into_deploy () } fn into_deploy < D : LocalDeploy < 'a > > (mut self) -> DeployFlow < 'a , D > { self . used = true ; let processes = if D :: has_trivial_node () { self . processes . iter () . map (| id | (* id , D :: trivial_process (* id))) . collect () } else { HashMap :: new () } ; let clusters = if D :: has_trivial_node () { self . clusters . iter () . map (| id | (* id , D :: trivial_cluster (* id))) . collect () } else { HashMap :: new () } ; DeployFlow { ir : std :: mem :: take (& mut self . ir) , nodes : processes , clusters , externals : HashMap :: new () , used : false , _phantom : PhantomData , } } pub fn with_process < P , D : LocalDeploy < 'a > > (self , process : & Process < P > , spec : impl IntoProcessSpec < 'a , D > ,) -> DeployFlow < 'a , D > { self . into_deploy () . with_process (process , spec) } pub fn with_external < P , D : LocalDeploy < 'a > > (self , process : & ExternalProcess < P > , spec : impl ExternalSpec < 'a , D > ,) -> DeployFlow < 'a , D > { self . into_deploy () . with_external (process , spec) } pub fn with_cluster < C , D : LocalDeploy < 'a > > (self , cluster : & Cluster < C > , spec : impl ClusterSpec < 'a , D > ,) -> DeployFlow < 'a , D > { self . into_deploy () . with_cluster (cluster , spec) } pub fn compile < D : Deploy < 'a > + 'a > (self , env : & D :: CompileEnv) -> CompiledFlow < 'a , D :: GraphId > { self . into_deploy :: < D > () . compile (env) } pub fn compile_no_network < D : LocalDeploy < 'a > + 'a > (self) -> CompiledFlow < 'a , D :: GraphId > { self . into_deploy :: < D > () . compile_no_network () } pub fn deploy < D : Deploy < 'a , CompileEnv = () > + 'a > (self , env : & mut D :: InstantiateEnv ,) -> DeployResult < 'a , D > { self . into_deploy :: < D > () . deploy (env) } } } pub mod compiled { pub use std :: collections :: BTreeMap ; pub use std :: marker :: PhantomData ; pub use hydroflow :: scheduled :: graph :: Hydroflow ; pub use hydroflow_lang :: graph :: { partition_graph , HydroflowGraph } ; pub use proc_macro2 :: TokenStream ; pub use quote :: quote ; pub use stageleft :: runtime_support :: FreeVariable ; pub use stageleft :: Quoted ; pub use crate :: builder :: compiled :: CompiledFlow ; # [cfg (stageleft_macro)] impl < ID > CompiledFlow < '_ , ID > { pub fn hydroflow_ir (& self) -> & BTreeMap < usize , HydroflowGraph > { & self . hydroflow_ir } pub fn take_ir (self) -> BTreeMap < usize , HydroflowGraph > { self . hydroflow_ir } } # [cfg (stageleft_macro)] impl < 'a > CompiledFlow < 'a , usize > { pub fn with_dynamic_id (self , id : impl Quoted < 'a , usize >) -> CompiledFlowWithId < 'a > { let hydroflow_crate = proc_macro_crate :: crate_name ("hydroflow_plus") . expect ("hydroflow_plus should be present in `Cargo.toml`") ; let root = match hydroflow_crate { proc_macro_crate :: FoundCrate :: Itself => quote ! { hydroflow_plus } , proc_macro_crate :: FoundCrate :: Name (name) => { let ident = syn :: Ident :: new (& name , proc_macro2 :: Span :: call_site ()) ; quote ! { # ident } } } ; let mut conditioned_tokens = None ; for (subgraph_id , flat_graph) in self . hydroflow_ir { let partitioned_graph = partition_graph (flat_graph) . expect ("Failed to partition (cycle detected).") ; let mut diagnostics = Vec :: new () ; let tokens = partitioned_graph . as_code (& root , true , quote :: quote ! () , & mut diagnostics) ; let my_extra_stmts = self . extra_stmts . get (& subgraph_id) . cloned () . unwrap_or_default () ; if let Some (conditioned_tokens) = conditioned_tokens . as_mut () { * conditioned_tokens = syn :: parse_quote ! { # conditioned_tokens else if __given_id == # subgraph_id { # (# my_extra_stmts) * # tokens } } ; } else { conditioned_tokens = Some (syn :: parse_quote ! { if __given_id == # subgraph_id { # (# my_extra_stmts) * # tokens } }) ; } } let conditioned_tokens : TokenStream = conditioned_tokens . unwrap () ; let id = id . splice_untyped () ; CompiledFlowWithId { tokens : syn :: parse_quote ! ({ let __given_id = # id ; # conditioned_tokens else { panic ! ("Invalid node id: {}" , __given_id) ; } }) , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < 'a > Quoted < 'a , Hydroflow < 'a > > for CompiledFlow < 'a , () > { } # [cfg (stageleft_macro)] impl < 'a > FreeVariable < Hydroflow < 'a > > for CompiledFlow < 'a , () > { fn to_tokens (mut self) -> (Option < TokenStream > , Option < TokenStream >) { let hydroflow_crate = proc_macro_crate :: crate_name ("hydroflow_plus") . expect ("hydroflow_plus should be present in `Cargo.toml`") ; let root = match hydroflow_crate { proc_macro_crate :: FoundCrate :: Itself => quote ! { hydroflow_plus } , proc_macro_crate :: FoundCrate :: Name (name) => { let ident = syn :: Ident :: new (& name , proc_macro2 :: Span :: call_site ()) ; quote ! { # ident } } } ; if self . hydroflow_ir . len () != 1 { panic ! ("Expected exactly one subgraph in the Hydroflow IR") ; } let flat_graph = self . hydroflow_ir . remove (& 0) . unwrap () ; let partitioned_graph = partition_graph (flat_graph) . expect ("Failed to partition (cycle detected).") ; let mut diagnostics = Vec :: new () ; let tokens = partitioned_graph . as_code (& root , true , quote :: quote ! () , & mut diagnostics) ; (None , Some (tokens)) } } pub use crate :: builder :: compiled :: CompiledFlowWithId ; # [cfg (stageleft_macro)] impl < 'a > Quoted < 'a , Hydroflow < 'a > > for CompiledFlowWithId < 'a > { } # [cfg (stageleft_macro)] impl < 'a > FreeVariable < Hydroflow < 'a > > for CompiledFlowWithId < 'a > { fn to_tokens (self) -> (Option < TokenStream > , Option < TokenStream >) { (None , Some (self . tokens)) } } } pub mod deploy { pub use std :: collections :: { BTreeMap , HashMap } ; pub use std :: io :: Error ; pub use std :: marker :: PhantomData ; pub use std :: pin :: Pin ; pub use hydroflow :: bytes :: Bytes ; pub use hydroflow :: futures :: { Sink , Stream } ; pub use proc_macro2 :: Span ; pub use serde :: de :: DeserializeOwned ; pub use serde :: Serialize ; pub use stageleft :: Quoted ; pub use super :: built :: build_inner ; pub use super :: compiled :: CompiledFlow ; pub use crate :: __staged :: deploy :: { ExternalSpec , IntoProcessSpec , LocalDeploy , Node , RegisterPort } ; pub use crate :: __staged :: ir :: HfPlusLeaf ; pub use crate :: __staged :: location :: external_process :: { ExternalBincodeSink , ExternalBincodeStream , ExternalBytesPort , } ; pub use crate :: __staged :: location :: { ExternalProcess , Location , LocationId } ; pub use crate :: __staged :: { Cluster , ClusterSpec , Deploy , Process , ProcessSpec } ; pub use crate :: builder :: deploy :: DeployFlow ; # [cfg (stageleft_macro)] impl < 'a , D : LocalDeploy < 'a > > Drop for DeployFlow < 'a , D > { fn drop (& mut self) { if ! self . used { panic ! ("Dropped DeployFlow without instantiating, you may have forgotten to call `compile` or `deploy`.") ; } } } # [cfg (stageleft_macro)] impl < 'a , D : LocalDeploy < 'a > > DeployFlow < 'a , D > { pub fn ir (& self) -> & Vec < HfPlusLeaf > { & self . ir } pub fn with_process < P > (mut self , process : & Process < P > , spec : impl IntoProcessSpec < 'a , D > ,) -> Self { let tag_name = std :: any :: type_name :: < P > () . to_string () ; self . nodes . insert (process . id , spec . into_process_spec () . build (process . id , & tag_name) ,) ; self } pub fn with_external < P > (mut self , process : & ExternalProcess < P > , spec : impl ExternalSpec < 'a , D > ,) -> Self { let tag_name = std :: any :: type_name :: < P > () . to_string () ; self . externals . insert (process . id , spec . build (process . id , & tag_name)) ; self } pub fn with_cluster < C > (mut self , cluster : & Cluster < C > , spec : impl ClusterSpec < 'a , D >) -> Self { let tag_name = std :: any :: type_name :: < C > () . to_string () ; self . clusters . insert (cluster . id , spec . build (cluster . id , & tag_name)) ; self } pub fn compile_no_network (mut self) -> CompiledFlow < 'a , D :: GraphId > { self . used = true ; CompiledFlow { hydroflow_ir : build_inner (& mut self . ir) , extra_stmts : BTreeMap :: new () , _phantom : PhantomData , } } } # [cfg (stageleft_macro)] impl < 'a , D : Deploy < 'a > > DeployFlow < 'a , D > { pub fn compile (mut self , env : & D :: CompileEnv) -> CompiledFlow < 'a , D :: GraphId > { self . used = true ; let mut seen_tees : HashMap < _ , _ > = HashMap :: new () ; let mut flow_state_networked : Vec < HfPlusLeaf > = std :: mem :: take (& mut self . ir) . into_iter () . map (| leaf | { leaf . compile_network :: < D > (env , & mut seen_tees , & self . nodes , & self . clusters , & self . externals ,) }) . collect () ; let extra_stmts = self . extra_stmts (env) ; CompiledFlow { hydroflow_ir : build_inner (& mut flow_state_networked) , extra_stmts , _phantom : PhantomData , } } fn extra_stmts (& self , env : & < D as Deploy < 'a > > :: CompileEnv) -> BTreeMap < usize , Vec < syn :: Stmt > > { let all_locations_count = self . nodes . len () + self . clusters . len () ; let mut extra_stmts : BTreeMap < usize , Vec < syn :: Stmt > > = BTreeMap :: new () ; for & c_id in self . clusters . keys () { let self_id_ident = syn :: Ident :: new (& format ! ("__hydroflow_plus_cluster_self_id_{}" , c_id) , Span :: call_site () ,) ; let self_id_expr = D :: cluster_self_id (env) . splice_untyped () ; extra_stmts . entry (c_id) . or_default () . push (syn :: parse_quote ! { let # self_id_ident = # self_id_expr ; }) ; for other_location in 0 .. all_locations_count { let other_id_ident = syn :: Ident :: new (& format ! ("__hydroflow_plus_cluster_ids_{}" , c_id) , Span :: call_site () ,) ; let other_id_expr = D :: cluster_ids (env , c_id) . splice_untyped () ; extra_stmts . entry (other_location) . or_default () . push (syn :: parse_quote ! { let # other_id_ident = # other_id_expr ; }) ; } } extra_stmts } } # [cfg (stageleft_macro)] impl < 'a , D : Deploy < 'a , CompileEnv = () > > DeployFlow < 'a , D > { # [must_use] pub fn deploy (mut self , env : & mut D :: InstantiateEnv) -> DeployResult < 'a , D > { self . used = true ; let mut seen_tees_instantiate : HashMap < _ , _ > = HashMap :: new () ; let mut flow_state_networked : Vec < HfPlusLeaf > = std :: mem :: take (& mut self . ir) . into_iter () . map (| leaf | { leaf . compile_network :: < D > (& () , & mut seen_tees_instantiate , & self . nodes , & self . clusters , & self . externals ,) }) . collect () ; let mut compiled = build_inner (& mut flow_state_networked) ; let mut extra_stmts = self . extra_stmts (& ()) ; let mut meta = D :: Meta :: default () ; let (mut processes , mut clusters , mut externals) = (std :: mem :: take (& mut self . nodes) . into_iter () . map (| (node_id , node) | { node . instantiate (env , & mut meta , compiled . remove (& node_id) . unwrap () , extra_stmts . remove (& node_id) . unwrap_or_default () ,) ; (node_id , node) }) . collect :: < HashMap < _ , _ > > () , std :: mem :: take (& mut self . clusters) . into_iter () . map (| (cluster_id , cluster) | { cluster . instantiate (env , & mut meta , compiled . remove (& cluster_id) . unwrap () , extra_stmts . remove (& cluster_id) . unwrap_or_default () ,) ; (cluster_id , cluster) }) . collect :: < HashMap < _ , _ > > () , std :: mem :: take (& mut self . externals) . into_iter () . map (| (external_id , external) | { external . instantiate (env , & mut meta , compiled . remove (& external_id) . unwrap () , extra_stmts . remove (& external_id) . unwrap_or_default () ,) ; (external_id , external) }) . collect :: < HashMap < _ , _ > > () ,) ; for node in processes . values_mut () { node . update_meta (& meta) ; } for cluster in clusters . values_mut () { cluster . update_meta (& meta) ; } for external in externals . values_mut () { external . update_meta (& meta) ; } let mut seen_tees_connect = HashMap :: new () ; for leaf in flow_state_networked { leaf . connect_network (& mut seen_tees_connect) ; } DeployResult { processes , clusters , externals , } } } pub use crate :: builder :: deploy :: DeployResult ; # [cfg (stageleft_macro)] impl < 'a , D : Deploy < 'a > > DeployResult < 'a , D > { pub fn get_process < P > (& self , p : & Process < P >) -> & D :: Process { let id = match p . id () { LocationId :: Process (id) => id , _ => panic ! ("Process ID expected") , } ; self . processes . get (& id) . unwrap () } pub fn get_cluster < C > (& self , c : & Cluster < 'a , C >) -> & D :: Cluster { let id = match c . id () { LocationId :: Cluster (id) => id , _ => panic ! ("Cluster ID expected") , } ; self . clusters . get (& id) . unwrap () } pub fn get_external < P > (& self , p : & ExternalProcess < P >) -> & D :: ExternalProcess { self . externals . get (& p . id) . unwrap () } pub fn raw_port (& self , port : ExternalBytesPort) -> D :: ExternalRawPort { self . externals . get (& port . process_id) . unwrap () . raw_port (port . port_id) } pub async fn connect_sink_bytes (& self , port : ExternalBytesPort ,) -> Pin < Box < dyn Sink < Bytes , Error = Error > > > { self . externals . get (& port . process_id) . unwrap () . as_bytes_sink (port . port_id) . await } pub async fn connect_sink_bincode < T : Serialize + DeserializeOwned + 'static > (& self , port : ExternalBincodeSink < T > ,) -> Pin < Box < dyn Sink < T , Error = Error > > > { self . externals . get (& port . process_id) . unwrap () . as_bincode_sink (port . port_id) . await } pub async fn connect_source_bytes (& self , port : ExternalBytesPort ,) -> Pin < Box < dyn Stream < Item = Bytes > > > { self . externals . get (& port . process_id) . unwrap () . as_bytes_source (port . port_id) . await } pub async fn connect_source_bincode < T : Serialize + DeserializeOwned + 'static > (& self , port : ExternalBincodeStream < T > ,) -> Pin < Box < dyn Stream < Item = T > > > { self . externals . get (& port . process_id) . unwrap () . as_bincode_source (port . port_id) . await } } } pub use crate :: builder :: FlowStateInner ; pub type FlowState = Rc < RefCell < FlowStateInner > > ; pub const FLOW_USED_MESSAGE : & str = "Attempted to add a leaf to a flow that has already been finalized. No leaves can be added after the flow has been compiled." ; pub use crate :: builder :: FlowBuilder ; # [cfg (stageleft_macro)] impl Drop for FlowBuilder < '_ > { fn drop (& mut self) { if ! self . finalized { panic ! ("Dropped FlowBuilder without finalizing, you may have forgotten to call `with_default_optimize`, `optimize_with`, or `finalize`.") ; } } } # [cfg (stageleft_macro)] impl QuotedContext for FlowBuilder < '_ > { fn create () -> Self { FlowBuilder :: new () } } # [cfg (stageleft_macro)] impl < 'a > FlowBuilder < 'a > { # [expect (clippy :: new_without_default , reason = "call `new` explicitly, not `default`")] pub fn new () -> FlowBuilder < 'a > { FlowBuilder { flow_state : Rc :: new (RefCell :: new (FlowStateInner { leaves : Some (vec ! []) , next_external_out : 0 , cycle_counts : HashMap :: new () , next_clock_id : 0 , })) , nodes : RefCell :: new (vec ! []) , clusters : RefCell :: new (vec ! []) , next_node_id : RefCell :: new (0) , finalized : false , _phantom : PhantomData , } } pub fn finalize (mut self) -> built :: BuiltFlow < 'a > { self . finalized = true ; built :: BuiltFlow { ir : self . flow_state . borrow_mut () . leaves . take () . unwrap () , processes : self . nodes . replace (vec ! []) , clusters : self . clusters . replace (vec ! []) , used : false , _phantom : PhantomData , } } pub fn with_default_optimize < D : LocalDeploy < 'a > > (self) -> DeployFlow < 'a , D > { self . finalize () . with_default_optimize () } pub fn optimize_with (self , f : impl FnOnce (Vec < HfPlusLeaf >) -> Vec < HfPlusLeaf > ,) -> built :: BuiltFlow < 'a > { self . finalize () . optimize_with (f) } pub fn flow_state (& self) -> & FlowState { & self . flow_state } pub fn process < P > (& self) -> Process < 'a , P > { let mut next_node_id = self . next_node_id . borrow_mut () ; let id = * next_node_id ; * next_node_id += 1 ; self . nodes . borrow_mut () . push (id) ; Process { id , flow_state : self . flow_state () . clone () , _phantom : PhantomData , } } pub fn external_process < P > (& self) -> ExternalProcess < 'a , P > { let mut next_node_id = self . next_node_id . borrow_mut () ; let id = * next_node_id ; * next_node_id += 1 ; self . nodes . borrow_mut () . push (id) ; ExternalProcess { id , flow_state : self . flow_state () . clone () , _phantom : PhantomData , } } pub fn cluster < C > (& self) -> Cluster < 'a , C > { let mut next_node_id = self . next_node_id . borrow_mut () ; let id = * next_node_id ; * next_node_id += 1 ; self . clusters . borrow_mut () . push (id) ; Cluster { id , flow_state : self . flow_state () . clone () , _phantom : PhantomData , } } pub fn runtime_context (& self) -> RuntimeContext < 'a > { RuntimeContext :: new () } pub fn with_process < P , D : LocalDeploy < 'a > > (self , process : & Process < P > , spec : impl IntoProcessSpec < 'a , D > ,) -> DeployFlow < 'a , D > { self . with_default_optimize () . with_process (process , spec) } pub fn with_external < P , D : LocalDeploy < 'a > > (self , process : & ExternalProcess < P > , spec : impl ExternalSpec < 'a , D > ,) -> DeployFlow < 'a , D > { self . with_default_optimize () . with_external (process , spec) } pub fn with_cluster < C , D : LocalDeploy < 'a > > (self , cluster : & Cluster < C > , spec : impl ClusterSpec < 'a , D > ,) -> DeployFlow < 'a , D > { self . with_default_optimize () . with_cluster (cluster , spec) } pub fn compile < D : Deploy < 'a > + 'a > (self , env : & D :: CompileEnv) -> CompiledFlow < 'a , D :: GraphId > { self . with_default_optimize :: < D > () . compile (env) } pub fn compile_no_network < D : LocalDeploy < 'a > + 'a > (self) -> CompiledFlow < 'a , D :: GraphId > { self . with_default_optimize :: < D > () . compile_no_network () } pub fn deploy < D : Deploy < 'a , CompileEnv = () > + 'a > (self , env : & mut D :: InstantiateEnv ,) -> DeployResult < 'a , D > { self . with_default_optimize () . deploy (env) } } } pub use builder :: FlowBuilder ; pub mod ir { pub use core :: panic ; pub use std :: cell :: RefCell ; pub use std :: collections :: { BTreeMap , HashMap } ; pub use std :: fmt :: Debug ; pub use std :: ops :: Deref ; pub use std :: rc :: Rc ; pub use hydroflow_lang :: graph :: FlatGraphBuilder ; pub use hydroflow_lang :: parse :: Pipeline ; pub use proc_macro2 :: { Span , TokenStream } ; pub use quote :: ToTokens ; pub use syn :: parse_quote ; pub use crate :: __staged :: deploy :: { Deploy , RegisterPort } ; pub use crate :: __staged :: location :: LocationId ; pub use crate :: ir :: DebugExpr ; # [cfg (stageleft_macro)] impl From < syn :: Expr > for DebugExpr { fn from (expr : syn :: Expr) -> DebugExpr { DebugExpr (expr) } } # [cfg (stageleft_macro)] impl Deref for DebugExpr { type Target = syn :: Expr ; fn deref (& self) -> & Self :: Target { & self . 0 } } # [cfg (stageleft_macro)] impl ToTokens for DebugExpr { fn to_tokens (& self , tokens : & mut TokenStream) { self . 0 . to_tokens (tokens) ; } } # [cfg (stageleft_macro)] impl Debug for DebugExpr { fn fmt (& self , f : & mut std :: fmt :: Formatter < '_ >) -> std :: fmt :: Result { write ! (f , "{}" , self . 0 . to_token_stream ()) } } pub use crate :: ir :: DebugInstantiate ; # [cfg (stageleft_macro)] impl Debug for DebugInstantiate { fn fmt (& self , f : & mut std :: fmt :: Formatter < '_ >) -> std :: fmt :: Result { write ! (f , "") } } pub use crate :: ir :: DebugPipelineFn ; # [cfg (stageleft_macro)] impl Debug for DebugPipelineFn { fn fmt (& self , f : & mut std :: fmt :: Formatter < '_ >) -> std :: fmt :: Result { write ! (f , "") } } pub use crate :: ir :: HfPlusSource ; pub use crate :: ir :: HfPlusLeaf ; # [cfg (stageleft_macro)] impl HfPlusLeaf { pub fn compile_network < 'a , D : Deploy < 'a > + 'a > (self , compile_env : & D :: CompileEnv , seen_tees : & mut SeenTees , nodes : & HashMap < usize , D :: Process > , clusters : & HashMap < usize , D :: Cluster > , externals : & HashMap < usize , D :: ExternalProcess > ,) -> HfPlusLeaf { self . transform_children (| n , s | { n . compile_network :: < D > (compile_env , s , nodes , clusters , externals) ; } , seen_tees ,) } pub fn connect_network (self , seen_tees : & mut SeenTees) -> HfPlusLeaf { self . transform_children (| n , s | { n . connect_network (s) ; } , seen_tees ,) } pub fn transform_children (self , mut transform : impl FnMut (& mut HfPlusNode , & mut SeenTees) , seen_tees : & mut SeenTees ,) -> HfPlusLeaf { match self { HfPlusLeaf :: ForEach { f , mut input } => { transform (& mut input , seen_tees) ; HfPlusLeaf :: ForEach { f , input } } HfPlusLeaf :: DestSink { sink , mut input } => { transform (& mut input , seen_tees) ; HfPlusLeaf :: DestSink { sink , input } } HfPlusLeaf :: CycleSink { ident , location_kind , mut input , } => { transform (& mut input , seen_tees) ; HfPlusLeaf :: CycleSink { ident , location_kind , input , } } } } pub fn emit (& self , graph_builders : & mut BTreeMap < usize , FlatGraphBuilder > , built_tees : & mut HashMap < * const RefCell < HfPlusNode > , (syn :: Ident , usize) > , next_stmt_id : & mut usize ,) { match self { HfPlusLeaf :: ForEach { f , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; graph_builders . entry (input_location_id) . or_default () . add_statement (parse_quote ! { # input_ident -> for_each (# f) ; }) ; } HfPlusLeaf :: DestSink { sink , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; graph_builders . entry (input_location_id) . or_default () . add_statement (parse_quote ! { # input_ident -> dest_sink (# sink) ; }) ; } HfPlusLeaf :: CycleSink { ident , location_kind , input , } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let location_id = match location_kind . root () { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (_) => panic ! () , } ; assert_eq ! (input_location_id , * location_id , "cycle_sink location mismatch") ; graph_builders . entry (* location_id) . or_default () . add_statement (parse_quote ! { # ident = # input_ident ; }) ; } } } } type PrintedTees = RefCell < Option < (usize , HashMap < * const RefCell < HfPlusNode > , usize >) > > ; thread_local ! { static PRINTED_TEES : PrintedTees = const { RefCell :: new (None) } ; } pub fn dbg_dedup_tee < T > (f : impl FnOnce () -> T) -> T { PRINTED_TEES . with (| printed_tees | { let mut printed_tees_mut = printed_tees . borrow_mut () ; * printed_tees_mut = Some ((0 , HashMap :: new ())) ; drop (printed_tees_mut) ; let ret = f () ; let mut printed_tees_mut = printed_tees . borrow_mut () ; * printed_tees_mut = None ; ret }) } pub use crate :: ir :: TeeNode ; # [cfg (stageleft_macro)] impl Debug for TeeNode { fn fmt (& self , f : & mut std :: fmt :: Formatter < '_ >) -> std :: fmt :: Result { PRINTED_TEES . with (| printed_tees | { let mut printed_tees_mut_borrow = printed_tees . borrow_mut () ; let printed_tees_mut = printed_tees_mut_borrow . as_mut () ; if let Some (printed_tees_mut) = printed_tees_mut { if let Some (existing) = printed_tees_mut . 1 . get (& (self . 0 . as_ref () as * const RefCell < HfPlusNode >)) { write ! (f , "" , existing) } else { let next_id = printed_tees_mut . 0 ; printed_tees_mut . 0 += 1 ; printed_tees_mut . 1 . insert (self . 0 . as_ref () as * const RefCell < HfPlusNode > , next_id) ; drop (printed_tees_mut_borrow) ; write ! (f , ": " , next_id) ? ; Debug :: fmt (& self . 0 . borrow () , f) } } else { drop (printed_tees_mut_borrow) ; write ! (f , ": ") ? ; Debug :: fmt (& self . 0 . borrow () , f) } }) } } pub use crate :: ir :: HfPlusNode ; pub type SeenTees = HashMap < * const RefCell < HfPlusNode > , Rc < RefCell < HfPlusNode > > > ; # [cfg (stageleft_macro)] impl < 'a > HfPlusNode { pub fn compile_network < D : Deploy < 'a > + 'a > (& mut self , compile_env : & D :: CompileEnv , seen_tees : & mut SeenTees , nodes : & HashMap < usize , D :: Process > , clusters : & HashMap < usize , D :: Cluster > , externals : & HashMap < usize , D :: ExternalProcess > ,) { self . transform_children (| n , s | n . compile_network :: < D > (compile_env , s , nodes , clusters , externals) , seen_tees ,) ; if let HfPlusNode :: Network { from_location , from_key , to_location , to_key , instantiate_fn , .. } = self { let (sink_expr , source_expr , connect_fn) = match instantiate_fn { DebugInstantiate :: Building () => instantiate_network :: < D > (from_location , * from_key , to_location , * to_key , nodes , clusters , externals , compile_env ,) , DebugInstantiate :: Finalized (_ , _ , _) => panic ! ("network already finalized") , } ; * instantiate_fn = DebugInstantiate :: Finalized (sink_expr , source_expr , Some (connect_fn)) ; } } pub fn connect_network (& mut self , seen_tees : & mut SeenTees) { self . transform_children (| n , s | n . connect_network (s) , seen_tees) ; if let HfPlusNode :: Network { instantiate_fn , .. } = self { match instantiate_fn { DebugInstantiate :: Building () => panic ! ("network not built") , DebugInstantiate :: Finalized (_ , _ , connect_fn) => { connect_fn . take () . unwrap () () ; } } } } pub fn transform_bottom_up < C > (& mut self , mut transform : impl FnMut (& mut HfPlusNode , & mut C) + Copy , seen_tees : & mut SeenTees , ctx : & mut C ,) { self . transform_children (| n , s | n . transform_bottom_up (transform , s , ctx) , seen_tees) ; transform (self , ctx) } # [inline (always)] pub fn transform_children (& mut self , mut transform : impl FnMut (& mut HfPlusNode , & mut SeenTees) , seen_tees : & mut SeenTees ,) { match self { HfPlusNode :: Placeholder => { panic ! () ; } HfPlusNode :: Source { .. } => { } HfPlusNode :: CycleSource { .. } => { } HfPlusNode :: Tee { inner } => { if let Some (transformed) = seen_tees . get (& (inner . 0 . as_ref () as * const RefCell < HfPlusNode >)) { * inner = TeeNode (transformed . clone ()) ; } else { let transformed_cell = Rc :: new (RefCell :: new (HfPlusNode :: Placeholder)) ; seen_tees . insert (inner . 0 . as_ref () as * const RefCell < HfPlusNode > , transformed_cell . clone () ,) ; let mut orig = inner . 0 . replace (HfPlusNode :: Placeholder) ; transform (& mut orig , seen_tees) ; * transformed_cell . borrow_mut () = orig ; * inner = TeeNode (transformed_cell) ; } } HfPlusNode :: Persist (inner) => transform (inner . as_mut () , seen_tees) , HfPlusNode :: Unpersist (inner) => transform (inner . as_mut () , seen_tees) , HfPlusNode :: Delta (inner) => transform (inner . as_mut () , seen_tees) , HfPlusNode :: Union (left , right) => { transform (left . as_mut () , seen_tees) ; transform (right . as_mut () , seen_tees) ; } HfPlusNode :: CrossProduct (left , right) => { transform (left . as_mut () , seen_tees) ; transform (right . as_mut () , seen_tees) ; } HfPlusNode :: CrossSingleton (left , right) => { transform (left . as_mut () , seen_tees) ; transform (right . as_mut () , seen_tees) ; } HfPlusNode :: Join (left , right) => { transform (left . as_mut () , seen_tees) ; transform (right . as_mut () , seen_tees) ; } HfPlusNode :: Difference (left , right) => { transform (left . as_mut () , seen_tees) ; transform (right . as_mut () , seen_tees) ; } HfPlusNode :: AntiJoin (left , right) => { transform (left . as_mut () , seen_tees) ; transform (right . as_mut () , seen_tees) ; } HfPlusNode :: Map { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: FlatMap { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Filter { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: FilterMap { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Sort (input) => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: DeferTick (input) => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Enumerate (input) => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Inspect { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Unique (input) => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Fold { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: FoldKeyed { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Reduce { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: ReduceKeyed { input , .. } => { transform (input . as_mut () , seen_tees) ; } HfPlusNode :: Network { input , .. } => { transform (input . as_mut () , seen_tees) ; } } } pub fn emit (& self , graph_builders : & mut BTreeMap < usize , FlatGraphBuilder > , built_tees : & mut HashMap < * const RefCell < HfPlusNode > , (syn :: Ident , usize) > , next_stmt_id : & mut usize ,) -> (syn :: Ident , usize) { match self { HfPlusNode :: Placeholder => { panic ! () } HfPlusNode :: Persist (inner) => { let (inner_ident , location) = inner . emit (graph_builders , built_tees , next_stmt_id) ; let persist_id = * next_stmt_id ; * next_stmt_id += 1 ; let persist_ident = syn :: Ident :: new (& format ! ("stream_{}" , persist_id) , Span :: call_site ()) ; let builder = graph_builders . entry (location) . or_default () ; builder . add_statement (parse_quote ! { # persist_ident = # inner_ident -> persist ::<'static > () ; }) ; (persist_ident , location) } HfPlusNode :: Unpersist (_) => { panic ! ("Unpersist is a marker node and should have been optimized away. This is likely a compiler bug.") } HfPlusNode :: Delta (inner) => { let (inner_ident , location) = inner . emit (graph_builders , built_tees , next_stmt_id) ; let delta_id = * next_stmt_id ; * next_stmt_id += 1 ; let delta_ident = syn :: Ident :: new (& format ! ("stream_{}" , delta_id) , Span :: call_site ()) ; let builder = graph_builders . entry (location) . or_default () ; builder . add_statement (parse_quote ! { # delta_ident = # inner_ident -> multiset_delta () ; }) ; (delta_ident , location) } HfPlusNode :: Source { source , location_kind , } => { let location_id = match location_kind { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (id) => id , } ; if let HfPlusSource :: ExternalNetwork () = source { (syn :: Ident :: new ("DUMMY" , Span :: call_site ()) , * location_id) } else { let source_id = * next_stmt_id ; * next_stmt_id += 1 ; let source_ident = syn :: Ident :: new (& format ! ("stream_{}" , source_id) , Span :: call_site ()) ; let source_stmt = match source { HfPlusSource :: Stream (expr) => { parse_quote ! { # source_ident = source_stream (# expr) ; } } HfPlusSource :: ExternalNetwork () => { unreachable ! () } HfPlusSource :: Iter (expr) => { parse_quote ! { # source_ident = source_iter (# expr) ; } } HfPlusSource :: Spin () => { parse_quote ! { # source_ident = spin () ; } } } ; graph_builders . entry (* location_id) . or_default () . add_statement (source_stmt) ; (source_ident , * location_id) } } HfPlusNode :: CycleSource { ident , location_kind , } => { let location_id = match location_kind . root () { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (_) => panic ! () , } ; (ident . clone () , * location_id) } HfPlusNode :: Tee { inner } => { if let Some (ret) = built_tees . get (& (inner . 0 . as_ref () as * const RefCell < HfPlusNode >)) { ret . clone () } else { let (inner_ident , inner_location_id) = inner . 0 . borrow () . emit (graph_builders , built_tees , next_stmt_id) ; let tee_id = * next_stmt_id ; * next_stmt_id += 1 ; let tee_ident = syn :: Ident :: new (& format ! ("stream_{}" , tee_id) , Span :: call_site ()) ; let builder = graph_builders . entry (inner_location_id) . or_default () ; builder . add_statement (parse_quote ! { # tee_ident = # inner_ident -> tee () ; }) ; built_tees . insert (inner . 0 . as_ref () as * const RefCell < HfPlusNode > , (tee_ident . clone () , inner_location_id) ,) ; (tee_ident , inner_location_id) } } HfPlusNode :: Union (left , right) => { let (left_ident , left_location_id) = left . emit (graph_builders , built_tees , next_stmt_id) ; let (right_ident , right_location_id) = right . emit (graph_builders , built_tees , next_stmt_id) ; assert_eq ! (left_location_id , right_location_id , "union inputs must be in the same location") ; let union_id = * next_stmt_id ; * next_stmt_id += 1 ; let union_ident = syn :: Ident :: new (& format ! ("stream_{}" , union_id) , Span :: call_site ()) ; let builder = graph_builders . entry (left_location_id) . or_default () ; builder . add_statement (parse_quote ! { # union_ident = union () ; }) ; builder . add_statement (parse_quote ! { # left_ident -> [0] # union_ident ; }) ; builder . add_statement (parse_quote ! { # right_ident -> [1] # union_ident ; }) ; (union_ident , left_location_id) } HfPlusNode :: CrossSingleton (left , right) => { let (left_ident , left_location_id) = left . emit (graph_builders , built_tees , next_stmt_id) ; let (right_ident , right_location_id) = right . emit (graph_builders , built_tees , next_stmt_id) ; assert_eq ! (left_location_id , right_location_id , "cross_singleton inputs must be in the same location") ; let union_id = * next_stmt_id ; * next_stmt_id += 1 ; let cross_ident = syn :: Ident :: new (& format ! ("stream_{}" , union_id) , Span :: call_site ()) ; let builder = graph_builders . entry (left_location_id) . or_default () ; builder . add_statement (parse_quote ! { # cross_ident = cross_singleton () ; }) ; builder . add_statement (parse_quote ! { # left_ident -> [input] # cross_ident ; }) ; builder . add_statement (parse_quote ! { # right_ident -> [single] # cross_ident ; }) ; (cross_ident , left_location_id) } HfPlusNode :: CrossProduct (..) | HfPlusNode :: Join (..) => { let operator : syn :: Ident = if matches ! (self , HfPlusNode :: CrossProduct (..)) { parse_quote ! (cross_join_multiset) } else { parse_quote ! (join_multiset) } ; let (HfPlusNode :: CrossProduct (left , right) | HfPlusNode :: Join (left , right)) = self else { unreachable ! () } ; let (left_inner , left_was_persist) = if let HfPlusNode :: Persist (left) = left . as_ref () { (left , true) } else { (left , false) } ; let (right_inner , right_was_persist) = if let HfPlusNode :: Persist (right) = right . as_ref () { (right , true) } else { (right , false) } ; let (left_ident , left_location_id) = left_inner . emit (graph_builders , built_tees , next_stmt_id) ; let (right_ident , right_location_id) = right_inner . emit (graph_builders , built_tees , next_stmt_id) ; assert_eq ! (left_location_id , right_location_id , "join / cross product inputs must be in the same location") ; let stream_id = * next_stmt_id ; * next_stmt_id += 1 ; let stream_ident = syn :: Ident :: new (& format ! ("stream_{}" , stream_id) , Span :: call_site ()) ; let builder = graph_builders . entry (left_location_id) . or_default () ; match (left_was_persist , right_was_persist) { (true , true) => { builder . add_statement (parse_quote ! { # stream_ident = # operator ::<'static , 'static > () ; }) ; } (true , false) => { builder . add_statement (parse_quote ! { # stream_ident = # operator ::<'static , 'tick > () ; }) ; } (false , true) => { builder . add_statement (parse_quote ! { # stream_ident = # operator ::<'tick , 'static > () ; }) ; } (false , false) => { builder . add_statement (parse_quote ! { # stream_ident = # operator ::<'tick , 'tick > () ; }) ; } } ; builder . add_statement (parse_quote ! { # left_ident -> [0] # stream_ident ; }) ; builder . add_statement (parse_quote ! { # right_ident -> [1] # stream_ident ; }) ; (stream_ident , left_location_id) } HfPlusNode :: Difference (..) | HfPlusNode :: AntiJoin (..) => { let operator : syn :: Ident = if matches ! (self , HfPlusNode :: Difference (..)) { parse_quote ! (difference_multiset) } else { parse_quote ! (anti_join_multiset) } ; let (HfPlusNode :: Difference (left , right) | HfPlusNode :: AntiJoin (left , right)) = self else { unreachable ! () } ; let (right , right_was_persist) = if let HfPlusNode :: Persist (right) = right . as_ref () { (right , true) } else { (right , false) } ; let (left_ident , left_location_id) = left . emit (graph_builders , built_tees , next_stmt_id) ; let (right_ident , right_location_id) = right . emit (graph_builders , built_tees , next_stmt_id) ; assert_eq ! (left_location_id , right_location_id , "difference / anti join inputs must be in the same location") ; let stream_id = * next_stmt_id ; * next_stmt_id += 1 ; let stream_ident = syn :: Ident :: new (& format ! ("stream_{}" , stream_id) , Span :: call_site ()) ; let builder = graph_builders . entry (left_location_id) . or_default () ; if right_was_persist { builder . add_statement (parse_quote ! { # stream_ident = # operator ::<'tick , 'static > () ; }) ; } else { builder . add_statement (parse_quote ! { # stream_ident = # operator ::<'tick , 'tick > () ; }) ; } builder . add_statement (parse_quote ! { # left_ident -> [pos] # stream_ident ; }) ; builder . add_statement (parse_quote ! { # right_ident -> [neg] # stream_ident ; }) ; (stream_ident , left_location_id) } HfPlusNode :: Map { f , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let map_id = * next_stmt_id ; * next_stmt_id += 1 ; let map_ident = syn :: Ident :: new (& format ! ("stream_{}" , map_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # map_ident = # input_ident -> map (# f) ; }) ; (map_ident , input_location_id) } HfPlusNode :: FlatMap { f , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let flat_map_id = * next_stmt_id ; * next_stmt_id += 1 ; let flat_map_ident = syn :: Ident :: new (& format ! ("stream_{}" , flat_map_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # flat_map_ident = # input_ident -> flat_map (# f) ; }) ; (flat_map_ident , input_location_id) } HfPlusNode :: Filter { f , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let filter_id = * next_stmt_id ; * next_stmt_id += 1 ; let filter_ident = syn :: Ident :: new (& format ! ("stream_{}" , filter_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # filter_ident = # input_ident -> filter (# f) ; }) ; (filter_ident , input_location_id) } HfPlusNode :: FilterMap { f , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let filter_map_id = * next_stmt_id ; * next_stmt_id += 1 ; let filter_map_ident = syn :: Ident :: new (& format ! ("stream_{}" , filter_map_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # filter_map_ident = # input_ident -> filter_map (# f) ; }) ; (filter_map_ident , input_location_id) } HfPlusNode :: Sort (input) => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let sort_id = * next_stmt_id ; * next_stmt_id += 1 ; let sort_ident = syn :: Ident :: new (& format ! ("stream_{}" , sort_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # sort_ident = # input_ident -> sort () ; }) ; (sort_ident , input_location_id) } HfPlusNode :: DeferTick (input) => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let defer_tick_id = * next_stmt_id ; * next_stmt_id += 1 ; let defer_tick_ident = syn :: Ident :: new (& format ! ("stream_{}" , defer_tick_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # defer_tick_ident = # input_ident -> defer_tick_lazy () ; }) ; (defer_tick_ident , input_location_id) } HfPlusNode :: Enumerate (input) => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let enumerate_id = * next_stmt_id ; * next_stmt_id += 1 ; let enumerate_ident = syn :: Ident :: new (& format ! ("stream_{}" , enumerate_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # enumerate_ident = # input_ident -> enumerate () ; }) ; (enumerate_ident , input_location_id) } HfPlusNode :: Inspect { f , input } => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let inspect_id = * next_stmt_id ; * next_stmt_id += 1 ; let inspect_ident = syn :: Ident :: new (& format ! ("stream_{}" , inspect_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # inspect_ident = # input_ident -> inspect (# f) ; }) ; (inspect_ident , input_location_id) } HfPlusNode :: Unique (input) => { let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let unique_id = * next_stmt_id ; * next_stmt_id += 1 ; let unique_ident = syn :: Ident :: new (& format ! ("stream_{}" , unique_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; builder . add_statement (parse_quote ! { # unique_ident = # input_ident -> unique ::<'tick > () ; }) ; (unique_ident , input_location_id) } HfPlusNode :: Fold { .. } | HfPlusNode :: FoldKeyed { .. } => { let operator : syn :: Ident = if matches ! (self , HfPlusNode :: Fold { .. }) { parse_quote ! (fold) } else { parse_quote ! (fold_keyed) } ; let (HfPlusNode :: Fold { init , acc , input } | HfPlusNode :: FoldKeyed { init , acc , input }) = self else { unreachable ! () } ; let (input , input_was_persist) = if let HfPlusNode :: Persist (input) = input . as_ref () { (input , true) } else { (input , false) } ; let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let reduce_id = * next_stmt_id ; * next_stmt_id += 1 ; let fold_ident = syn :: Ident :: new (& format ! ("stream_{}" , reduce_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; if input_was_persist { builder . add_statement (parse_quote ! { # fold_ident = # input_ident -> # operator ::<'static > (# init , # acc) ; }) ; } else { builder . add_statement (parse_quote ! { # fold_ident = # input_ident -> # operator ::<'tick > (# init , # acc) ; }) ; } (fold_ident , input_location_id) } HfPlusNode :: Reduce { .. } | HfPlusNode :: ReduceKeyed { .. } => { let operator : syn :: Ident = if matches ! (self , HfPlusNode :: Reduce { .. }) { parse_quote ! (reduce) } else { parse_quote ! (reduce_keyed) } ; let (HfPlusNode :: Reduce { f , input } | HfPlusNode :: ReduceKeyed { f , input }) = self else { unreachable ! () } ; let (input , input_was_persist) = if let HfPlusNode :: Persist (input) = input . as_ref () { (input , true) } else { (input , false) } ; let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let reduce_id = * next_stmt_id ; * next_stmt_id += 1 ; let reduce_ident = syn :: Ident :: new (& format ! ("stream_{}" , reduce_id) , Span :: call_site ()) ; let builder = graph_builders . entry (input_location_id) . or_default () ; if input_was_persist { builder . add_statement (parse_quote ! { # reduce_ident = # input_ident -> # operator ::<'static > (# f) ; }) ; } else { builder . add_statement (parse_quote ! { # reduce_ident = # input_ident -> # operator ::<'tick > (# f) ; }) ; } (reduce_ident , input_location_id) } HfPlusNode :: Network { from_location : _ , from_key : _ , to_location , to_key : _ , serialize_pipeline , instantiate_fn , deserialize_pipeline , input , } => { let (sink_expr , source_expr , _connect_fn) = match instantiate_fn { DebugInstantiate :: Building () => { panic ! ("Expected the network to be finalized") } DebugInstantiate :: Finalized (sink , source , connect_fn) => { (sink , source , connect_fn) } } ; let (input_ident , input_location_id) = input . emit (graph_builders , built_tees , next_stmt_id) ; let sender_builder = graph_builders . entry (input_location_id) . or_default () ; if let Some (serialize_pipeline) = serialize_pipeline { sender_builder . add_statement (parse_quote ! { # input_ident -> # serialize_pipeline -> dest_sink (# sink_expr) ; }) ; } else { sender_builder . add_statement (parse_quote ! { # input_ident -> dest_sink (# sink_expr) ; }) ; } let to_id = match to_location { LocationId :: Process (id) => id , LocationId :: Cluster (id) => id , LocationId :: Tick (_ , _) => panic ! () , LocationId :: ExternalProcess (id) => id , } ; let receiver_builder = graph_builders . entry (* to_id) . or_default () ; let receiver_stream_id = * next_stmt_id ; * next_stmt_id += 1 ; let receiver_stream_ident = syn :: Ident :: new (& format ! ("stream_{}" , receiver_stream_id) , Span :: call_site ()) ; if let Some (deserialize_pipeline) = deserialize_pipeline { receiver_builder . add_statement (parse_quote ! { # receiver_stream_ident = source_stream (# source_expr) -> # deserialize_pipeline ; }) ; } else { receiver_builder . add_statement (parse_quote ! { # receiver_stream_ident = source_stream (# source_expr) ; }) ; } (receiver_stream_ident , * to_id) } } } } # [expect (clippy :: too_many_arguments , reason = "networking internals")] pub fn instantiate_network < 'a , D : Deploy < 'a > + 'a > (from_location : & mut LocationId , from_key : Option < usize > , to_location : & mut LocationId , to_key : Option < usize > , nodes : & HashMap < usize , D :: Process > , clusters : & HashMap < usize , D :: Cluster > , externals : & HashMap < usize , D :: ExternalProcess > , compile_env : & D :: CompileEnv ,) -> (syn :: Expr , syn :: Expr , Box < dyn FnOnce () >) { let ((sink , source) , connect_fn) = match (from_location , to_location) { (LocationId :: Process (from) , LocationId :: Process (to)) => { let from_node = nodes . get (from) . unwrap_or_else (| | { panic ! ("A process used in the graph was not instantiated: {}" , from) }) . clone () ; let to_node = nodes . get (to) . unwrap_or_else (| | { panic ! ("A process used in the graph was not instantiated: {}" , to) }) . clone () ; let sink_port = D :: allocate_process_port (& from_node) ; let source_port = D :: allocate_process_port (& to_node) ; (D :: o2o_sink_source (compile_env , & from_node , & sink_port , & to_node , & source_port) , D :: o2o_connect (& from_node , & sink_port , & to_node , & source_port) ,) } (LocationId :: Process (from) , LocationId :: Cluster (to)) => { let from_node = nodes . get (from) . unwrap_or_else (| | { panic ! ("A process used in the graph was not instantiated: {}" , from) }) . clone () ; let to_node = clusters . get (to) . unwrap_or_else (| | { panic ! ("A cluster used in the graph was not instantiated: {}" , to) }) . clone () ; let sink_port = D :: allocate_process_port (& from_node) ; let source_port = D :: allocate_cluster_port (& to_node) ; (D :: o2m_sink_source (compile_env , & from_node , & sink_port , & to_node , & source_port) , D :: o2m_connect (& from_node , & sink_port , & to_node , & source_port) ,) } (LocationId :: Cluster (from) , LocationId :: Process (to)) => { let from_node = clusters . get (from) . unwrap_or_else (| | { panic ! ("A cluster used in the graph was not instantiated: {}" , from) }) . clone () ; let to_node = nodes . get (to) . unwrap_or_else (| | { panic ! ("A process used in the graph was not instantiated: {}" , to) }) . clone () ; let sink_port = D :: allocate_cluster_port (& from_node) ; let source_port = D :: allocate_process_port (& to_node) ; (D :: m2o_sink_source (compile_env , & from_node , & sink_port , & to_node , & source_port) , D :: m2o_connect (& from_node , & sink_port , & to_node , & source_port) ,) } (LocationId :: Cluster (from) , LocationId :: Cluster (to)) => { let from_node = clusters . get (from) . unwrap_or_else (| | { panic ! ("A cluster used in the graph was not instantiated: {}" , from) }) . clone () ; let to_node = clusters . get (to) . unwrap_or_else (| | { panic ! ("A cluster used in the graph was not instantiated: {}" , to) }) . clone () ; let sink_port = D :: allocate_cluster_port (& from_node) ; let source_port = D :: allocate_cluster_port (& to_node) ; (D :: m2m_sink_source (compile_env , & from_node , & sink_port , & to_node , & source_port) , D :: m2m_connect (& from_node , & sink_port , & to_node , & source_port) ,) } (LocationId :: ExternalProcess (from) , LocationId :: Process (to)) => { let from_node = externals . get (from) . unwrap_or_else (| | { panic ! ("A external used in the graph was not instantiated: {}" , from) }) . clone () ; let to_node = nodes . get (to) . unwrap_or_else (| | { panic ! ("A process used in the graph was not instantiated: {}" , to) }) . clone () ; let sink_port = D :: allocate_external_port (& from_node) ; let source_port = D :: allocate_process_port (& to_node) ; from_node . register (from_key . unwrap () , sink_port . clone ()) ; ((parse_quote ! (DUMMY) , D :: e2o_source (compile_env , & from_node , & sink_port , & to_node , & source_port) ,) , D :: e2o_connect (& from_node , & sink_port , & to_node , & source_port) ,) } (LocationId :: ExternalProcess (_from) , LocationId :: Cluster (_to)) => { todo ! ("NYI") } (LocationId :: ExternalProcess (_) , LocationId :: ExternalProcess (_)) => { panic ! ("Cannot send from external to external") } (LocationId :: Process (from) , LocationId :: ExternalProcess (to)) => { let from_node = nodes . get (from) . unwrap_or_else (| | { panic ! ("A process used in the graph was not instantiated: {}" , from) }) . clone () ; let to_node = externals . get (to) . unwrap_or_else (| | { panic ! ("A external used in the graph was not instantiated: {}" , to) }) . clone () ; let sink_port = D :: allocate_process_port (& from_node) ; let source_port = D :: allocate_external_port (& to_node) ; to_node . register (to_key . unwrap () , source_port . clone ()) ; ((D :: o2e_sink (compile_env , & from_node , & sink_port , & to_node , & source_port) , parse_quote ! (DUMMY) ,) , D :: o2e_connect (& from_node , & sink_port , & to_node , & source_port) ,) } (LocationId :: Cluster (_from) , LocationId :: ExternalProcess (_to)) => { todo ! ("NYI") } (LocationId :: Tick (_ , _) , _) => panic ! () , (_ , LocationId :: Tick (_ , _)) => panic ! () , } ; (sink , source , connect_fn) } } pub mod rewrites { pub mod persist_pullup { pub use std :: cell :: RefCell ; pub use std :: collections :: HashSet ; pub use crate :: __staged :: ir :: * ; pub fn persist_pullup_node (node : & mut HfPlusNode , persist_pulled_tees : & mut HashSet < * const RefCell < HfPlusNode > > ,) { * node = match std :: mem :: replace (node , HfPlusNode :: Placeholder) { HfPlusNode :: Unpersist (box HfPlusNode :: Persist (box behind_persist)) => behind_persist , HfPlusNode :: Delta (box HfPlusNode :: Persist (box behind_persist)) => behind_persist , HfPlusNode :: Tee { inner } => { if persist_pulled_tees . contains (& (inner . 0 . as_ref () as * const RefCell < HfPlusNode >)) { HfPlusNode :: Persist (Box :: new (HfPlusNode :: Tee { inner : TeeNode (inner . 0 . clone ()) , })) } else if matches ! (* inner . 0 . borrow () , HfPlusNode :: Persist (_)) { persist_pulled_tees . insert (inner . 0 . as_ref () as * const RefCell < HfPlusNode >) ; if let HfPlusNode :: Persist (box behind_persist) = inner . 0 . replace (HfPlusNode :: Placeholder) { * inner . 0 . borrow_mut () = behind_persist ; } else { unreachable ! () } HfPlusNode :: Persist (Box :: new (HfPlusNode :: Tee { inner : TeeNode (inner . 0 . clone ()) , })) } else { HfPlusNode :: Tee { inner } } } HfPlusNode :: Map { f , input : box HfPlusNode :: Persist (behind_persist) , } => HfPlusNode :: Persist (Box :: new (HfPlusNode :: Map { f , input : behind_persist , })) , HfPlusNode :: FilterMap { f , input : box HfPlusNode :: Persist (behind_persist) , } => HfPlusNode :: Persist (Box :: new (HfPlusNode :: FilterMap { f , input : behind_persist , })) , HfPlusNode :: FlatMap { f , input : box HfPlusNode :: Persist (behind_persist) , } => HfPlusNode :: Persist (Box :: new (HfPlusNode :: FlatMap { f , input : behind_persist , })) , HfPlusNode :: Filter { f , input : box HfPlusNode :: Persist (behind_persist) , } => HfPlusNode :: Persist (Box :: new (HfPlusNode :: Filter { f , input : behind_persist , })) , HfPlusNode :: Network { from_location , from_key , to_location , to_key , serialize_pipeline , instantiate_fn , deserialize_pipeline , input : box HfPlusNode :: Persist (behind_persist) , .. } => HfPlusNode :: Persist (Box :: new (HfPlusNode :: Network { from_location , from_key , to_location , to_key , serialize_pipeline , instantiate_fn , deserialize_pipeline , input : behind_persist , })) , HfPlusNode :: Union (box HfPlusNode :: Persist (left) , box HfPlusNode :: Persist (right)) => { HfPlusNode :: Persist (Box :: new (HfPlusNode :: Union (left , right))) } HfPlusNode :: CrossProduct (box HfPlusNode :: Persist (left) , box HfPlusNode :: Persist (right)) => { HfPlusNode :: Persist (Box :: new (HfPlusNode :: Delta (Box :: new (HfPlusNode :: CrossProduct (Box :: new (HfPlusNode :: Persist (left)) , Box :: new (HfPlusNode :: Persist (right)) ,) ,)))) } HfPlusNode :: Join (box HfPlusNode :: Persist (left) , box HfPlusNode :: Persist (right)) => { HfPlusNode :: Persist (Box :: new (HfPlusNode :: Delta (Box :: new (HfPlusNode :: Join (Box :: new (HfPlusNode :: Persist (left)) , Box :: new (HfPlusNode :: Persist (right)) ,))))) } HfPlusNode :: Unique (box HfPlusNode :: Persist (inner)) => { HfPlusNode :: Persist (Box :: new (HfPlusNode :: Delta (Box :: new (HfPlusNode :: Unique (Box :: new (HfPlusNode :: Persist (inner)) ,))))) } node => node , } ; } pub fn persist_pullup (ir : Vec < HfPlusLeaf >) -> Vec < HfPlusLeaf > { let mut seen_tees = Default :: default () ; let mut persist_pulled_tees = Default :: default () ; ir . into_iter () . map (| l | { l . transform_children (| n , s | n . transform_bottom_up (persist_pullup_node , s , & mut persist_pulled_tees) , & mut seen_tees ,) }) . collect () } # [cfg (stageleft_macro)] pub mod tests { pub use stageleft :: * ; pub use crate :: __staged :: deploy :: MultiGraph ; pub use crate :: __staged :: location :: Location ; # [test] pub fn persist_pullup_through_map () { let flow = crate :: builder :: FlowBuilder :: new () ; let process = flow . process :: < () > () ; process . source_iter (q ! (0 .. 10)) . map (q ! (| v | v + 1)) . for_each (q ! (| n | println ! ("{}" , n))) ; let built = flow . finalize () ; insta :: assert_debug_snapshot ! (built . ir ()) ; let optimized = built . optimize_with (super :: persist_pullup) ; insta :: assert_debug_snapshot ! (optimized . ir ()) ; for (id , graph) in optimized . compile_no_network :: < MultiGraph > () . hydroflow_ir () { insta :: with_settings ! ({ snapshot_suffix => format ! ("surface_graph_{id}") } , { insta :: assert_snapshot ! (graph . surface_syntax_string ()) ; }) ; } } # [test] pub fn persist_pullup_behind_tee () { let flow = crate :: builder :: FlowBuilder :: new () ; let process = flow . process :: < () > () ; let tick = process . tick () ; let before_tee = process . source_iter (q ! (0 .. 10)) . tick_batch (& tick) . persist () ; before_tee . clone () . map (q ! (| v | v + 1)) . all_ticks () . for_each (q ! (| n | println ! ("{}" , n))) ; before_tee . clone () . map (q ! (| v | v + 1)) . all_ticks () . for_each (q ! (| n | println ! ("{}" , n))) ; let built = flow . finalize () ; insta :: assert_debug_snapshot ! (built . ir ()) ; let optimized = built . optimize_with (super :: persist_pullup) ; insta :: assert_debug_snapshot ! (optimized . ir ()) ; for (id , graph) in optimized . compile_no_network :: < MultiGraph > () . hydroflow_ir () { insta :: with_settings ! ({ snapshot_suffix => format ! ("surface_graph_{id}") } , { insta :: assert_snapshot ! (graph . surface_syntax_string ()) ; }) ; } } } } pub mod profiler { pub use std :: cell :: RefCell ; pub use hydroflow :: futures :: channel :: mpsc :: UnboundedSender ; pub use stageleft :: * ; pub use super :: profiler as myself ; pub use crate :: __staged :: ir :: * ; pub use crate :: __staged :: RuntimeContext ; pub fn increment_counter (count : & mut u64) { * count += 1 ; } pub fn quoted_any_fn < 'a , F : Fn (& usize) + 'a , Q : IntoQuotedMut < 'a , F > > (q : Q) -> Q { q } # [doc = " Add a profiling node before each node to count the cardinality of its input"] pub fn add_profiling_node < 'a > (node : & mut HfPlusNode , _context : RuntimeContext < 'a > , counters : RuntimeData < & 'a RefCell < Vec < u64 > > > , counter_queue : RuntimeData < & 'a RefCell < UnboundedSender < (usize , u64) > > > , id : & mut u32 , seen_tees : & mut SeenTees ,) { let my_id = * id ; * id += 1 ; node . transform_children (| node , seen_tees | { add_profiling_node (node , _context , counters , counter_queue , id , seen_tees) } , seen_tees ,) ; let orig_node = std :: mem :: replace (node , HfPlusNode :: Placeholder) ; * node = HfPlusNode :: Inspect { f : quoted_any_fn (q ! ({ counter_queue . borrow () . unbounded_send ((my_id as usize , counters . borrow () [my_id as usize])) . unwrap () ; counters . borrow_mut () [my_id as usize] = 0 ; move | _ | { myself :: increment_counter (& mut counters . borrow_mut () [my_id as usize]) ; } })) . splice_untyped () . into () , input : Box :: new (orig_node) , } } # [doc = " Count the cardinality of each input and periodically output to a file"] pub fn profiling < 'a > (ir : Vec < HfPlusLeaf > , context : RuntimeContext < 'a > , counters : RuntimeData < & 'a RefCell < Vec < u64 > > > , counter_queue : RuntimeData < & 'a RefCell < UnboundedSender < (usize , u64) > > > ,) -> Vec < HfPlusLeaf > { let mut id = 0 ; let mut seen_tees = Default :: default () ; ir . into_iter () . map (| l | { l . transform_children (| node , seen_tees | { add_profiling_node (node , context , counters , counter_queue , & mut id , seen_tees) } , & mut seen_tees ,) }) . collect () } # [cfg (stageleft_macro)] pub mod tests { pub use stageleft :: * ; pub use crate :: __staged :: deploy :: MultiGraph ; pub use crate :: __staged :: location :: Location ; # [test] pub fn profiler_wrapping_all_operators () { let flow = crate :: builder :: FlowBuilder :: new () ; let process = flow . process :: < () > () ; process . source_iter (q ! (0 .. 10)) . map (q ! (| v | v + 1)) . for_each (q ! (| n | println ! ("{}" , n))) ; let runtime_context = flow . runtime_context () ; let built = flow . finalize () ; insta :: assert_debug_snapshot ! (& built . ir ()) ; let counters = RuntimeData :: new ("Fake") ; let counter_queue = RuntimeData :: new ("Fake") ; let pushed_down = built . optimize_with (crate :: rewrites :: persist_pullup :: persist_pullup) . optimize_with (| ir | super :: profiling (ir , runtime_context , counters , counter_queue)) ; insta :: assert_debug_snapshot ! (& pushed_down . ir ()) ; let _ = pushed_down . compile_no_network :: < MultiGraph > () ; } } } pub mod properties { pub use std :: collections :: HashSet ; pub use stageleft :: * ; pub use crate :: __staged :: ir :: { HfPlusLeaf , HfPlusNode , SeenTees } ; pub use crate :: rewrites :: properties :: PropertyDatabase ; # [doc = " Allows us to convert the hydroflow datatype for folds to a binary operation for the algebra"] # [doc = " property tests."] # [allow (clippy :: allow_attributes , dead_code , reason = "staged programming")] pub fn convert_hf_to_binary < I , A : Default , F : Fn (& mut A , I) > (f : F) -> impl Fn (I , I) -> A { move | a , b | { let mut acc = Default :: default () ; f (& mut acc , a) ; f (& mut acc , b) ; acc } } # [cfg (stageleft_macro)] impl PropertyDatabase { # [doc = " Tags the expression as commutative."] pub fn add_commutative_tag < 'a , I , A , F : Fn (& mut A , I) , Q : Quoted < 'a , F > + Clone > (& mut self , expr : Q ,) -> Q { let expr_clone = expr . clone () ; self . commutative . insert (expr_clone . splice_untyped ()) ; expr } pub fn is_tagged_commutative (& self , expr : & syn :: Expr) -> bool { self . commutative . contains (expr) } } pub fn properties_optimize_node (node : & mut HfPlusNode , db : & PropertyDatabase , seen_tees : & mut SeenTees ,) { node . transform_children (| node , seen_tees | properties_optimize_node (node , db , seen_tees) , seen_tees ,) ; match node { HfPlusNode :: ReduceKeyed { f , .. } if db . is_tagged_commutative (& f . 0) => { dbg ! ("IDENTIFIED COMMUTATIVE OPTIMIZATION for {:?}" , & f) ; } _ => { } } } pub fn properties_optimize (ir : Vec < HfPlusLeaf > , db : & PropertyDatabase) -> Vec < HfPlusLeaf > { let mut seen_tees = Default :: default () ; ir . into_iter () . map (| l | { l . transform_children (| node , seen_tees | properties_optimize_node (node , db , seen_tees) , & mut seen_tees ,) }) . collect () } # [cfg (stageleft_macro)] pub mod tests { pub use super :: * ; pub use crate :: __staged :: deploy :: SingleProcessGraph ; pub use crate :: __staged :: location :: Location ; pub use crate :: __staged :: FlowBuilder ; # [test] pub fn test_property_database () { let mut db = PropertyDatabase :: default () ; assert ! (! db . is_tagged_commutative (& (q ! (| a : & mut i32 , b : i32 | * a += b) . splice_untyped ()))) ; let _ = db . add_commutative_tag (q ! (| a : & mut i32 , b : i32 | * a += b)) ; assert ! (db . is_tagged_commutative (& (q ! (| a : & mut i32 , b : i32 | * a += b) . splice_untyped ()))) ; } # [test] pub fn test_property_optimized () { let flow = FlowBuilder :: new () ; let mut database = PropertyDatabase :: default () ; let process = flow . process :: < () > () ; let tick = process . tick () ; let counter_func = q ! (| count : & mut i32 , _ | * count += 1) ; let _ = database . add_commutative_tag (counter_func) ; process . source_iter (q ! (vec ! [])) . map (q ! (| string : String | (string , ()))) . tick_batch (& tick) . fold_keyed (q ! (|| 0) , counter_func) . all_ticks () . for_each (q ! (| (string , count) | println ! ("{}: {}" , string , count))) ; let built = flow . optimize_with (| ir | properties_optimize (ir , & database)) . with_default_optimize :: < SingleProcessGraph > () ; insta :: assert_debug_snapshot ! (built . ir ()) ; let _ = built . compile_no_network () ; } } } } pub mod staging_util { pub use proc_macro2 :: { Span , TokenStream } ; pub use quote :: quote ; pub fn get_this_crate () -> TokenStream { let hydroflow_crate = proc_macro_crate :: crate_name ("hydroflow_plus") . expect ("hydroflow_plus should be present in `Cargo.toml`") ; match hydroflow_crate { proc_macro_crate :: FoundCrate :: Itself => quote ! { hydroflow_plus } , proc_macro_crate :: FoundCrate :: Name (name) => { let ident = syn :: Ident :: new (& name , Span :: call_site ()) ; quote ! { # ident } } } } } # [cfg (stageleft_macro)] # [stageleft :: runtime] # [cfg (test)] mod tests { # [ctor :: ctor] fn init () { crate :: deploy :: init_test () ; } } [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/deploy/macro_runtime.rs:189:17 [INFO] [stdout] | [INFO] [stdout] 189 | fn raw_port(&self, _key: usize) -> ::ExternalRawPort { [INFO] [stdout] | ^^^^^ ------------------------------------------ the same lifetime is hidden here [INFO] [stdout] | | [INFO] [stdout] | the lifetime is elided here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 189 | fn raw_port(&self, _key: usize) -> >::ExternalRawPort { [INFO] [stdout] | ++++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/deploy/deploy_runtime.rs:17:22 [INFO] [stdout] | [INFO] [stdout] 17 | cli: RuntimeData<&DeployPorts>, [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the lifetime is elided here [INFO] [stdout] 18 | of_cluster: usize, [INFO] [stdout] 19 | ) -> impl Quoted<&Vec> + Copy { [INFO] [stdout] | ----------------------------- [INFO] [stdout] | | | [INFO] [stdout] | | the same lifetime is elided here [INFO] [stdout] | the same lifetime is hidden here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 19 | ) -> impl Quoted<'_, &Vec> + Copy { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] warning: hiding a lifetime that's elided elsewhere is confusing [INFO] [stdout] --> src/deploy/deploy_runtime.rs:24:22 [INFO] [stdout] | [INFO] [stdout] 24 | cli: RuntimeData<&DeployPorts>, [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the lifetime is elided here [INFO] [stdout] 25 | ) -> impl Quoted + Copy { [INFO] [stdout] | ----------------------- the same lifetime is hidden here [INFO] [stdout] | [INFO] [stdout] = help: the same lifetime is referred to in inconsistent ways, making the signature confusing [INFO] [stdout] help: use `'_` for type paths [INFO] [stdout] | [INFO] [stdout] 25 | ) -> impl Quoted<'_, u32> + Copy { [INFO] [stdout] | +++ [INFO] [stdout] [INFO] [stdout] [INFO] [stderr] Checking ssh2 v0.9.5 [INFO] [stderr] Checking async-ssh2-lite v0.5.0 [INFO] [stderr] Checking hydro_deploy v0.10.0 [INFO] [stdout] error[E0425]: cannot find function `init_test` in module `crate::deploy` [INFO] [stdout] --> src/lib.rs:47:24 [INFO] [stdout] | [INFO] [stdout] 47 | crate::deploy::init_test(); [INFO] [stdout] | ^^^^^^^^^ not found in `crate::deploy` [INFO] [stdout] | [INFO] [stdout] note: found an item that was configured out [INFO] [stdout] --> src/deploy/mod.rs:20:19 [INFO] [stdout] | [INFO] [stdout] 19 | #[cfg(feature = "deploy")] [INFO] [stdout] | ------------------ the item is gated behind the `deploy` feature [INFO] [stdout] 20 | pub use trybuild::init_test; [INFO] [stdout] | ^^^^^^^^^ [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] error[E0277]: the trait bound `Arc: IntoProcessSpec<'_, _>` is not satisfied [INFO] [stdout] --> src/stream.rs:867:40 [INFO] [stdout] | [INFO] [stdout] 867 | .with_process(&first_node, deployment.Localhost()) [INFO] [stdout] | ------------ ^^^^^^^^^^^^^^^^^^^^^^ the trait `ProcessSpec<'_, _>` is not implemented for `Arc` [INFO] [stdout] | | [INFO] [stdout] | required by a bound introduced by this call [INFO] [stdout] | [INFO] [stdout] = help: the following other types implement trait `ProcessSpec<'a, D>`: [INFO] [stdout] `()` implements `ProcessSpec<'_, DeployRuntime>` [INFO] [stdout] `()` implements `ProcessSpec<'_, deploy::in_memory_graph::MultiGraph>` [INFO] [stdout] `()` implements `ProcessSpec<'_, deploy::in_memory_graph::SingleProcessGraph>` [INFO] [stdout] note: required for `Arc` to implement `IntoProcessSpec<'_, _>` [INFO] [stdout] --> src/deploy/mod.rs:216:62 [INFO] [stdout] | [INFO] [stdout] 216 | impl<'a, D: LocalDeploy<'a> + ?Sized, T: ProcessSpec<'a, D>> IntoProcessSpec<'a, D> for T { [INFO] [stdout] | ------------------ ^^^^^^^^^^^^^^^^^^^^^^ ^ [INFO] [stdout] | | [INFO] [stdout] | unsatisfied trait bound introduced here [INFO] [stdout] note: required by a bound in `FlowBuilder::<'a>::with_process` [INFO] [stdout] --> src/builder/mod.rs:168:20 [INFO] [stdout] | [INFO] [stdout] 165 | pub fn with_process>( [INFO] [stdout] | ------------ required by a bound in this associated function [INFO] [stdout] ... [INFO] [stdout] 168 | spec: impl IntoProcessSpec<'a, D>, [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `FlowBuilder::<'a>::with_process` [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] error[E0277]: the trait bound `Arc: IntoProcessSpec<'_, _>` is not satisfied [INFO] [stdout] --> src/stream.rs:868:41 [INFO] [stdout] | [INFO] [stdout] 868 | .with_process(&second_node, deployment.Localhost()) [INFO] [stdout] | ------------ ^^^^^^^^^^^^^^^^^^^^^^ the trait `ProcessSpec<'_, _>` is not implemented for `Arc` [INFO] [stdout] | | [INFO] [stdout] | required by a bound introduced by this call [INFO] [stdout] | [INFO] [stdout] = help: the following other types implement trait `ProcessSpec<'a, D>`: [INFO] [stdout] `()` implements `ProcessSpec<'_, DeployRuntime>` [INFO] [stdout] `()` implements `ProcessSpec<'_, deploy::in_memory_graph::MultiGraph>` [INFO] [stdout] `()` implements `ProcessSpec<'_, deploy::in_memory_graph::SingleProcessGraph>` [INFO] [stdout] note: required for `Arc` to implement `IntoProcessSpec<'_, _>` [INFO] [stdout] --> src/deploy/mod.rs:216:62 [INFO] [stdout] | [INFO] [stdout] 216 | impl<'a, D: LocalDeploy<'a> + ?Sized, T: ProcessSpec<'a, D>> IntoProcessSpec<'a, D> for T { [INFO] [stdout] | ------------------ ^^^^^^^^^^^^^^^^^^^^^^ ^ [INFO] [stdout] | | [INFO] [stdout] | unsatisfied trait bound introduced here [INFO] [stdout] note: required by a bound in `DeployFlow::<'a, D>::with_process` [INFO] [stdout] --> src/builder/deploy.rs:49:20 [INFO] [stdout] | [INFO] [stdout] 46 | pub fn with_process

( [INFO] [stdout] | ------------ required by a bound in this associated function [INFO] [stdout] ... [INFO] [stdout] 49 | spec: impl IntoProcessSpec<'a, D>, [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `DeployFlow::<'a, D>::with_process` [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] error[E0277]: the trait bound `Arc: ExternalSpec<'_, _>` is not satisfied [INFO] [stdout] --> src/stream.rs:869:39 [INFO] [stdout] | [INFO] [stdout] 869 | .with_external(&external, deployment.Localhost()) [INFO] [stdout] | ------------- ^^^^^^^^^^^^^^^^^^^^^^ the trait `ExternalSpec<'_, _>` is not implemented for `Arc` [INFO] [stdout] | | [INFO] [stdout] | required by a bound introduced by this call [INFO] [stdout] | [INFO] [stdout] = help: the trait `ExternalSpec<'_, DeployRuntime>` is implemented for `()` [INFO] [stdout] note: required by a bound in `DeployFlow::<'a, D>::with_external` [INFO] [stdout] --> src/builder/deploy.rs:62:20 [INFO] [stdout] | [INFO] [stdout] 59 | pub fn with_external

( [INFO] [stdout] | ------------- required by a bound in this associated function [INFO] [stdout] ... [INFO] [stdout] 62 | spec: impl ExternalSpec<'a, D>, [INFO] [stdout] | ^^^^^^^^^^^^^^^^^^^ required by this bound in `DeployFlow::<'a, D>::with_external` [INFO] [stdout] [INFO] [stdout] [INFO] [stdout] Some errors have detailed explanations: E0277, E0425. [INFO] [stdout] [INFO] [stdout] For more information about an error, try `rustc --explain E0277`. [INFO] [stdout] [INFO] [stderr] error: could not compile `hydroflow_plus` (lib test) due to 4 previous errors [INFO] running `Command { std: "docker" "inspect" "5afa19b2952e29819cbfb4e0ac601e2027c0fdefe8b30d3ba7d0a5d4cd1fedc6", kill_on_drop: false }` [INFO] running `Command { std: "docker" "rm" "-f" "5afa19b2952e29819cbfb4e0ac601e2027c0fdefe8b30d3ba7d0a5d4cd1fedc6", kill_on_drop: false }` [INFO] [stdout] 5afa19b2952e29819cbfb4e0ac601e2027c0fdefe8b30d3ba7d0a5d4cd1fedc6