diff --git a/Cargo.lock b/Cargo.lock index 5349a59..0341b4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,9 +82,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.43" +version = "1.2.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "739eb0f94557554b3ca9a86d2d37bebd49c5e6d0c1d2bda35ba5bdac830befc2" +checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3" dependencies = [ "find-msvc-tools", "shlex", @@ -131,18 +131,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.50" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2cfd7bf8a6017ddaa4e32ffe7403d547790db06bd171c1c53926faab501623" +checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.50" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4c05b9e80c5ccd3a7ef080ad7b6ba7d6fc00a985b8b157197075677c82c7a0" +checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" dependencies = [ "anstyle", "clap_lex", @@ -187,6 +187,15 @@ dependencies = [ "itertools", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -628,9 +637,10 @@ dependencies = [ [[package]] name = "kiru" -version = "0.1.9" +version = "0.1.10" dependencies = [ "criterion", + "crossbeam-channel", "glob", "proptest", "rayon", @@ -643,7 +653,7 @@ dependencies = [ [[package]] name = "kiru-py" -version = "0.1.9" +version = "0.1.10" dependencies = [ "kiru", "pyo3", @@ -1491,9 +1501,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-ident" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unindent" diff --git a/kiru-core/Cargo.toml b/kiru-core/Cargo.toml index 5139d1d..45671f0 100644 --- a/kiru-core/Cargo.toml +++ b/kiru-core/Cargo.toml @@ -1,7 +1,7 @@ # kiru/Cargo.toml [package] name = "kiru" -version = "0.1.9" +version = "0.1.10" edition = "2021" description = "Fast text chunking for Rust" license = "MIT" @@ -25,6 +25,7 @@ glob = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +crossbeam-channel = "0.5.15" [dev-dependencies] tempfile = { workspace = true } diff --git a/kiru-core/src/chunker.rs b/kiru-core/src/chunker.rs index a81ae50..06d0a70 100644 --- a/kiru-core/src/chunker.rs +++ b/kiru-core/src/chunker.rs @@ -1,8 +1,8 @@ +use crossbeam_channel::bounded; use glob::glob; use rayon::prelude::*; -use std::sync::{mpsc, Mutex}; +use std::io; use std::thread; -use std::{io, sync::Arc}; use thiserror::Error; use crate::{BytesChunker, CharactersChunker, StreamType}; @@ -249,9 +249,8 @@ impl ChunkerWithStrategy { StreamType::from_source(source)?; // This validates the source } - let (sender, receiver) = mpsc::sync_channel(channel_size); + let (sender, receiver) = bounded(channel_size); let chunker_params = self.chunker_params.clone(); - let receiver: Arc>> = Arc::new(Mutex::new(receiver)); thread::spawn({ move || { @@ -273,13 +272,7 @@ impl ChunkerWithStrategy { } }); - let iterator = std::iter::from_fn({ - let receiver: Arc>> = Arc::clone(&receiver); // Clone Arc for iterator - move || { - let receiver = receiver.lock().unwrap(); - receiver.recv().ok() - } - }); + let iterator = std::iter::from_fn(move || receiver.recv().ok()); Ok(Box::new(iterator)) } diff --git a/kiru-py/Cargo.toml b/kiru-py/Cargo.toml index 559088d..5c71546 100644 --- a/kiru-py/Cargo.toml +++ b/kiru-py/Cargo.toml @@ -1,7 +1,7 @@ # kiru-py/Cargo.toml [package] name = "kiru-py" -version = "0.1.9" +version = "0.1.10" edition = "2021" description = "Python bindings for kiru text chunking library" repository = "https://github.com/bitswired/kiru" @@ -17,6 +17,6 @@ pyo3 = { workspace = true } [dependencies.kiru] path = "../kiru-core" -version = "0.1.9" +version = "0.1.10" [build-dependencies] # None needed for simple bindings