diff --git a/README.md b/README.md index a11aea2..6ebdd0f 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,11 @@ [![Documentation](https://docs.rs/sje/badge.svg)](https://docs.rs/sje/) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -# sje +# Simple Json Encoding Fast JSON deserialisation and serialisation schema based framework. -## Example +## Examples Make sure the `derive` feature is enabled. @@ -15,6 +15,9 @@ Make sure the `derive` feature is enabled. sje = { version = "0.0.4", features = ["derive"]} ``` +Simply annotate your struct using `#[sje]` attribute, The `len` field can be used when you know the exact size of the value field which means the +decoder can handle it more efficiently. + ```rust #[derive(Decoder)] #[sje(object)] @@ -46,4 +49,80 @@ assert_eq!("trade", trade.event_type_as_str()); assert_eq!(1705085312569, trade.event_time()); ``` +We can also handle arrays and tuples. In this case if we want to use generated `PositionDecoder` we need to explicitly mark it with `decoder = true`. + +```rust +#[derive(Decoder)] +#[sje(object)] +struct Position { + #[sje(rename = "s")] + symbol: String, + #[sje(rename = "a")] + amount: u32, +} + +#[derive(Decoder)] +#[sje(object)] +struct PositionUpdate { + #[sje(rename = "t")] + timestamp: u64, + #[sje(rename = "u", decoder = true)] + updates: Vec, +} +``` + +The generated code will contain iterators that already know the length of the array. + +```rust +let update = PositionUpdateDecoder::decode(br#"{"t":1746699621,"u":[{"s":"btcusdt","a":100},{"s":"ethusdt","a":200}]}"#).unwrap(); +assert_eq!(2, update.updates_count()); + +let mut updates = update.updates().into_iter(); + +let position = updates.next().unwrap(); +assert_eq!("btcusdt", position.symbol_as_str()); +assert_eq!(100, position.amount()); + +let position = updates.next().unwrap(); +assert_eq!("ethusdt", position.symbol_as_str()); +assert_eq!(200, position.amount()); +assert!(positions.next().is_none()); +``` + +The framework also handles user defined types that don't require an explicit `Decoder`. In this case, the only requirement is that the type +implements `FromStr` trait. We also need to tell the parser what is the underlying json type for our user defined type, in this case `ty = "string"`. + +```rust +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +struct Price(u64); + +impl FromStr for Price { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + Ok(Self(s.parse().map_err(|_| "unable to parse the price")?)) + } +} + +#[derive(Decoder)] +#[sje(object)] +pub struct Trade { + #[sje(rename = "p", ty = "string")] + price: Price, +} + +let trade = TradeDecoder::decode(br#"{"p":"12345"}"#).unwrap(); +assert_eq!(Price(12345), trade.price()); +``` + +## Benchmarks + +There are [benchmarks](sje/benches) against [serde_json](https://crates.io/crates/serde_json) that show an order of magnitude +speedup. Please note `sje` is not a generic purpose json parser - it's fast because it takes advantage of fixed schema and +lacks a lot of features you would find in `serde_json` to handle more dynamic content. + +```shell +RUSTFLAGS='-C target-cpu=native' cargo bench --bench=ticker +``` +![img.png](docs/benchmark.png) \ No newline at end of file diff --git a/docs/benchmark.png b/docs/benchmark.png new file mode 100644 index 0000000..647faad Binary files /dev/null and b/docs/benchmark.png differ diff --git a/sje/array_of_objects/custom.rs b/sje/array_of_objects/custom.rs new file mode 100644 index 0000000..bb8a8f2 --- /dev/null +++ b/sje/array_of_objects/custom.rs @@ -0,0 +1,30 @@ +use sje_derive::Decoder; +use std::str::FromStr; + +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +struct Price(u64); + +impl FromStr for Price { + type Err = (); + + fn from_str(s: &str) -> Result { + Ok(Self(s.parse().map_err(|_| ())?)) + } +} + +#[derive(Decoder)] +#[sje(object)] +#[allow(dead_code)] +pub struct Trade { + #[sje(rename = "p", ty = "string")] + price: Price, +} + +#[test] +fn should_parse_custom_field() { + let json = r#"{"p":"12345"}"#; + let trade = TradeDecoder::decode(json.as_bytes()).unwrap(); + assert_eq!(&Price(12345), trade.price_as_lazy_field().get_ref().unwrap()); + assert_eq!(Price(12345), trade.price_as_lazy_field().get().unwrap()); + assert_eq!(Price(12345), trade.price()); +} diff --git a/sje/array_of_objects/decoder.rs b/sje/array_of_objects/decoder.rs new file mode 100644 index 0000000..3d5f7b2 --- /dev/null +++ b/sje/array_of_objects/decoder.rs @@ -0,0 +1,74 @@ +use sje_derive::Decoder; + +#[derive(Decoder)] +#[sje(object)] +#[allow(dead_code)] +pub struct Trade { + #[sje(rename = "e", len = 5)] + event_type: String, + #[sje(rename = "E", len = 13)] + event_time: u64, + #[sje(rename = "s")] + symbol: String, + #[sje(rename = "t", len = 10)] + trade_id: u64, + #[sje(rename = "p")] + price: String, + #[sje(rename = "q")] + quantity: String, + #[sje(rename = "b", len = 11)] + buyer_order_id: u64, + #[sje(rename = "a", len = 11)] + seller_order_id: u64, + #[sje(rename = "T", len = 13)] + transaction_time: u64, + #[sje(rename = "m")] + is_buyer_maker: bool, +} + +#[derive(Decoder, Debug)] +#[sje(object)] +#[allow(dead_code)] +struct ListenKeyExpired { + #[sje(rename = "e", len = 16, offset = 1)] + event_type: String, + #[sje(rename = "E", ty = "string", len = 13, offset = 1)] + event_time: u64, + #[sje(rename = "listenKey", offset = 1)] + listen_key: String, +} + +#[cfg(test)] +mod tests { + use crate::{ListenKeyExpiredDecoder, Trade, TradeDecoder}; + use std::str::from_utf8_unchecked; + + #[test] + fn should_decode_trade() { + let trade = TradeDecoder::decode(br#"{"e":"trade","E":1705085312569,"s":"BTCUSDT","t":3370034463,"p":"43520.00000000","q":"0.00022000","b":24269765071,"a":24269767699,"T":1705085312568,"m":true,"M":true}"#).unwrap(); + assert_eq!("trade", trade.event_type()); + assert_eq!("BTCUSDT", unsafe { from_utf8_unchecked(trade.symbol_as_slice()) }); + assert_eq!("BTCUSDT", trade.symbol_as_str()); + assert_eq!("BTCUSDT", trade.symbol()); + + let trade: Trade = trade.into(); + assert_eq!("trade", trade.event_type); + assert_eq!(1705085312569, trade.event_time); + assert_eq!("BTCUSDT", trade.symbol); + assert_eq!(3370034463, trade.trade_id); + assert_eq!("43520.00000000", trade.price); + assert_eq!("0.00022000", trade.quantity); + assert_eq!(24269765071, trade.buyer_order_id); + assert_eq!(24269767699, trade.seller_order_id); + assert_eq!(1705085312568, trade.transaction_time); + assert!(trade.is_buyer_maker); + } + + #[test] + fn should_decode_listen_key_expired() { + let listen_key_expired = ListenKeyExpiredDecoder::decode(br#"{"e": "listenKeyExpired","E": "1743606297156","listenKey": "FdffIUjdfd343DtLMw2tKS87iL2HpYRniDWpkoxWCb4fwP2yzJXalBlBNnz471cE"}"#).unwrap(); + assert_eq!("listenKeyExpired", listen_key_expired.event_type()); + assert_eq!(1743606297156, listen_key_expired.event_time()); + assert_eq!("FdffIUjdfd343DtLMw2tKS87iL2HpYRniDWpkoxWCb4fwP2yzJXalBlBNnz471cE", listen_key_expired.listen_key()); + } +} diff --git a/sje/array_of_objects/iter.rs b/sje/array_of_objects/iter.rs new file mode 100644 index 0000000..603a9ee --- /dev/null +++ b/sje/array_of_objects/iter.rs @@ -0,0 +1,117 @@ +use sje_derive::Decoder; +use std::str::FromStr; + +#[derive(Debug, Copy, Clone, PartialEq)] +#[repr(transparent)] +pub struct Price(f64); + +impl FromStr for Price { + type Err = ::Err; + + fn from_str(s: &str) -> Result { + Ok(Self(s.parse()?)) + } +} + +#[derive(Debug, Copy, Clone, PartialEq)] +#[repr(transparent)] +pub struct Quantity(f64); + +impl FromStr for Quantity { + type Err = ::Err; + + fn from_str(s: &str) -> Result { + Ok(Self(s.parse()?)) + } +} + +#[derive(Decoder, Debug)] +#[sje(object)] +#[allow(dead_code)] +pub struct L2Update { + #[sje(rename = "e", len = 11)] + event_type: String, + #[sje(rename = "b")] + bids: Vec<(Price, Quantity)>, + #[sje(rename = "a")] + asks: Vec<(Price, Quantity)>, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn should_iterate_over_bids_and_asks() { + let update = L2UpdateDecoder::decode( + br#"{"e":"depthUpdate","b":[["2.6461","6404.9"],["2.6468","22540.8"]],"a":[["2.6461","6404.9"],["2.6468","22540.8"]]}"#, + ).unwrap(); + let mut bids = update.bids().into_iter(); + assert_eq!(2, update.bids_count()); + assert_eq!(2, bids.len()); + assert_eq!(Some((Price(2.6461), Quantity(6404.9))), bids.next()); + assert_eq!(1, bids.len()); + assert_eq!(Some((Price(2.6468), Quantity(22540.8))), bids.next()); + assert_eq!(0, bids.len()); + assert_eq!(None, bids.next()); + assert_eq!(0, bids.len()); + let mut asks = update.asks().into_iter(); + assert_eq!(2, update.asks_count()); + assert_eq!(2, asks.len()); + assert_eq!(Some((Price(2.6461), Quantity(6404.9))), asks.next()); + assert_eq!(1, asks.len()); + assert_eq!(Some((Price(2.6468), Quantity(22540.8))), asks.next()); + assert_eq!(0, asks.len()); + assert_eq!(None, asks.next()); + assert_eq!(0, asks.len()); + + let update = + L2UpdateDecoder::decode(br#"{"e":"depthUpdate","b":[["2.6461","6404.9"]],"a":[["2.6461","6404.9"]]}"#) + .unwrap(); + let mut bids = update.bids().into_iter(); + assert_eq!(1, update.bids_count()); + assert_eq!(1, bids.len()); + assert_eq!(Some((Price(2.6461), Quantity(6404.9))), bids.next()); + assert_eq!(0, bids.len()); + assert_eq!(None, bids.next()); + assert_eq!(0, bids.len()); + let mut asks = update.asks().into_iter(); + assert_eq!(1, update.asks_count()); + assert_eq!(1, asks.len()); + assert_eq!(Some((Price(2.6461), Quantity(6404.9))), asks.next()); + assert_eq!(0, asks.len()); + assert_eq!(None, asks.next()); + assert_eq!(0, asks.len()); + + let update = L2UpdateDecoder::decode(br#"{"e":"depthUpdate","b":[],"a":[]}"#).unwrap(); + let mut bids = update.bids().into_iter(); + assert_eq!(0, update.bids_count()); + assert_eq!(0, bids.len()); + assert_eq!(None, bids.next()); + assert_eq!(0, bids.len()); + let mut asks = update.asks().into_iter(); + assert_eq!(0, update.asks_count()); + assert_eq!(0, asks.len()); + assert_eq!(None, asks.next()); + assert_eq!(0, asks.len()); + } + + #[test] + fn should_convert_to_owned() { + let update = L2UpdateDecoder::decode( + br#"{"e":"depthUpdate","b":[["2.6461","6404.9"],["2.6468","22540.8"]],"a":[["2.6461","6404.9"],["2.6468","22540.8"]]}"#, + ).unwrap(); + let update: L2Update = update.into(); + assert_eq!("depthUpdate", update.event_type); + + let mut bids = update.bids.into_iter(); + assert_eq!(Some((Price(2.6461), Quantity(6404.9))), bids.next()); + assert_eq!(Some((Price(2.6468), Quantity(22540.8))), bids.next()); + assert_eq!(None, bids.next()); + + let mut asks = update.asks.into_iter(); + assert_eq!(Some((Price(2.6461), Quantity(6404.9))), asks.next()); + assert_eq!(Some((Price(2.6468), Quantity(22540.8))), asks.next()); + assert_eq!(None, asks.next()); + } +} diff --git a/sje/benches/ticker.rs b/sje/benches/ticker.rs index 7e2fef4..bacf3a4 100644 --- a/sje/benches/ticker.rs +++ b/sje/benches/ticker.rs @@ -8,7 +8,7 @@ const JSON: &[u8] = br#"{"e":"bookTicker","u":6780157666962,"s":"BTCUSDT","b":"9 #[sje(object)] #[allow(dead_code)] pub struct Ticker { - #[sje(rename = "e", len = 10, offset = 2)] + #[sje(rename = "e", len = 10)] #[serde(rename = "e")] event_type: String, #[sje(rename = "u", len = 13)] diff --git a/sje/src/macros.rs b/sje/src/macros.rs index 8963ac1..a7a8b61 100644 --- a/sje/src/macros.rs +++ b/sje/src/macros.rs @@ -47,25 +47,66 @@ macro_rules! composite_impl { ($method_name:ident, $open_char:literal, $close_char:literal) => { impl<'a> JsonScanner<'a> { #[inline] - pub fn $method_name(&mut self) -> Option<(usize, usize)> { - let offset = self.cursor; - let mut counter = 1u32; - for (index, &item) in unsafe { self.bytes.get_unchecked(offset + 1..) } - .iter() - .enumerate() - { - match item { + pub const fn $method_name(&mut self) -> Option<(usize, usize)> { + let bytes = self.bytes; + let start = self.cursor; + let mut counter: u32 = 1; + let mut i: usize = 0; + + loop { + // if we've run off the end, give up + let idx = start + 1 + i; + if idx >= bytes.len() { + return None; + } + + // fetch the next byte after the opening char + let b = bytes[idx]; + + // bump the nesting counter + match b { $open_char => counter += 1, $close_char => counter -= 1, _ => {} } + + // if we've closed the top‐level object, return its span if counter == 0 { - self.cursor += index + 2; - return Some((offset, index + 2)); + self.cursor = start + i + 2; + return Some((start, i + 2)); } + + i += 1; } - None } } }; } + +// #[macro_export] +// macro_rules! composite_impl { +// ($method_name:ident, $open_char:literal, $close_char:literal) => { +// impl<'a> JsonScanner<'a> { +// #[inline] +// pub fn $method_name(&mut self) -> Option<(usize, usize)> { +// let offset = self.cursor; +// let mut counter = 1u32; +// for (index, &item) in unsafe { self.bytes.get_unchecked(offset + 1..) } +// .iter() +// .enumerate() +// { +// match item { +// $open_char => counter += 1, +// $close_char => counter -= 1, +// _ => {} +// } +// if counter == 0 { +// self.cursor += index + 2; +// return Some((offset, index + 2)); +// } +// } +// None +// } +// } +// }; +// } diff --git a/sje/src/scanner.rs b/sje/src/scanner.rs index ab4e355..c7dacae 100644 --- a/sje/src/scanner.rs +++ b/sje/src/scanner.rs @@ -34,34 +34,100 @@ composite_impl!(next_tuple, b'[', b']'); composite_impl!(next_object, b'{', b'}'); impl JsonScanner<'_> { - #[inline] - pub fn next_array(&mut self) -> Option<(usize, usize, usize)> { - let offset = self.cursor; - let mut counter = 1u32; - let mut array_len = 0; - for (index, &item) in unsafe { self.bytes.get_unchecked(offset + 1..) }.iter().enumerate() { - match item { - b'[' => counter += 1, - b']' => counter -= 1, - _ => {} + pub const fn next_array(&mut self) -> Option<(usize, usize, usize)> { + let bytes = self.bytes; + let start = self.cursor; + + // state + let mut array_depth = 0usize; + let mut obj_depth = 0usize; + let mut in_string = false; + let mut escaped = false; + let mut commas = 0usize; + let mut saw_value = false; + let mut index = 0usize; + + // iterate until we run off the end + loop { + // bounds-check + if start + index >= bytes.len() { + return None; } - - if item == b',' && counter == 1 { - array_len += 1; + let b = bytes[start + index]; + + // 1) handle strings & escapes + if in_string { + if escaped { + escaped = false; + } else if b == b'\\' { + escaped = true; + } else if b == b'"' { + in_string = false; + } + index += 1; + continue; + } else if b == b'"' { + in_string = true; + index += 1; + continue; } - if counter == 0 { - if index > 0 { - let previous = unsafe { *self.bytes.get_unchecked(index - 1) } as char; - if previous != '[' { - array_len += 1; + // 2) track nesting and detect top-level elements + match b { + // entering any array + b'[' => { + array_depth += 1; + if array_depth == 1 { + // this is the '[' of *our* array + saw_value = false; + } else if array_depth == 2 { + // nested array => counts as an element + saw_value = true; + } + } + + // leaving an array + b']' => { + array_depth -= 1; + if array_depth == 0 { + // done with this array + let element_count = if saw_value { commas + 1 } else { 0 }; + // advance cursor past the closing ] + self.cursor = start + index + 1; + return Some((start, index + 1, element_count)); } } - self.cursor += index + 2; - return Some((offset, index + 2, array_len)); + + // entering an object (only matters inside our array) + b'{' if array_depth > 0 => { + if array_depth == 1 && obj_depth == 0 { + // top-level object => counts as element + saw_value = true; + } + obj_depth += 1; + } + + // leaving an object + b'}' if array_depth > 0 => { + obj_depth -= 1; + } + + // a comma that really separates two top-level elements + b',' if array_depth == 1 && obj_depth == 0 => { + commas += 1; + saw_value = false; // now look for next element + } + + // any other non-whitespace at top-level marks “we saw a value” + _ if array_depth == 1 && !b.is_ascii_whitespace() => { + saw_value = true; + } + + _ => {} } + + index += 1; } - None } } @@ -187,6 +253,15 @@ mod tests { assert_eq!(1, count); } + #[test] + fn should_scan_bool_array() { + let bytes = br#"[true,false,false]"#; + let mut scanner = JsonScanner::wrap(bytes); + + let (_, _, count) = scanner.next_array().unwrap(); + assert_eq!(3, count) + } + #[test] fn should_scan_object() { let bytes = br#"{"b":{"id":1},"a":[4,5],"E":1704907109810,"c":{"id":1,"foo":{"id":2}}}"#; @@ -256,6 +331,15 @@ mod tests { assert_eq!("-541.56".as_bytes(), &bytes[offset..offset + len]); } + #[test] + fn should_scan_array_of_objects() { + let bytes = br#"[{"s":"btcusdt","a":100},{"s":"ethusdt","a":200}]"#; + let mut scanner = JsonScanner::wrap(bytes); + scanner.skip(0); + let (_, _, count) = scanner.next_array().unwrap(); + assert_eq!(2, count) + } + mod decoder { use std::str::from_utf8; diff --git a/sje/tests/array_of_objects.rs b/sje/tests/array_of_objects.rs new file mode 100644 index 0000000..5b15bf1 --- /dev/null +++ b/sje/tests/array_of_objects.rs @@ -0,0 +1,41 @@ +use sje_derive::Decoder; + +#[derive(Decoder)] +#[sje(object)] +#[allow(dead_code)] +struct Position { + #[sje(rename = "s")] + symbol: String, + #[sje(rename = "a")] + amount: u32, +} + +#[derive(Decoder)] +#[sje(object)] +#[allow(dead_code)] +struct PositionUpdate { + #[sje(rename = "t")] + timestamp: u64, + #[sje(rename = "u", decoder = true)] + updates: Vec, +} + +#[test] +fn should_decode_array_of_objects() { + let json = r#"{"t":1746699621,"u":[{"s":"btcusdt","a":100},{"s":"ethusdt","a":200}]}"#; + + let update = PositionUpdateDecoder::decode(json.as_bytes()).unwrap(); + assert_eq!(2, update.updates_count()); + + let mut positions = update.updates().into_iter(); + + let position = positions.next().unwrap(); + assert_eq!("btcusdt", position.symbol_as_str()); + assert_eq!(100, position.amount()); + + let position = positions.next().unwrap(); + assert_eq!("ethusdt", position.symbol_as_str()); + assert_eq!(200, position.amount()); + + assert!(positions.next().is_none()); +} diff --git a/sje_derive/src/lib.rs b/sje_derive/src/lib.rs index 2dc261d..e632013 100644 --- a/sje_derive/src/lib.rs +++ b/sje_derive/src/lib.rs @@ -6,8 +6,8 @@ use std::str::FromStr; use syn::parse::{Parse, ParseStream}; use syn::spanned::Spanned; use syn::{ - parse_macro_input, Data, DataEnum, DataStruct, DeriveInput, Error, Fields, Ident, LitInt, LitStr, PathArguments, - PathSegment, Token, Type, + parse_macro_input, Data, DataEnum, DataStruct, DeriveInput, Error, Fields, Ident, LitBool, LitInt, LitStr, + PathArguments, PathSegment, Token, Type, TypePath, }; #[derive(Debug, Copy, Clone)] @@ -58,6 +58,7 @@ struct SjeFieldAttribute { also_as: Option, /// offset at which value begins offset: usize, + decoder: bool, } impl Parse for SjeFieldAttribute { @@ -67,6 +68,7 @@ impl Parse for SjeFieldAttribute { let mut ty = None; let mut also_as = None; let mut offset = 0; + let mut decoder = false; while !input.is_empty() { let lookahead = input.lookahead1(); @@ -92,6 +94,10 @@ impl Parse for SjeFieldAttribute { input.parse::()?; let offset_lit: LitInt = input.parse()?; offset = offset_lit.base10_parse()?; + } else if ident == "decoder" { + input.parse::()?; + let decoder_lit: LitBool = input.parse()?; + decoder = decoder_lit.value(); } else { return Err(syn::Error::new_spanned(ident, "expected ['len' | 'rename' | 'ty']")); } @@ -111,6 +117,7 @@ impl Parse for SjeFieldAttribute { ty, also_as, offset, + decoder, }) } } @@ -346,6 +353,12 @@ fn handle_sje_object(name: &syn::Ident, data_struct: DataStruct, _sje_attr: SjeA }); let iterators = fields.iter().map(|field| { + let mut decoder = false; + if let Some(sje_attr) = field.attrs.iter().find(|attr| attr.path().is_ident("sje")) { + let sje_field = sje_attr.parse_args::().expect("unable to parse"); + decoder = sje_field.decoder + } + let field_name = &field.ident; let field_type = &field.ty; @@ -359,8 +372,9 @@ fn handle_sje_object(name: &syn::Ident, data_struct: DataStruct, _sje_attr: SjeA let array_fn_name = format_ident!("{}", field_name.as_ref().unwrap().to_string()); let iterator_name = format_ident!("{}Iter", field_name.as_ref().unwrap().to_string().to_upper_camel_case()); - let next_impl = iterator_next_impl(arg_type); - return quote! { + let next_impl = iterator_next_impl(arg_type, decoder); + + let mut code = quote! { #[derive(Debug)] pub struct #array_struct_name<'a> { bytes: &'a [u8], @@ -373,52 +387,90 @@ fn handle_sje_object(name: &syn::Ident, data_struct: DataStruct, _sje_attr: SjeA #array_struct_name { bytes: self.#array_fn_name.0, remaining: self.#array_fn_name.1 } } } + pub struct #iterator_name<'a> { + scanner: sje::scanner::JsonScanner<'a>, + remaining: usize, + } + impl ExactSizeIterator for #iterator_name<'_> { - impl From<#array_struct_name<'_>> for Vec<#arg_type> { - fn from(value: #array_struct_name) -> Self { - value.into_iter().collect() + #[inline] + fn len(&self) -> usize { + self.remaining } } + }; - impl<'a> IntoIterator for #array_struct_name<'a> { - type Item = #arg_type; - type IntoIter = #iterator_name<'a>; - - fn into_iter(self) -> Self::IntoIter { - #iterator_name { - scanner: sje::scanner::JsonScanner::wrap(self.bytes), - remaining: self.remaining + if decoder { + let arg_type_decoder = format_ident!("{}Decoder", type_to_ident(arg_type).unwrap()); + code.extend(quote! { + impl <'a> From<#array_struct_name<'a>> for Vec<#arg_type_decoder<'a>> { + fn from(value: #array_struct_name<'a>) -> Self { + value.into_iter().collect() } } - } - pub struct #iterator_name<'a> { - scanner: sje::scanner::JsonScanner<'a>, - remaining: usize, - } + impl<'a> IntoIterator for #array_struct_name<'a> { + type Item = #arg_type_decoder<'a>; + type IntoIter = #iterator_name<'a>; + fn into_iter(self) -> Self::IntoIter { + #iterator_name { + scanner: sje::scanner::JsonScanner::wrap(self.bytes), + remaining: self.remaining + } + } + } + impl <'a> Iterator for #iterator_name<'a> { + type Item = #arg_type_decoder<'a>; + #[inline] + fn next(&mut self) -> Option { + #next_impl + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + (self.remaining, Some(self.remaining)) + } + } + impl From<#array_struct_name<'_>> for Vec<#arg_type> { + fn from(value: #array_struct_name<'_>) -> Self { + value.into_iter().map(|decoder| decoder.into()).collect() + } + } + }); + } else { + code.extend(quote! { + impl From<#array_struct_name<'_>> for Vec<#arg_type> { + fn from(value: #array_struct_name) -> Self { + value.into_iter().collect() + } + } - impl Iterator for #iterator_name<'_> { - type Item = #arg_type; + impl<'a> IntoIterator for #array_struct_name<'a> { + type Item = #arg_type; + type IntoIter = #iterator_name<'a>; - #[inline] - fn next(&mut self) -> Option { - #next_impl + fn into_iter(self) -> Self::IntoIter { + #iterator_name { + scanner: sje::scanner::JsonScanner::wrap(self.bytes), + remaining: self.remaining + } + } } - #[inline] - fn size_hint(&self) -> (usize, Option) { - (self.remaining, Some(self.remaining)) - } - } + impl Iterator for #iterator_name<'_> { + type Item = #arg_type; - impl ExactSizeIterator for #iterator_name<'_> { - - #[inline] - fn len(&self) -> usize { - self.remaining + #[inline] + fn next(&mut self) -> Option { + #next_impl + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + (self.remaining, Some(self.remaining)) + } } - } - }; + }); + } + return code; } } } @@ -480,48 +532,87 @@ fn resolve_type(ty: &Type, ty_override: Option) -> syn::Result<&'static } } -fn iterator_next_impl(ty: &Type) -> proc_macro2::TokenStream { - if let Type::Tuple(tuple) = ty { - // Generate code for processing each element - let mut code = quote! {}; - let mut tuple_values = Vec::new(); +fn iterator_next_impl(ty: &Type, decoder: bool) -> proc_macro2::TokenStream { + match ty { + Type::Path(path) => { + let mut code = quote! {}; + let mut p = path.path.clone(); + if let Some(last) = p.segments.last_mut() { + // last.ident = format_ident!("{}Decoder", last.ident); + let ident = match decoder { + true => format_ident!("{}Decoder", last.ident.clone()), + false => format_ident!("{}", last.ident.clone()), + }; - code.extend(quote! { - if self.scanner.position() + 1 == self.scanner.bytes().len() { - return None; - } - self.scanner.skip(1); - let (offset, len) = self.scanner.next_tuple()?; - let mut tuple_scanner = unsafe { sje::scanner::JsonScanner::wrap(self.scanner.bytes().get_unchecked(offset..offset + len)) }; - }); + let last = match decoder { + true => quote! { + Some(#ident::decode(bytes).unwrap()) + }, + false => quote! { + let s = unsafe { std::str::from_utf8_unchecked(bytes) }; + Some(#ident::from_str(s).unwrap()) + }, + }; - // Iterate over the tuple elements and generate code for each element - for (i, _) in tuple.elems.iter().enumerate() { - // Dynamically generate a variable name based on the index - let var_name = format_ident!("val_{i}"); + code.extend(quote! { + if self.scanner.position() + 1 == self.scanner.bytes().len() { + return None; + } + self.scanner.skip(1); + let (offset, len) = self.scanner.next_object()?; + self.remaining -= 1; + + let bytes = &self.scanner.bytes()[offset..offset + len]; + let bytes = unsafe { std::slice::from_raw_parts(bytes.as_ptr(), bytes.len()) }; + #last + // Some(#ident::decode(bytes).unwrap()) + }); + } + code + } + Type::Tuple(tuple) => { + // Generate code for processing each element + let mut code = quote! {}; + let mut tuple_values = Vec::new(); - // Generate the code for processing this element code.extend(quote! { - tuple_scanner.skip(1); - let (offset, len) = tuple_scanner.next_string()?; - let str = unsafe { std::str::from_utf8_unchecked(tuple_scanner.bytes().get_unchecked(offset..offset + len)) }; - let #var_name = str.parse().unwrap(); + if self.scanner.position() + 1 == self.scanner.bytes().len() { + return None; + } + self.scanner.skip(1); + let (offset, len) = self.scanner.next_tuple()?; + let mut tuple_scanner = unsafe { sje::scanner::JsonScanner::wrap(self.scanner.bytes().get_unchecked(offset..offset + len)) }; }); - // Add the variable to the tuple values vector for dynamic construction - tuple_values.push(quote! { #var_name }); - } + // Iterate over the tuple elements and generate code for each element + for (i, _) in tuple.elems.iter().enumerate() { + // Dynamically generate a variable name based on the index + let var_name = format_ident!("val_{i}"); + + // Generate the code for processing this element + code.extend(quote! { + tuple_scanner.skip(1); + let (offset, len) = tuple_scanner.next_string()?; + let str = unsafe { std::str::from_utf8_unchecked(tuple_scanner.bytes().get_unchecked(offset..offset + len)) }; + let #var_name = str.parse().unwrap(); + }); - // Combine the generated code and the `Some(...)` expression - code.extend(quote! { - self.remaining -= 1; - Some((#(#tuple_values),*)) - }); + // Add the variable to the tuple values vector for dynamic construction + tuple_values.push(quote! { #var_name }); + } - code - } else { - // If it's not a tuple, return an empty TokenStream - quote! {} + // Combine the generated code and the `Some(...)` expression + code.extend(quote! { + self.remaining -= 1; + Some((#(#tuple_values),*)) + }); + + code + } + _ => { + // If it's not a tuple, return an empty TokenStream + quote! {} + } } } @@ -538,6 +629,17 @@ fn is_integer_type(ty: &Type) -> bool { false } +/// Try to extract the bare `Ident` from a `&Type::Path`. +fn type_to_ident(ty: &Type) -> Option { + if let Type::Path(TypePath { qself: None, path }) = ty { + // if it's something like `Foo` or `my::crate::Bar`, + // `.segments.last()` is the `Bar` segment + path.segments.last().map(|seg| seg.ident.clone()) + } else { + None + } +} + #[cfg(test)] mod tests { use syn::{parse_quote, parse_str, Attribute};