From cb99f5c921afaa80951088c9fa05a6533251275e Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 14 Mar 2023 14:15:35 +0200 Subject: [PATCH 01/43] Take notification configs --- client/network/src/protocol.rs | 47 +++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index e7214d814dda8..5ddf3e4d7dee7 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -158,33 +158,38 @@ impl Protocol { sc_peerset::Peerset::from_config(sc_peerset::PeersetConfig { sets }) }; - let behaviour = { - Notifications::new( - peerset, - // NOTE: Block announcement protocol is still very much hardcoded into `Protocol`. - // This protocol must be the first notification protocol given to - // `Notifications` - iter::once(notifications::ProtocolConfig { - name: block_announces_protocol.notifications_protocol.clone(), - fallback_names: block_announces_protocol.fallback_names.clone(), - handshake: block_announces_protocol.handshake.as_ref().unwrap().to_vec(), - max_notification_size: block_announces_protocol.max_notification_size, - }) - .chain(notification_protocols.iter().map(|s| notifications::ProtocolConfig { - name: s.notifications_protocol.clone(), - fallback_names: s.fallback_names.clone(), - handshake: s.handshake.as_ref().map_or(roles.encode(), |h| (*h).to_vec()), - max_notification_size: s.max_notification_size, - })), + let (behaviour, notification_protocols) = { + let installed_protocols = + iter::once(block_announces_protocol.notifications_protocol.clone()) + .chain(notification_protocols.iter().map(|p| p.notifications_protocol.clone())) + .collect::>(); + ( + Notifications::new( + peerset, + // NOTE: Block announcement protocol is still very much hardcoded into + // `Protocol`. This protocol must be the first notification protocol given to + // `Notifications` + iter::once(notifications::ProtocolConfig { + name: block_announces_protocol.notifications_protocol.clone(), + fallback_names: block_announces_protocol.fallback_names.clone(), + handshake: block_announces_protocol.handshake.as_ref().unwrap().to_vec(), + max_notification_size: block_announces_protocol.max_notification_size, + }) + .chain(notification_protocols.iter().map(|s| notifications::ProtocolConfig { + name: s.notifications_protocol.clone(), + fallback_names: s.fallback_names.clone(), + handshake: s.handshake.as_ref().map_or(roles.encode(), |h| (*h).to_vec()), + max_notification_size: s.max_notification_size, + })), + ), + installed_protocols, ) }; let protocol = Self { peerset_handle: peerset_handle.clone(), behaviour, - notification_protocols: iter::once(block_announces_protocol.notifications_protocol) - .chain(notification_protocols.iter().map(|s| s.notifications_protocol.clone())) - .collect(), + notification_protocols, bad_handshake_substreams: Default::default(), peers: HashMap::new(), sync_substream_validations: FuturesUnordered::new(), From 471616a792847a5f72ae9a5dff3a685183f35ecc Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 7 Mar 2023 13:16:18 +0200 Subject: [PATCH 02/43] Return `NonDefaultSetConfig` from `TransactionsHandlerPrototype::new()` --- client/network/transactions/src/lib.rs | 26 +++++++++----------------- client/service/src/builder.rs | 22 +++++++++++----------- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/client/network/transactions/src/lib.rs b/client/network/transactions/src/lib.rs index f57556d3986b0..7afdcf8bd49b6 100644 --- a/client/network/transactions/src/lib.rs +++ b/client/network/transactions/src/lib.rs @@ -121,7 +121,6 @@ impl Future for PendingTransaction { /// Prototype for a [`TransactionsHandler`]. pub struct TransactionsHandlerPrototype { protocol_name: ProtocolName, - fallback_protocol_names: Vec, } impl TransactionsHandlerPrototype { @@ -130,26 +129,17 @@ impl TransactionsHandlerPrototype { protocol_id: ProtocolId, genesis_hash: Hash, fork_id: Option<&str>, - ) -> Self { + ) -> (Self, NonDefaultSetConfig) { let genesis_hash = genesis_hash.as_ref(); - let protocol_name = if let Some(fork_id) = fork_id { + let protocol_name: ProtocolName = if let Some(fork_id) = fork_id { format!("/{}/{}/transactions/1", array_bytes::bytes2hex("", genesis_hash), fork_id) } else { format!("/{}/transactions/1", array_bytes::bytes2hex("", genesis_hash)) - }; - let legacy_protocol_name = format!("/{}/transactions/1", protocol_id.as_ref()); - - Self { - protocol_name: protocol_name.into(), - fallback_protocol_names: iter::once(legacy_protocol_name.into()).collect(), } - } - - /// Returns the configuration of the set to put in the network configuration. - pub fn set_config(&self) -> NonDefaultSetConfig { - NonDefaultSetConfig { - notifications_protocol: self.protocol_name.clone(), - fallback_names: self.fallback_protocol_names.clone(), + .into(); + let config = NonDefaultSetConfig { + notifications_protocol: protocol_name.clone(), + fallback_names: vec![format!("/{}/transactions/1", protocol_id.as_ref()).into()], max_notification_size: MAX_TRANSACTIONS_SIZE, handshake: None, set_config: SetConfig { @@ -158,7 +148,9 @@ impl TransactionsHandlerPrototype { reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - } + }; + + (Self { protocol_name }, config) } /// Turns the prototype into the actual handler. Returns a controller that allows controlling diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 5237a6166b012..24d4422f852ba 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -883,17 +883,17 @@ where } // create transactions protocol and add it to the list of supported protocols of - // `network_params` - let transactions_handler_proto = sc_network_transactions::TransactionsHandlerPrototype::new( - protocol_id.clone(), - client - .block_hash(0u32.into()) - .ok() - .flatten() - .expect("Genesis block exists; qed"), - config.chain_spec.fork_id(), - ); - net_config.add_notification_protocol(transactions_handler_proto.set_config()); + let (transactions_handler_proto, transactions_config) = + sc_network_transactions::TransactionsHandlerPrototype::new( + protocol_id.clone(), + client + .block_hash(0u32.into()) + .ok() + .flatten() + .expect("Genesis block exists; qed"), + config.chain_spec.fork_id(), + ); + net_config.add_notification_protocol(transactions_config); let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); From c456edd43761f8744712f3df15ff4f6de810fb6d Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 14 Mar 2023 16:37:12 +0200 Subject: [PATCH 03/43] Make fields of `NonDefaultSetConfig` private --- .../consensus/beefy/src/communication/mod.rs | 8 ++- client/consensus/grandpa/src/lib.rs | 14 ++--- client/network/src/config.rs | 56 +++++++++++++------ client/network/src/protocol.rs | 35 ++++++------ client/network/src/service.rs | 4 +- client/network/statement/src/lib.rs | 14 ++--- client/network/sync/src/engine.rs | 4 +- client/network/sync/src/lib.rs | 22 +++----- client/network/test/src/lib.rs | 40 ++++++++++--- client/network/test/src/service.rs | 28 +++++----- client/network/transactions/src/lib.rs | 14 ++--- 11 files changed, 144 insertions(+), 95 deletions(-) diff --git a/client/consensus/beefy/src/communication/mod.rs b/client/consensus/beefy/src/communication/mod.rs index 0de67f6062339..45800392c532e 100644 --- a/client/consensus/beefy/src/communication/mod.rs +++ b/client/consensus/beefy/src/communication/mod.rs @@ -68,7 +68,13 @@ pub(crate) mod beefy_protocol_name { pub fn beefy_peers_set_config( gossip_protocol_name: sc_network::ProtocolName, ) -> sc_network::config::NonDefaultSetConfig { - let mut cfg = sc_network::config::NonDefaultSetConfig::new(gossip_protocol_name, 1024 * 1024); + let mut cfg = sc_network::config::NonDefaultSetConfig::new( + gossip_protocol_name, + Vec::new(), + 1024 * 1024, + None, + Default::default(), + ); cfg.allow_non_reserved(25, 25); cfg } diff --git a/client/consensus/grandpa/src/lib.rs b/client/consensus/grandpa/src/lib.rs index c11e873eca738..e75ceb02a31b2 100644 --- a/client/consensus/grandpa/src/lib.rs +++ b/client/consensus/grandpa/src/lib.rs @@ -697,19 +697,19 @@ pub fn grandpa_peers_set_config( protocol_name: ProtocolName, ) -> sc_network::config::NonDefaultSetConfig { use communication::grandpa_protocol_name; - sc_network::config::NonDefaultSetConfig { - notifications_protocol: protocol_name, - fallback_names: grandpa_protocol_name::LEGACY_NAMES.iter().map(|&n| n.into()).collect(), + sc_network::config::NonDefaultSetConfig::new( + protocol_name, + grandpa_protocol_name::LEGACY_NAMES.iter().map(|&n| n.into()).collect(), // Notifications reach ~256kiB in size at the time of writing on Kusama and Polkadot. - max_notification_size: 1024 * 1024, - handshake: None, - set_config: sc_network::config::SetConfig { + 1024 * 1024, + None, + sc_network::config::SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: sc_network::config::NonReservedPeerMode::Deny, }, - } + ) } /// Run a GRANDPA voter as a task. Provide configuration and a link to a diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 17ca8335653de..1649dfbdf2fe5 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -487,7 +487,7 @@ pub struct NonDefaultSetConfig { /// /// > **Note**: This field isn't present for the default set, as this is handled internally /// > by the networking code. - pub notifications_protocol: ProtocolName, + protocol_name: ProtocolName, /// If the remote reports that it doesn't support the protocol indicated in the /// `notifications_protocol` field, then each of these fallback names will be tried one by @@ -495,37 +495,57 @@ pub struct NonDefaultSetConfig { /// /// If a fallback is used, it will be reported in /// `sc_network::protocol::event::Event::NotificationStreamOpened::negotiated_fallback` - pub fallback_names: Vec, + fallback_names: Vec, /// Handshake of the protocol /// /// NOTE: Currently custom handshakes are not fully supported. See issue #5685 for more /// details. This field is temporarily used to allow moving the hardcoded block announcement /// protocol out of `protocol.rs`. - pub handshake: Option, + handshake: Option, /// Maximum allowed size of single notifications. - pub max_notification_size: u64, + max_notification_size: u64, /// Base configuration. - pub set_config: SetConfig, + set_config: SetConfig, } impl NonDefaultSetConfig { /// Creates a new [`NonDefaultSetConfig`]. Zero slots and accepts only reserved nodes. - pub fn new(notifications_protocol: ProtocolName, max_notification_size: u64) -> Self { - Self { - notifications_protocol, - max_notification_size, - fallback_names: Vec::new(), - handshake: None, - set_config: SetConfig { - in_peers: 0, - out_peers: 0, - reserved_nodes: Vec::new(), - non_reserved_mode: NonReservedPeerMode::Deny, - }, - } + pub fn new( + protocol_name: ProtocolName, + fallback_names: Vec, + max_notification_size: u64, + handshake: Option, + set_config: SetConfig, + ) -> Self { + Self { protocol_name, max_notification_size, fallback_names, handshake, set_config } + } + + /// Get reference to protocol name. + pub fn protocol_name(&self) -> &ProtocolName { + &self.protocol_name + } + + /// Get refer + pub fn fallback_names(&self) -> impl Iterator { + self.fallback_names.iter() + } + + /// Get reference to handshake. + pub fn handshake(&self) -> &Option { + &self.handshake + } + + /// Get maximum notification size. + pub fn max_notification_size(&self) -> u64 { + self.max_notification_size + } + + /// Get reference to `SetConfig`. + pub fn set_config(&self) -> &SetConfig { + &self.set_config } /// Modifies the configuration to allow non-reserved nodes. diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 5ddf3e4d7dee7..95592edd7e693 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -138,17 +138,17 @@ impl Protocol { for set_cfg in ¬ification_protocols { let mut reserved_nodes = HashSet::new(); - for reserved in set_cfg.set_config.reserved_nodes.iter() { + for reserved in set_cfg.set_config().reserved_nodes.iter() { reserved_nodes.insert(reserved.peer_id); known_addresses.push((reserved.peer_id, reserved.multiaddr.clone())); } let reserved_only = - set_cfg.set_config.non_reserved_mode == NonReservedPeerMode::Deny; + set_cfg.set_config().non_reserved_mode == NonReservedPeerMode::Deny; sets.push(sc_peerset::SetConfig { - in_peers: set_cfg.set_config.in_peers, - out_peers: set_cfg.set_config.out_peers, + in_peers: set_cfg.set_config().in_peers, + out_peers: set_cfg.set_config().out_peers, bootnodes: Vec::new(), reserved_nodes, reserved_only, @@ -159,10 +159,10 @@ impl Protocol { }; let (behaviour, notification_protocols) = { - let installed_protocols = - iter::once(block_announces_protocol.notifications_protocol.clone()) - .chain(notification_protocols.iter().map(|p| p.notifications_protocol.clone())) - .collect::>(); + let installed_protocols = iter::once(block_announces_protocol.protocol_name().clone()) + .chain(notification_protocols.iter().map(|p| p.protocol_name().clone())) + .collect::>(); + ( Notifications::new( peerset, @@ -170,16 +170,19 @@ impl Protocol { // `Protocol`. This protocol must be the first notification protocol given to // `Notifications` iter::once(notifications::ProtocolConfig { - name: block_announces_protocol.notifications_protocol.clone(), - fallback_names: block_announces_protocol.fallback_names.clone(), - handshake: block_announces_protocol.handshake.as_ref().unwrap().to_vec(), - max_notification_size: block_announces_protocol.max_notification_size, + name: block_announces_protocol.protocol_name().clone(), + fallback_names: block_announces_protocol + .fallback_names() + .cloned() + .collect(), + handshake: block_announces_protocol.handshake().as_ref().unwrap().to_vec(), + max_notification_size: block_announces_protocol.max_notification_size(), }) .chain(notification_protocols.iter().map(|s| notifications::ProtocolConfig { - name: s.notifications_protocol.clone(), - fallback_names: s.fallback_names.clone(), - handshake: s.handshake.as_ref().map_or(roles.encode(), |h| (*h).to_vec()), - max_notification_size: s.max_notification_size, + name: s.protocol_name().clone(), + fallback_names: s.fallback_names().cloned().collect(), + handshake: s.handshake().as_ref().map_or(roles.encode(), |h| (*h).to_vec()), + max_notification_size: s.max_notification_size(), })), ), installed_protocols, diff --git a/client/network/src/service.rs b/client/network/src/service.rs index cd8e18a2e7d9f..2574d0cbc8d7e 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -193,7 +193,7 @@ where )?; for notification_protocol in ¬ification_protocols { ensure_addresses_consistent_with_transport( - notification_protocol.set_config.reserved_nodes.iter().map(|x| &x.multiaddr), + notification_protocol.set_config().reserved_nodes.iter().map(|x| &x.multiaddr), &network_config.transport, )?; } @@ -235,7 +235,7 @@ where .map(|cfg| usize::try_from(cfg.max_response_size).unwrap_or(usize::MAX)); let notifs_max = notification_protocols .iter() - .map(|cfg| usize::try_from(cfg.max_notification_size).unwrap_or(usize::MAX)); + .map(|cfg| usize::try_from(cfg.max_notification_size()).unwrap_or(usize::MAX)); // A "default" max is added to cover all the other protocols: ping, identify, // kademlia, block announces, and transactions. diff --git a/client/network/statement/src/lib.rs b/client/network/statement/src/lib.rs index 02cbab27a6a15..6520154b6ed22 100644 --- a/client/network/statement/src/lib.rs +++ b/client/network/statement/src/lib.rs @@ -120,18 +120,18 @@ impl StatementHandlerPrototype { /// Returns the configuration of the set to put in the network configuration. pub fn set_config(&self) -> NonDefaultSetConfig { - NonDefaultSetConfig { - notifications_protocol: self.protocol_name.clone(), - fallback_names: Vec::new(), - max_notification_size: MAX_STATEMENT_SIZE, - handshake: None, - set_config: SetConfig { + NonDefaultSetConfig::new( + self.protocol_name.clone(), + Vec::new(), + MAX_STATEMENT_SIZE, + None, + SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - } + ) } /// Turns the prototype into the actual handler. diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index a6db5a5d54c8c..6268275ba1d94 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -300,7 +300,7 @@ where } for config in net_config.notification_protocols() { let peer_ids = config - .set_config + .set_config() .reserved_nodes .iter() .map(|info| info.peer_id) @@ -356,7 +356,7 @@ where warp_sync_protocol_name, )?; - let block_announce_protocol_name = block_announce_config.notifications_protocol.clone(); + let block_announce_protocol_name = block_announce_config.protocol_name().clone(); let (tx, service_rx) = tracing_unbounded("mpsc_chain_sync", 100_000); let num_connected = Arc::new(AtomicUsize::new(0)); let is_major_syncing = Arc::new(AtomicBool::new(false)); diff --git a/client/network/sync/src/lib.rs b/client/network/sync/src/lib.rs index fbdb275a444cd..68b834bc46f31 100644 --- a/client/network/sync/src/lib.rs +++ b/client/network/sync/src/lib.rs @@ -1458,10 +1458,7 @@ where state_request_protocol_name, warp_sync_params, warp_sync_protocol_name, - block_announce_protocol_name: block_announce_config - .notifications_protocol - .clone() - .into(), + block_announce_protocol_name: block_announce_config.protocol_name().clone().into(), pending_responses: HashMap::new(), import_queue, metrics: if let Some(r) = &metrics_registry { @@ -1974,14 +1971,11 @@ where } }; - NonDefaultSetConfig { - notifications_protocol: block_announces_protocol.into(), - fallback_names: iter::once( - format!("/{}/block-announces/1", protocol_id.as_ref()).into(), - ) - .collect(), - max_notification_size: MAX_BLOCK_ANNOUNCE_SIZE, - handshake: Some(NotificationHandshake::new(BlockAnnouncesHandshake::::build( + NonDefaultSetConfig::new( + block_announces_protocol.into(), + iter::once(format!("/{}/block-announces/1", protocol_id.as_ref()).into()).collect(), + MAX_BLOCK_ANNOUNCE_SIZE, + Some(NotificationHandshake::new(BlockAnnouncesHandshake::::build( roles, best_number, best_hash, @@ -1989,13 +1983,13 @@ where ))), // NOTE: `set_config` will be ignored by `protocol.rs` as the block announcement // protocol is still hardcoded into the peerset. - set_config: SetConfig { + SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - } + ) } fn decode_block_response(response: &[u8]) -> Result { diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index a9ff38e4ea608..6f5212b3d4d1f 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -800,6 +800,32 @@ where network_config.transport = TransportConfig::MemoryOnly; network_config.listen_addresses = vec![listen_addr.clone()]; network_config.allow_non_globals_in_dht = true; + // <<<<<<< HEAD + // ||||||| parent of 7a8fd6570a (Make fields of `NonDefaultSetConfig` private) + // network_config + // .request_response_protocols + // .extend(config.request_response_protocols); + // network_config.extra_sets = config + // .notifications_protocols + // .into_iter() + // .map(|p| NonDefaultSetConfig { + // notifications_protocol: p, + // fallback_names: Vec::new(), + // max_notification_size: 1024 * 1024, + // handshake: None, + // set_config: Default::default(), + // }) + // .collect(); + // ======= + // network_config + // .request_response_protocols + // .extend(config.request_response_protocols); + // network_config.extra_sets = config + // .notifications_protocols + // .into_iter() + // .map(|p| NonDefaultSetConfig::new(p, Vec::new(), 1024 * 1024, None, Default::default())) + // .collect(); + // >>>>>>> 7a8fd6570a (Make fields of `NonDefaultSetConfig` private) if let Some(connect_to) = config.connect_to_peers { let addrs = connect_to .iter() @@ -906,13 +932,13 @@ where } for protocol in config.notifications_protocols { - full_net_config.add_notification_protocol(NonDefaultSetConfig { - notifications_protocol: protocol, - fallback_names: Vec::new(), - max_notification_size: 1024 * 1024, - handshake: None, - set_config: Default::default(), - }); + full_net_config.add_notification_protocol(NonDefaultSetConfig::new( + protocol, + Vec::new(), + 1024 * 1024, + None, + Default::default(), + )); } let genesis_hash = diff --git a/client/network/test/src/service.rs b/client/network/test/src/service.rs index 8c15d6b09ea45..c747565c0df71 100644 --- a/client/network/test/src/service.rs +++ b/client/network/test/src/service.rs @@ -203,13 +203,13 @@ impl TestNetworkBuilder { full_net_config.add_notification_protocol(config); } } else { - full_net_config.add_notification_protocol(config::NonDefaultSetConfig { - notifications_protocol: PROTOCOL_NAME.into(), - fallback_names: Vec::new(), - max_notification_size: 1024 * 1024, - handshake: None, - set_config: self.set_config.unwrap_or_default(), - }); + full_net_config.add_notification_protocol(config::NonDefaultSetConfig::new( + PROTOCOL_NAME.into(), + Vec::new(), + 1024 * 1024, + None, + self.set_config.unwrap_or_default(), + )); } for config in [ @@ -570,13 +570,13 @@ async fn fallback_name_working() { let listen_addr = config::build_multiaddr![Memory(rand::random::())]; let (node1, mut events_stream1) = TestNetworkBuilder::new() - .with_notification_protocol(config::NonDefaultSetConfig { - notifications_protocol: NEW_PROTOCOL_NAME.into(), - fallback_names: vec![PROTOCOL_NAME.into()], - max_notification_size: 1024 * 1024, - handshake: None, - set_config: Default::default(), - }) + .with_notification_protocol(config::NonDefaultSetConfig::new( + NEW_PROTOCOL_NAME.into(), + vec![PROTOCOL_NAME.into()], + 1024 * 1024, + None, + Default::default(), + )) .with_config(config::NetworkConfiguration { listen_addresses: vec![listen_addr.clone()], transport: TransportConfig::MemoryOnly, diff --git a/client/network/transactions/src/lib.rs b/client/network/transactions/src/lib.rs index 7afdcf8bd49b6..3ebc7ce74676f 100644 --- a/client/network/transactions/src/lib.rs +++ b/client/network/transactions/src/lib.rs @@ -137,18 +137,18 @@ impl TransactionsHandlerPrototype { format!("/{}/transactions/1", array_bytes::bytes2hex("", genesis_hash)) } .into(); - let config = NonDefaultSetConfig { - notifications_protocol: protocol_name.clone(), - fallback_names: vec![format!("/{}/transactions/1", protocol_id.as_ref()).into()], - max_notification_size: MAX_TRANSACTIONS_SIZE, - handshake: None, - set_config: SetConfig { + let config = NonDefaultSetConfig::new( + protocol_name.clone(), + vec![format!("/{}/transactions/1", protocol_id.as_ref()).into()], + MAX_TRANSACTIONS_SIZE, + None, + SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - }; + ); (Self { protocol_name }, config) } From 5593a5317be360d99f366d0f91019a4e3e9f1aef Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 14 Mar 2023 16:41:36 +0200 Subject: [PATCH 04/43] Introduce `NotificationService` --- client/network/src/lib.rs | 2 +- client/network/src/service/traits.rs | 119 +++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index a66c187cacf7b..22d7040e04172 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -274,7 +274,7 @@ pub use service::{ KademliaKey, NetworkBlock, NetworkDHTProvider, NetworkEventStream, NetworkNotification, NetworkPeers, NetworkRequest, NetworkSigner, NetworkStateInfo, NetworkStatus, NetworkStatusProvider, NetworkSyncForkRequest, NotificationSender as NotificationSenderT, - NotificationSenderError, NotificationSenderReady, + NotificationSenderError, NotificationSenderReady, NotificationService, }, DecodingError, Keypair, NetworkService, NetworkWorker, NotificationSender, NotificationsSink, OutboundFailure, PublicKey, diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 787ef4b5ae445..3cc3540128454 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -632,3 +632,122 @@ where T::new_best_block_imported(self, hash, number) } } + +/// Substream acceptance result. +pub enum ValidationResult { + /// Accept inbound substream. + Accept, + + /// Reject inbound substream. + Reject, +} + +/// Events received by the protocol from `Notifications`. +pub enum NotificationEvent { + /// Validate inbound substream. + ValidateInboundSubstream { + /// Peer ID. + peer: PeerId, + + /// Received handshake. + handshake: Vec, + + /// `oneshot::Sender` for sending validation result back to `Notifications` + result_tx: oneshot::Sender, + }, + + /// Remote identified by `PeerId` opened a substream and sent `Handshake`. + /// Validate `Handshake` and report status (accept/reject) to `Notifications`. + NotificationStreamOpened { + /// Peer ID. + peer: PeerId, + + /// Role of the peer. + role: ObservedRole, + + /// Negotiated fallback. + negotiated_fallback: Option, + }, + + /// Substream was closed. + NotificationStreamClosed { + /// Peer Id. + peer: PeerId, + }, + + /// Notification was received from the substream. + NotificationReceived { + /// Peer IDJ. + peer: PeerId, + + /// Received notification. + notification: Vec, + }, +} + +/// Notification service +/// +/// Defines traits that specify the behavior both protocol implementations and `Notifications` can +/// expect from each other. +/// +/// `Notifications` can send two different kinds of information to protocol: +/// * substream-related information +/// * notification-related information +/// +/// When an unvalidated, inbound substream is received by `Notifications`, it sends the inbound +/// stream information (peer ID, handshake) to protocol for validation. Protocol must then verify +/// that the handshake is valid (and in the future that it has a slot it can allocate for the peer) +/// and then report back `ValidationResult` which is either `Accept` or `Reject`. +/// +/// After the validation result has been received by `Notifications`, it prepares the +/// substream for communication by initializing the necessary sinks and emits +/// `NotificationStreamOpened` which informs the protocol that the remote peer is ready to receive +/// notifications. +/// +/// Two different flavors of sending options are provided: +/// * synchronous sending ([`ProtocolNotificationService::send_sync_notification()`]) +/// * asynchronous sending ([`ProtocolNotificationService::send_async_notification()`]) +/// +/// The former is used by protocols that don't have the implementation ready for exercising +/// backpressure and the latter for the protocols that can do it. +/// +/// Both local and remote peer can close the substream at any time. Local peer can do so by calling +/// [`ProtocolNotificationService::close_substream()`] which instrucs `Notifications` to close the +/// substream. Remote closing the substream is indicated to the local peer by receiving +/// `NotificationEvent::SubstreamClosed` event +/// +/// In case the protocol must update its handshake while it's operating (such as updating the best +/// block information), it can do so by calling [`ProtocolNotificationService::set_handshake()`] +/// which instructs `Notifications` to update the handshake it stored during protocol +/// initialization. +/// +/// All peer events are multiplexed on the same incoming event stream from `Notifications` and thus +/// each event carries a `PeerId` so the protocol knows whose information to update when receiving +/// an event. +#[async_trait::async_trait] +pub trait NotificationService: Debug + Send { + /// Instruct `Notifications` to open a new substream for `peer`. + /// + /// `dial_if_disconnected` informs `Notifications` whether to dial + // the peer if there is currently no active connection to it. + async fn open_substream(&mut self, peer: PeerId) -> Result<(), ()>; + + /// Instruct `Notifications` to close substream for `peer`. + async fn close_substream(&mut self, peer: PeerId) -> Result<(), ()>; + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification(&mut self, peer: PeerId, notification: Vec) -> Result<(), ()>; + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + async fn send_async_notification( + &mut self, + peer: PeerId, + notification: Vec, + ) -> Result<(), ()>; + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_hanshake(&mut self, handshake: Vec) -> Result<(), ()>; + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option; +} From 22cd46c08358b899461f2d6b30d8258186365430 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 14 Mar 2023 17:07:25 +0200 Subject: [PATCH 05/43] Add implementation of `NotificationService` --- Cargo.lock | 1 + bin/node-template/node/src/service.rs | 6 +- bin/node/cli/src/service.rs | 27 +- .../consensus/beefy/src/communication/mod.rs | 6 +- client/consensus/grandpa/src/lib.rs | 4 +- client/network/Cargo.toml | 2 + client/network/common/src/role.rs | 2 +- client/network/src/config.rs | 30 +- client/network/src/error.rs | 9 + client/network/src/protocol.rs | 4 +- client/network/src/protocol/notifications.rs | 2 + .../src/protocol/notifications/handler.rs | 30 +- .../src/protocol/notifications/service.rs | 706 ++++++++++++++++++ client/network/src/service/traits.rs | 20 +- client/network/statement/src/lib.rs | 26 +- client/network/sync/src/engine.rs | 10 +- client/network/sync/src/lib.rs | 25 +- client/network/test/src/lib.rs | 66 +- client/network/test/src/service.rs | 20 +- client/network/transactions/src/lib.rs | 13 +- 20 files changed, 901 insertions(+), 108 deletions(-) create mode 100644 client/network/src/protocol/notifications/service.rs diff --git a/Cargo.lock b/Cargo.lock index f387b5ef95227..6464d6ffe3a83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9462,6 +9462,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tokio-stream", "tokio-test", "tokio-util", "unsigned-varint", diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index ca827001b5bcc..3a0f5ee08fcca 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -156,9 +156,9 @@ pub fn new_full(config: Configuration) -> Result { &client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"), &config.chain_spec, ); - net_config.add_notification_protocol(sc_consensus_grandpa::grandpa_peers_set_config( - grandpa_protocol_name.clone(), - )); + let (grandpa_protocol_config, grandpa_notification_handle) = + sc_consensus_grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone()); + net_config.add_notification_protocol(grandpa_protocol_config); let warp_sync = Arc::new(sc_consensus_grandpa::warp_proof::NetworkProvider::new( backend.clone(), diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 8fc44c7c5eddf..5c99a14627bfc 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -349,19 +349,20 @@ pub fn new_full_base( &client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"), &config.chain_spec, ); - net_config.add_notification_protocol(grandpa::grandpa_peers_set_config( - grandpa_protocol_name.clone(), - )); - - let statement_handler_proto = sc_network_statement::StatementHandlerPrototype::new( - client - .block_hash(0u32.into()) - .ok() - .flatten() - .expect("Genesis block exists; qed"), - config.chain_spec.fork_id(), - ); - net_config.add_notification_protocol(statement_handler_proto.set_config()); + let (grandpa_protocol_config, grandpa_notification_handle) = + grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone()); + net_config.add_notification_protocol(grandpa_protocol_config); + + let (statement_handler_proto, statement_config) = + sc_network_statement::StatementHandlerPrototype::new( + client + .block_hash(0u32.into()) + .ok() + .flatten() + .expect("Genesis block exists; qed"), + config.chain_spec.fork_id(), + ); + net_config.add_notification_protocol(statement_config); let warp_sync = Arc::new(grandpa::warp_proof::NetworkProvider::new( backend.clone(), diff --git a/client/consensus/beefy/src/communication/mod.rs b/client/consensus/beefy/src/communication/mod.rs index 45800392c532e..4f2ae69df94f1 100644 --- a/client/consensus/beefy/src/communication/mod.rs +++ b/client/consensus/beefy/src/communication/mod.rs @@ -67,8 +67,8 @@ pub(crate) mod beefy_protocol_name { /// For standard protocol name see [`beefy_protocol_name::gossip_protocol_name`]. pub fn beefy_peers_set_config( gossip_protocol_name: sc_network::ProtocolName, -) -> sc_network::config::NonDefaultSetConfig { - let mut cfg = sc_network::config::NonDefaultSetConfig::new( +) -> (sc_network::config::NonDefaultSetConfig, Box) { + let (mut cfg, notification_handle) = sc_network::config::NonDefaultSetConfig::new( gossip_protocol_name, Vec::new(), 1024 * 1024, @@ -76,7 +76,7 @@ pub fn beefy_peers_set_config( Default::default(), ); cfg.allow_non_reserved(25, 25); - cfg + (cfg, notification_handle) } // cost scalars for reporting peers. diff --git a/client/consensus/grandpa/src/lib.rs b/client/consensus/grandpa/src/lib.rs index e75ceb02a31b2..90651c3c22cd6 100644 --- a/client/consensus/grandpa/src/lib.rs +++ b/client/consensus/grandpa/src/lib.rs @@ -68,7 +68,7 @@ use sc_client_api::{ StorageProvider, TransactionFor, }; use sc_consensus::BlockImport; -use sc_network::types::ProtocolName; +use sc_network::{types::ProtocolName, NotificationService}; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; use sp_api::ProvideRuntimeApi; @@ -695,7 +695,7 @@ pub struct GrandpaParams { /// For standard protocol name see [`crate::protocol_standard_name`]. pub fn grandpa_peers_set_config( protocol_name: ProtocolName, -) -> sc_network::config::NonDefaultSetConfig { +) -> (sc_network::config::NonDefaultSetConfig, Box) { use communication::grandpa_protocol_name; sc_network::config::NonDefaultSetConfig::new( protocol_name, diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index de4c4c14a2587..f2f1e5b2ea5fd 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -37,6 +37,8 @@ serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.85" smallvec = "1.8.0" thiserror = "1.0" +tokio = { version = "1.22.0", features = ["macros", "sync"] } +tokio-stream = "0.1.12" unsigned-varint = { version = "0.7.1", features = ["futures", "asynchronous_codec"] } zeroize = "1.4.3" prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.10.0-dev", path = "../../utils/prometheus" } diff --git a/client/network/common/src/role.rs b/client/network/common/src/role.rs index cd43f6655b72c..a6364daa69829 100644 --- a/client/network/common/src/role.rs +++ b/client/network/common/src/role.rs @@ -24,7 +24,7 @@ use codec::{self, Encode, EncodeLike, Input, Output}; /// > **Note**: This enum is different from the `Role` enum. The `Role` enum indicates what a /// > node says about itself, while `ObservedRole` is a `Role` merged with the /// > information known locally about that node. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum ObservedRole { /// Full node. Full, diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 1649dfbdf2fe5..d105e803bcf0b 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -22,10 +22,11 @@ //! See the documentation of [`Params`]. pub use crate::{ - protocol::NotificationsSink, + protocol::{notification_service, NotificationsSink, ProtocolHandlePair}, request_responses::{ IncomingRequest, OutgoingResponse, ProtocolConfig as RequestResponseConfig, }, + service::traits::NotificationService, types::ProtocolName, }; @@ -480,7 +481,7 @@ impl Default for SetConfig { /// /// > **Note**: As new fields might be added in the future, please consider using the `new` method /// > and modifiers instead of creating this struct manually. -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct NonDefaultSetConfig { /// Name of the notifications protocols of this set. A substream on this set will be /// considered established once this protocol is open. @@ -509,18 +510,39 @@ pub struct NonDefaultSetConfig { /// Base configuration. set_config: SetConfig, + + /// Notification handle. + /// + /// Notification handle is created during `NonDefaultSetConfig` creation and its other half, + /// Box` is given to the protocol created the config and + /// `ProtocolHandle` is given to `Notifications` when it initializes itself. This handle allows + /// `Notifications ` to communicate with the protocol directly without relaying events through + /// `sc-network.` + protocol_handle_pair: ProtocolHandlePair, } impl NonDefaultSetConfig { /// Creates a new [`NonDefaultSetConfig`]. Zero slots and accepts only reserved nodes. + /// Also returns an object which allows the protocol to communicate with `Notifications`. pub fn new( protocol_name: ProtocolName, fallback_names: Vec, max_notification_size: u64, handshake: Option, set_config: SetConfig, - ) -> Self { - Self { protocol_name, max_notification_size, fallback_names, handshake, set_config } + ) -> (Self, Box) { + let (protocol_handle_pair, notification_handle) = notification_service(); + ( + Self { + protocol_name, + max_notification_size, + fallback_names, + handshake, + set_config, + protocol_handle_pair, + }, + notification_handle, + ) } /// Get reference to protocol name. diff --git a/client/network/src/error.rs b/client/network/src/error.rs index f0828fb821f35..01e8356fb5535 100644 --- a/client/network/src/error.rs +++ b/client/network/src/error.rs @@ -68,6 +68,15 @@ pub enum Error { /// Name of the protocol registered multiple times. protocol: ProtocolName, }, + /// Peer does not exist. + #[error("Peer `{0}` does not exist.")] + PeerDoesntExist(PeerId), + /// Channel closed. + #[error("Channel closed")] + ChannelClosed, + /// Connection closed. + #[error("Connection closed")] + ConnectionClosed, } // Make `Debug` use the `Display` implementation. diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 95592edd7e693..ef29058fff67b 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -50,7 +50,9 @@ use std::{ use message::{generic::Message as GenericMessage, Message}; use notifications::{Notifications, NotificationsOut}; -pub use notifications::{NotificationsSink, NotifsHandlerError, Ready}; +pub use notifications::{ + notification_service, NotificationsSink, NotifsHandlerError, ProtocolHandlePair, Ready, +}; mod notifications; diff --git a/client/network/src/protocol/notifications.rs b/client/network/src/protocol/notifications.rs index aa49cfcf9d44e..1a53df2cdf781 100644 --- a/client/network/src/protocol/notifications.rs +++ b/client/network/src/protocol/notifications.rs @@ -22,9 +22,11 @@ pub use self::{ behaviour::{Notifications, NotificationsOut, ProtocolConfig}, handler::{NotificationsSink, NotifsHandlerError, Ready}, + service::{notification_service, ProtocolHandlePair}, }; mod behaviour; mod handler; +mod service; mod tests; mod upgrade; diff --git a/client/network/src/protocol/notifications/handler.rs b/client/network/src/protocol/notifications/handler.rs index 665b646ecdcfa..260a38557421a 100644 --- a/client/network/src/protocol/notifications/handler.rs +++ b/client/network/src/protocol/notifications/handler.rs @@ -92,7 +92,7 @@ use std::{ /// Number of pending notifications in asynchronous contexts. /// See [`NotificationsSink::reserve_notification`] for context. -const ASYNC_NOTIFICATIONS_BUFFER_SIZE: usize = 8; +pub(crate) const ASYNC_NOTIFICATIONS_BUFFER_SIZE: usize = 8; /// Number of pending notifications in synchronous contexts. const SYNC_NOTIFICATIONS_BUFFER_SIZE: usize = 2048; @@ -329,6 +329,29 @@ pub struct NotificationsSink { inner: Arc, } +#[cfg(test)] +impl NotificationsSink { + /// Create new + pub fn new( + peer_id: PeerId, + ) -> (Self, mpsc::Receiver, mpsc::Receiver) + { + let (async_tx, async_rx) = mpsc::channel(ASYNC_NOTIFICATIONS_BUFFER_SIZE); + let (sync_tx, sync_rx) = mpsc::channel(SYNC_NOTIFICATIONS_BUFFER_SIZE); + ( + NotificationsSink { + inner: Arc::new(NotificationsSinkInner { + peer_id, + async_channel: FuturesMutex::new(async_tx), + sync_channel: Mutex::new(Some(sync_tx)), + }), + }, + async_rx, + sync_rx, + ) + } +} + #[derive(Debug)] struct NotificationsSinkInner { /// Target of the sink. @@ -346,8 +369,8 @@ struct NotificationsSinkInner { /// Message emitted through the [`NotificationsSink`] and processed by the background task /// dedicated to the peer. -#[derive(Debug)] -enum NotificationsSinkMessage { +#[derive(Debug, PartialEq, Eq)] +pub enum NotificationsSinkMessage { /// Message emitted by [`NotificationsSink::reserve_notification`] and /// [`NotificationsSink::write_notification_now`]. Notification { message: Vec }, @@ -379,6 +402,7 @@ impl NotificationsSink { tx.try_send(NotificationsSinkMessage::Notification { message: message.into() }); if result.is_err() { + println!("error happened"); // Cloning the `mpsc::Sender` guarantees the allocation of an extra spot in the // buffer, and therefore `try_send` will succeed. let _result2 = tx.clone().try_send(NotificationsSinkMessage::ForceClose); diff --git a/client/network/src/protocol/notifications/service.rs b/client/network/src/protocol/notifications/service.rs new file mode 100644 index 0000000000000..26a287406e77c --- /dev/null +++ b/client/network/src/protocol/notifications/service.rs @@ -0,0 +1,706 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Notification service implementation. + +#![allow(unused)] + +use crate::{ + error, + protocol::notifications::handler::{ + NotificationsSink, NotificationsSinkMessage, ASYNC_NOTIFICATIONS_BUFFER_SIZE, + }, + service::traits::{NotificationEvent, NotificationService, ValidationResult}, + types::ProtocolName, +}; + +use futures::{stream::Stream, SinkExt, StreamExt}; +use libp2p::PeerId; +use tokio::sync::{mpsc, oneshot}; + +use sc_network_common::role::ObservedRole; + +use std::{collections::HashMap, fmt::Debug}; + +/// Inner notification event to deal with `NotificationsSinks` without exposing that +/// implementation detail to [`NotificationService`] consumers. +enum InnerNotificationEvent { + /// Validate inbound substream. + ValidateInboundSubstream { + /// Peer ID. + peer: PeerId, + + /// Received handshake. + handshake: Vec, + + /// `oneshot::Sender` for sending validation result back to `Notifications` + result_tx: oneshot::Sender, + }, + + /// Remote identified by `PeerId` opened a substream and sent `Handshake`. + /// Validate `Handshake` and report status (accept/reject) to `Notifications`. + NotificationStreamOpened { + /// Peer ID. + peer: PeerId, + + /// Role of the peer. + role: ObservedRole, + + /// Negotiated fallback. + negotiated_fallback: Option, + + /// Notification sink. + sink: NotificationsSink, + }, + + /// Substream was closed. + NotificationStreamClosed { + /// Peer Id. + peer: PeerId, + }, + + /// Notification was received from the substream. + NotificationReceived { + /// Peer ID. + peer: PeerId, + + /// Received notification. + notification: Vec, + }, +} + +/// Notification commands. +/// +/// Commands sent by the notifications protocol to `Notifications` +/// in order to modify connectivity state and communicate with the remote peer. +pub enum NotificationCommand { + /// Instruct `Notifications` to open a substream to peer. + OpenSubstream(PeerId), + + /// Instruct `Notifications` to close the substream to peer. + CloseSubstream(PeerId), + + /// Send notification to peer. + SendNotification(PeerId, Vec), + + /// Set handshake for the notifications protocol. + SetHandshake(Vec), +} + +/// Handle that is passed on to the notifications protocol. +#[derive(Debug)] +pub struct NotificationHandle { + /// TX channel for sending commands to `Notifications`. + tx: mpsc::Sender, + + /// RX channel for receiving events from `Notifications`. + rx: mpsc::Receiver, + + /// Connected peers. + peers: HashMap, +} + +impl NotificationHandle { + /// Create new [`NotificationHandle`]. + fn new( + tx: mpsc::Sender, + rx: mpsc::Receiver, + ) -> Self { + Self { tx, rx, peers: HashMap::new() } + } +} + +#[async_trait::async_trait] +impl NotificationService for NotificationHandle { + /// Instruct `Notifications` to open a new substream for `peer`. + async fn open_substream(&mut self, peer: PeerId) -> Result<(), ()> { + todo!("support for opening substreams not implemented yet"); + } + + /// Instruct `Notifications` to close substream for `peer`. + async fn close_substream(&mut self, peer: PeerId) -> Result<(), ()> { + todo!("support for closing substreams not implemented yet"); + } + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification( + &self, + peer: &PeerId, + notification: Vec, + ) -> Result<(), error::Error> { + self.peers + .get(&peer) + .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? + .send_sync_notification(notification); + Ok(()) + } + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + async fn send_async_notification( + &self, + peer: &PeerId, + notification: Vec, + ) -> Result<(), error::Error> { + self.peers + .get(&peer) + .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? + .reserve_notification() + .await + .map_err(|_| error::Error::ConnectionClosed)? + .send(notification) + .map_err(|_| error::Error::ChannelClosed) + } + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_hanshake(&mut self, handshake: Vec) -> Result<(), ()> { + self.tx.send(NotificationCommand::SetHandshake(handshake)).await.map_err(|_| ()) + } + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option { + match self.rx.recv().await? { + InnerNotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => + Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }), + InnerNotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + sink, + } => { + self.peers.insert(peer, sink); + Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + }) + }, + InnerNotificationEvent::NotificationStreamClosed { peer } => { + self.peers.remove(&peer); + Some(NotificationEvent::NotificationStreamClosed { peer }) + }, + InnerNotificationEvent::NotificationReceived { peer, notification } => + Some(NotificationEvent::NotificationReceived { peer, notification }), + } + } +} + +/// Channel pair which allows `Notifications` to interact with a protocol. +#[derive(Debug)] +pub struct ProtocolHandlePair( + mpsc::Sender, + mpsc::Receiver, +); + +impl ProtocolHandlePair { + /// Create new [`ProtocolHandlePair`]. + fn new( + tx: mpsc::Sender, + rx: mpsc::Receiver, + ) -> Self { + Self(tx, rx) + } + + /// Split [`ProtocolHandlePair`] into a handle which allows it to send events to the protocol + /// into a stream of commands received from the protocol. + pub fn split(self) -> (ProtocolHandle, Box>) { + (ProtocolHandle::new(self.0), Box::new(tokio_stream::wrappers::ReceiverStream::new(self.1))) + } +} + +/// Handle that is passed on to `Notifications` and allows it to directly communicate +/// with the protocol. +#[derive(Debug)] +pub struct ProtocolHandle { + /// TX channel for sending events to protocol. + tx: mpsc::Sender, +} + +impl ProtocolHandle { + fn new(tx: mpsc::Sender) -> Self { + Self { tx } + } + + /// Report to the protocol that a substream has been opened and it must be validated by the + /// protocol. + /// + /// Return `oneshot::Receiver` which allows `Notifications` to poll for the validation result + /// from protocol. + async fn report_incoming_substream( + &self, + peer: PeerId, + handshake: Vec, + ) -> Result, ()> { + let (result_tx, rx) = oneshot::channel(); + + self.tx + .send(InnerNotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) + .await + .map(|_| rx) + .map_err(|_| ()) + } + + /// Report to the protocol that a substream has been opened and that it can now use the handle + /// to send notifications to the remote peer. + async fn report_substream_opened( + &self, + peer: PeerId, + role: ObservedRole, + negotiated_fallback: Option, + sink: NotificationsSink, + ) -> Result<(), ()> { + self.tx + .send(InnerNotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + sink, + }) + .await + .map_err(|_| ()) + } + + /// Substream was closed. + async fn report_substream_closed(&mut self, peer: PeerId) -> Result<(), ()> { + self.tx + .send(InnerNotificationEvent::NotificationStreamClosed { peer }) + .await + .map_err(|_| ()) + } + + /// Notification was received from the substream. + async fn report_notification_received( + &mut self, + peer: PeerId, + notification: Vec, + ) -> Result<(), ()> { + self.tx + .send(InnerNotificationEvent::NotificationReceived { peer, notification }) + .await + .map_err(|_| ()) + } +} + +/// Create new (protocol, notification) handle pair. +/// +/// Handle pair allows `Notifications` and the protocol to communicate with each other directly. +pub fn notification_service() -> (ProtocolHandlePair, Box) { + let (cmd_tx, cmd_rx) = mpsc::channel(64); // TODO: zzz + let (event_tx, event_rx) = mpsc::channel(64); // TODO: zzz + + (ProtocolHandlePair::new(event_tx, cmd_rx), Box::new(NotificationHandle::new(cmd_tx, event_rx))) +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::prelude::*; + + #[tokio::test] + async fn validate_and_accept_substream() { + let (proto, mut notif) = notification_service(); + let (mut handle, stream) = proto.split(); + + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + result_tx.send(ValidationResult::Accept); + } else { + panic!("invalid event received"); + } + + assert_eq!(rx.await.unwrap(), ValidationResult::Accept); + } + + #[tokio::test] + async fn substream_opened() { + let (proto, mut notif) = notification_service(); + let (sink, _, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, stream) = proto.split(); + + let peer_id = PeerId::random(); + handle + .report_substream_opened(peer_id, ObservedRole::Full, None, sink) + .await + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + } else { + panic!("invalid event received"); + } + } + + #[tokio::test] + async fn send_sync_notification() { + let (proto, mut notif) = notification_service(); + let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + result_tx.send(ValidationResult::Accept); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, ObservedRole::Full, None, sink) + .await + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + } else { + panic!("invalid event received"); + } + + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }) + ); + } + + #[tokio::test] + async fn send_async_notification() { + let (proto, mut notif) = notification_service(); + let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + result_tx.send(ValidationResult::Accept); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, ObservedRole::Full, None, sink) + .await + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + } else { + panic!("invalid event received"); + } + + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }) + ); + } + + #[tokio::test] + async fn send_sync_notification_to_non_existent_peer() { + let (proto, mut notif) = notification_service(); + let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, stream) = proto.split(); + let peer = PeerId::random(); + + if let Err(error::Error::PeerDoesntExist(peer_id)) = + notif.send_sync_notification(&peer, vec![1, 3, 3, 7]) + { + assert_eq!(peer, peer_id); + } else { + panic!("invalid error received from `send_sync_notification()`"); + } + } + + #[tokio::test] + async fn send_async_notification_to_non_existent_peer() { + let (proto, mut notif) = notification_service(); + let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, stream) = proto.split(); + let peer = PeerId::random(); + + if let Err(error::Error::PeerDoesntExist(peer_id)) = + notif.send_async_notification(&peer, vec![1, 3, 3, 7]).await + { + assert_eq!(peer, peer_id); + } else { + panic!("invalid error received from `send_sync_notification()`"); + } + } + + #[tokio::test] + async fn receive_notification() { + let (proto, mut notif) = notification_service(); + let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + result_tx.send(ValidationResult::Accept); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, ObservedRole::Full, None, sink) + .await + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + } else { + panic!("invalid event received"); + } + + // notification is received + handle.report_notification_received(peer_id, vec![1, 3, 3, 8]).await.unwrap(); + + if let Some(NotificationEvent::NotificationReceived { peer, notification }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(notification, vec![1, 3, 3, 8]); + } else { + panic!("invalid event received"); + } + } + + #[tokio::test] + async fn backpressure_works() { + let (proto, mut notif) = notification_service(); + let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + result_tx.send(ValidationResult::Accept); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, ObservedRole::Full, None, sink) + .await + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + } else { + panic!("invalid event received"); + } + + // fill the message buffer with messages + for i in 0..=ASYNC_NOTIFICATIONS_BUFFER_SIZE { + assert!( + futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, i as u8])) + .is_ready() + ); + } + + // try to send one more message and verify that the call blocks + assert!( + futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, 9])).is_pending() + ); + + // release one slot from the buffer for new message + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 0] }) + ); + + // verify that a message can be sent + assert!( + futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, 9])).is_ready() + ); + } + + #[tokio::test] + async fn peer_disconnects_then_sync_notification_is_sent() { + let (proto, mut notif) = notification_service(); + let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + result_tx.send(ValidationResult::Accept); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, ObservedRole::Full, None, sink) + .await + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + } else { + panic!("invalid event received"); + } + + // report that a substream has been closed but don't poll `notif` to receive this + // information + handle.report_substream_closed(peer_id).await.unwrap(); + drop(sync_rx); + + // as per documentation, error is not reported but the notification is silently dropped + assert!(notif.send_sync_notification(&peer_id, vec![1, 3, 3, 7]).is_ok()); + } + + #[tokio::test] + async fn peer_disconnects_then_async_notification_is_sent() { + let (proto, mut notif) = notification_service(); + let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + result_tx.send(ValidationResult::Accept); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, ObservedRole::Full, None, sink) + .await + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + } else { + panic!("invalid event received"); + } + + // report that a substream has been closed but don't poll `notif` to receive this + // information + handle.report_substream_closed(peer_id).await.unwrap(); + drop(async_rx); + + // as per documentation, error is not reported but the notification is silently dropped + if let Err(error::Error::ConnectionClosed) = + notif.send_async_notification(&peer_id, vec![1, 3, 3, 7]).await + { + } else { + panic!("invalid state after calling `send_async_notificatio()` on closed connection") + } + } +} diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 3cc3540128454..17aacbe3d3b05 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -20,6 +20,7 @@ use crate::{ config::MultiaddrWithPeerId, + error, event::Event, request_responses::{IfDisconnected, RequestFailure}, service::signature::Signature, @@ -29,9 +30,10 @@ use crate::{ use futures::{channel::oneshot, Stream}; use libp2p::{Multiaddr, PeerId}; +use sc_network_common::role::ObservedRole; use sc_peerset::ReputationChange; -use std::{collections::HashSet, future::Future, pin::Pin, sync::Arc}; +use std::{collections::HashSet, fmt::Debug, future::Future, pin::Pin, sync::Arc}; pub use libp2p::{identity::SigningError, kad::record::Key as KademliaKey}; @@ -634,6 +636,7 @@ where } /// Substream acceptance result. +#[derive(Debug, PartialEq, Eq)] pub enum ValidationResult { /// Accept inbound substream. Accept, @@ -643,6 +646,7 @@ pub enum ValidationResult { } /// Events received by the protocol from `Notifications`. +#[derive(Debug)] pub enum NotificationEvent { /// Validate inbound substream. ValidateInboundSubstream { @@ -653,7 +657,7 @@ pub enum NotificationEvent { handshake: Vec, /// `oneshot::Sender` for sending validation result back to `Notifications` - result_tx: oneshot::Sender, + result_tx: tokio::sync::oneshot::Sender, }, /// Remote identified by `PeerId` opened a substream and sent `Handshake`. @@ -736,14 +740,18 @@ pub trait NotificationService: Debug + Send { async fn close_substream(&mut self, peer: PeerId) -> Result<(), ()>; /// Send synchronous `notification` to `peer`. - fn send_sync_notification(&mut self, peer: PeerId, notification: Vec) -> Result<(), ()>; + fn send_sync_notification( + &self, + peer: &PeerId, + notification: Vec, + ) -> Result<(), error::Error>; /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. async fn send_async_notification( - &mut self, - peer: PeerId, + &self, + peer: &PeerId, notification: Vec, - ) -> Result<(), ()>; + ) -> Result<(), error::Error>; /// Set handshake for the notification protocol replacing the old handshake. async fn set_hanshake(&mut self, handshake: Vec) -> Result<(), ()>; diff --git a/client/network/statement/src/lib.rs b/client/network/statement/src/lib.rs index 6520154b6ed22..ac9b5f0efc1dc 100644 --- a/client/network/statement/src/lib.rs +++ b/client/network/statement/src/lib.rs @@ -27,6 +27,7 @@ //! `Future` that processes statements. use crate::config::*; + use codec::{Decode, Encode}; use futures::{channel::oneshot, prelude::*, stream::FuturesUnordered, FutureExt}; use libp2p::{multiaddr, PeerId}; @@ -37,7 +38,7 @@ use sc_network::{ event::Event, types::ProtocolName, utils::{interval, LruHashSet}, - NetworkEventStream, NetworkNotification, NetworkPeers, + NetworkEventStream, NetworkNotification, NetworkPeers, NotificationService, }; use sc_network_common::{ role::ObservedRole, @@ -103,25 +104,23 @@ impl Metrics { /// Prototype for a [`StatementHandler`]. pub struct StatementHandlerPrototype { protocol_name: ProtocolName, + notification_service: Box, } impl StatementHandlerPrototype { /// Create a new instance. - pub fn new>(genesis_hash: Hash, fork_id: Option<&str>) -> Self { + pub fn new>( + genesis_hash: Hash, + fork_id: Option<&str>, + ) -> (Self, NonDefaultSetConfig) { let genesis_hash = genesis_hash.as_ref(); let protocol_name = if let Some(fork_id) = fork_id { format!("/{}/{}/statement/1", array_bytes::bytes2hex("", genesis_hash), fork_id) } else { format!("/{}/statement/1", array_bytes::bytes2hex("", genesis_hash)) }; - - Self { protocol_name: protocol_name.into() } - } - - /// Returns the configuration of the set to put in the network configuration. - pub fn set_config(&self) -> NonDefaultSetConfig { - NonDefaultSetConfig::new( - self.protocol_name.clone(), + let (config, notification_service) = NonDefaultSetConfig::new( + protocol_name.clone().into(), Vec::new(), MAX_STATEMENT_SIZE, None, @@ -131,7 +130,9 @@ impl StatementHandlerPrototype { reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - ) + ); + + (Self { protocol_name: protocol_name.into(), notification_service }, config) } /// Turns the prototype into the actual handler. @@ -178,6 +179,7 @@ impl StatementHandlerPrototype { let handler = StatementHandler { protocol_name: self.protocol_name, + notification_service: self.notification_service, propagate_timeout: (Box::pin(interval(PROPAGATE_TIMEOUT)) as Pin + Send>>) .fuse(), @@ -225,6 +227,8 @@ pub struct StatementHandler< net_event_stream: stream::Fuse + Send>>>, /// Receiver for syncing-related events. sync_event_stream: stream::Fuse + Send>>>, + /// Notification service. + notification_service: Box, // All connected peers peers: HashMap, statement_store: Arc, diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index 6268275ba1d94..a2ad85ff77bff 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -39,8 +39,10 @@ use sc_network::{ config::{ FullNetworkConfiguration, NonDefaultSetConfig, ProtocolId, SyncMode as SyncOperationMode, }, + event::Event, + types::ProtocolName, utils::LruHashSet, - NotificationsSink, ProtocolName, + NotificationService, NotificationsSink, }; use sc_network_common::{ role::Roles, @@ -243,6 +245,9 @@ pub struct SyncingEngine { /// Prometheus metrics. metrics: Option, + + /// Handle that is used to communicate with `sc_network::Notifications`. + _notification_handle: Box, } impl SyncingEngine @@ -338,7 +343,7 @@ where total.saturating_sub(net_config.network_config.default_peers_set_num_full) as usize }; - let (chain_sync, block_announce_config) = ChainSync::new( + let (chain_sync, block_announce_config, _notification_handle) = ChainSync::new( mode, client.clone(), protocol_id, @@ -388,6 +393,7 @@ where default_peers_set_num_full, default_peers_set_num_light, event_streams: Vec::new(), + _notification_handle, tick_timeout: Delay::new(TICK_TIMEOUT), metrics: if let Some(r) = metrics_registry { match Metrics::register(r, is_major_syncing.clone()) { diff --git a/client/network/sync/src/lib.rs b/client/network/sync/src/lib.rs index 68b834bc46f31..1ad50aa869fea 100644 --- a/client/network/sync/src/lib.rs +++ b/client/network/sync/src/lib.rs @@ -55,6 +55,7 @@ use sc_network::{ }, request_responses::{IfDisconnected, RequestFailure}, types::ProtocolName, + NotificationService, }; use sc_network_common::{ role::Roles, @@ -1418,8 +1419,8 @@ where block_request_protocol_name: ProtocolName, state_request_protocol_name: ProtocolName, warp_sync_protocol_name: Option, - ) -> Result<(Self, NonDefaultSetConfig), ClientError> { - let block_announce_config = Self::get_block_announce_proto_config( + ) -> Result<(Self, NonDefaultSetConfig, Box), ClientError> { + let (block_announce_config, notification_handle) = Self::get_block_announce_proto_config( protocol_id, fork_id, roles, @@ -1475,7 +1476,7 @@ where }; sync.reset_sync_start_point()?; - Ok((sync, block_announce_config)) + Ok((sync, block_announce_config, notification_handle)) } /// Returns the median seen block number. @@ -1957,7 +1958,7 @@ where best_number: NumberFor, best_hash: B::Hash, genesis_hash: B::Hash, - ) -> NonDefaultSetConfig { + ) -> (NonDefaultSetConfig, Box) { let block_announces_protocol = { let genesis_hash = genesis_hash.as_ref(); if let Some(ref fork_id) = fork_id { @@ -3210,7 +3211,7 @@ mod test { let import_queue = Box::new(sc_consensus::import_queue::mock::MockImportQueueHandle::new()); let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -3277,7 +3278,7 @@ mod test { let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -3459,7 +3460,7 @@ mod test { let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -3586,7 +3587,7 @@ mod test { NetworkServiceProvider::new(); let info = client.info(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -3744,7 +3745,7 @@ mod test { let info = client.info(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -3887,7 +3888,7 @@ mod test { let info = client.info(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -4032,7 +4033,7 @@ mod test { let mut client = Arc::new(TestClientBuilder::new().build()); let blocks = (0..3).map(|_| build_block(&mut client, None, false)).collect::>(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -4078,7 +4079,7 @@ mod test { let empty_client = Arc::new(TestClientBuilder::new().build()); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, empty_client.clone(), ProtocolId::from("test-protocol-name"), diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 6f5212b3d4d1f..a023f70dbdaac 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -56,7 +56,7 @@ use sc_network::{ request_responses::ProtocolConfig as RequestResponseConfig, types::ProtocolName, Multiaddr, NetworkBlock, NetworkService, NetworkStateInfo, NetworkSyncForkRequest, - NetworkWorker, + NetworkWorker, NotificationService, }; use sc_network_common::{ role::Roles, @@ -240,6 +240,7 @@ pub struct Peer { imported_blocks_stream: Pin> + Send>>, finality_notification_stream: Pin> + Send>>, listen_addr: Multiaddr, + notification_handles: HashMap>, } impl Peer @@ -505,10 +506,19 @@ where self.network.service() } + /// Get `SyncingService`. pub fn sync_service(&self) -> &Arc> { &self.sync_service } + /// Take notification handle for enabled protocol. + pub fn take_notification_handle( + &mut self, + protocol: &ProtocolName, + ) -> Option> { + self.notification_handles.remove(protocol) + } + /// Get a reference to the network worker. pub fn network(&self) -> &NetworkWorker::Hash> { &self.network @@ -800,32 +810,23 @@ where network_config.transport = TransportConfig::MemoryOnly; network_config.listen_addresses = vec![listen_addr.clone()]; network_config.allow_non_globals_in_dht = true; - // <<<<<<< HEAD - // ||||||| parent of 7a8fd6570a (Make fields of `NonDefaultSetConfig` private) - // network_config - // .request_response_protocols - // .extend(config.request_response_protocols); - // network_config.extra_sets = config - // .notifications_protocols - // .into_iter() - // .map(|p| NonDefaultSetConfig { - // notifications_protocol: p, - // fallback_names: Vec::new(), - // max_notification_size: 1024 * 1024, - // handshake: None, - // set_config: Default::default(), - // }) - // .collect(); - // ======= - // network_config - // .request_response_protocols - // .extend(config.request_response_protocols); - // network_config.extra_sets = config - // .notifications_protocols - // .into_iter() - // .map(|p| NonDefaultSetConfig::new(p, Vec::new(), 1024 * 1024, None, Default::default())) - // .collect(); - // >>>>>>> 7a8fd6570a (Make fields of `NonDefaultSetConfig` private) + + let (notif_configs, notif_handles): (Vec<_>, Vec<_>) = config + .notifications_protocols + .into_iter() + .map(|p| { + let (config, handle) = NonDefaultSetConfig::new( + p.clone(), + Vec::new(), + 1024 * 1024, + None, + Default::default(), + ); + + (config, (p, handle)) + }) + .unzip(); + if let Some(connect_to) = config.connect_to_peers { let addrs = connect_to .iter() @@ -931,14 +932,8 @@ where full_net_config.add_request_response_protocol(config); } - for protocol in config.notifications_protocols { - full_net_config.add_notification_protocol(NonDefaultSetConfig::new( - protocol, - Vec::new(), - 1024 * 1024, - None, - Default::default(), - )); + for config in notif_configs { + full_net_config.add_notification_protocol(config); } let genesis_hash = @@ -990,6 +985,7 @@ where backend: Some(backend), imported_blocks_stream, finality_notification_stream, + notification_handles: HashMap::from_iter(notif_handles.into_iter()), block_import, verifier, network, diff --git a/client/network/test/src/service.rs b/client/network/test/src/service.rs index c747565c0df71..203a8c7105a61 100644 --- a/client/network/test/src/service.rs +++ b/client/network/test/src/service.rs @@ -203,13 +203,14 @@ impl TestNetworkBuilder { full_net_config.add_notification_protocol(config); } } else { - full_net_config.add_notification_protocol(config::NonDefaultSetConfig::new( + let (config, handle) = config::NonDefaultSetConfig::new( PROTOCOL_NAME.into(), Vec::new(), 1024 * 1024, None, self.set_config.unwrap_or_default(), - )); + ); + full_net_config.add_notification_protocol(config); } for config in [ @@ -569,14 +570,15 @@ async fn fallback_name_working() { const NEW_PROTOCOL_NAME: &str = "/new-shiny-protocol-that-isnt-PROTOCOL_NAME"; let listen_addr = config::build_multiaddr![Memory(rand::random::())]; + let (config, handle) = config::NonDefaultSetConfig::new( + NEW_PROTOCOL_NAME.into(), + vec![PROTOCOL_NAME.into()], + 1024 * 1024, + None, + Default::default(), + ); let (node1, mut events_stream1) = TestNetworkBuilder::new() - .with_notification_protocol(config::NonDefaultSetConfig::new( - NEW_PROTOCOL_NAME.into(), - vec![PROTOCOL_NAME.into()], - 1024 * 1024, - None, - Default::default(), - )) + .with_notification_protocol(config) .with_config(config::NetworkConfiguration { listen_addresses: vec![listen_addr.clone()], transport: TransportConfig::MemoryOnly, diff --git a/client/network/transactions/src/lib.rs b/client/network/transactions/src/lib.rs index 3ebc7ce74676f..7faf2b893a590 100644 --- a/client/network/transactions/src/lib.rs +++ b/client/network/transactions/src/lib.rs @@ -40,7 +40,7 @@ use sc_network::{ event::Event, types::ProtocolName, utils::{interval, LruHashSet}, - NetworkEventStream, NetworkNotification, NetworkPeers, + NetworkEventStream, NetworkNotification, NetworkPeers, NotificationService, }; use sc_network_common::{ role::ObservedRole, @@ -120,7 +120,11 @@ impl Future for PendingTransaction { /// Prototype for a [`TransactionsHandler`]. pub struct TransactionsHandlerPrototype { + /// Name of the transaction protocol. protocol_name: ProtocolName, + + /// Handle that is used to communicate with `sc_network::Notifications`. + notification_handle: Box, } impl TransactionsHandlerPrototype { @@ -137,7 +141,7 @@ impl TransactionsHandlerPrototype { format!("/{}/transactions/1", array_bytes::bytes2hex("", genesis_hash)) } .into(); - let config = NonDefaultSetConfig::new( + let (config, notification_handle) = NonDefaultSetConfig::new( protocol_name.clone(), vec![format!("/{}/transactions/1", protocol_id.as_ref()).into()], MAX_TRANSACTIONS_SIZE, @@ -150,7 +154,7 @@ impl TransactionsHandlerPrototype { }, ); - (Self { protocol_name }, config) + (Self { protocol_name, notification_handle }, config) } /// Turns the prototype into the actual handler. Returns a controller that allows controlling @@ -176,6 +180,7 @@ impl TransactionsHandlerPrototype { let handler = TransactionsHandler { protocol_name: self.protocol_name, + _notification_handle: self.notification_handle, propagate_timeout: (Box::pin(interval(PROPAGATE_TIMEOUT)) as Pin + Send>>) .fuse(), @@ -260,6 +265,8 @@ pub struct TransactionsHandler< from_controller: TracingUnboundedReceiver>, /// Prometheus metrics. metrics: Option, + /// Handle that is used to communicate with `sc_network::Notifications`. + _notification_handle: Box, } /// Peer information From b42b901075c1f36f45cf1ceba7419847d5b49b5b Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Thu, 16 Mar 2023 09:06:34 +0200 Subject: [PATCH 06/43] Initialize command streams and protocol handles in `Notifications` --- client/network/src/config.rs | 13 ++++- client/network/src/protocol.rs | 45 ++++++++++------ .../src/protocol/notifications/behaviour.rs | 51 ++++++++++++++----- .../src/protocol/notifications/service.rs | 2 +- client/network/sync/src/engine.rs | 1 - 5 files changed, 81 insertions(+), 31 deletions(-) diff --git a/client/network/src/config.rs b/client/network/src/config.rs index d105e803bcf0b..edde869f9f18a 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -518,7 +518,7 @@ pub struct NonDefaultSetConfig { /// `ProtocolHandle` is given to `Notifications` when it initializes itself. This handle allows /// `Notifications ` to communicate with the protocol directly without relaying events through /// `sc-network.` - protocol_handle_pair: ProtocolHandlePair, + protocol_handle_pair: Option, } impl NonDefaultSetConfig { @@ -539,7 +539,7 @@ impl NonDefaultSetConfig { fallback_names, handshake, set_config, - protocol_handle_pair, + protocol_handle_pair: Some(protocol_handle_pair), }, notification_handle, ) @@ -570,6 +570,15 @@ impl NonDefaultSetConfig { &self.set_config } + /// Take `ProtocolHandlePair` from `NonDefaultSetConfig` + // TODO: consume self instead? + pub(crate) fn take_protocol_handle(&mut self) -> ProtocolHandlePair { + std::mem::take(&mut self.protocol_handle_pair).expect( + "`take_protocol_handle()` is called only once for each `NonDefaultSetConfig`\ + during protocol initialization so it must exist. qed", + ) + } + /// Modifies the configuration to allow non-reserved nodes. pub fn allow_non_reserved(&mut self, in_peers: u32, out_peers: u32) { self.set_config.in_peers = in_peers; diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index ef29058fff67b..8973593212e85 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -105,7 +105,7 @@ impl Protocol { roles: Roles, network_config: &config::NetworkConfiguration, notification_protocols: Vec, - block_announces_protocol: config::NonDefaultSetConfig, + mut block_announces_protocol: config::NonDefaultSetConfig, tx: TracingUnboundedSender>, ) -> error::Result<(Self, sc_peerset::PeersetHandle, Vec<(PeerId, Multiaddr)>)> { let mut known_addresses = Vec::new(); @@ -171,20 +171,35 @@ impl Protocol { // NOTE: Block announcement protocol is still very much hardcoded into // `Protocol`. This protocol must be the first notification protocol given to // `Notifications` - iter::once(notifications::ProtocolConfig { - name: block_announces_protocol.protocol_name().clone(), - fallback_names: block_announces_protocol - .fallback_names() - .cloned() - .collect(), - handshake: block_announces_protocol.handshake().as_ref().unwrap().to_vec(), - max_notification_size: block_announces_protocol.max_notification_size(), - }) - .chain(notification_protocols.iter().map(|s| notifications::ProtocolConfig { - name: s.protocol_name().clone(), - fallback_names: s.fallback_names().cloned().collect(), - handshake: s.handshake().as_ref().map_or(roles.encode(), |h| (*h).to_vec()), - max_notification_size: s.max_notification_size(), + iter::once(( + notifications::ProtocolConfig { + name: block_announces_protocol.protocol_name().clone(), + fallback_names: block_announces_protocol + .fallback_names() + .cloned() + .collect(), + handshake: block_announces_protocol + .handshake() + .as_ref() + .unwrap() + .to_vec(), + max_notification_size: block_announces_protocol.max_notification_size(), + }, + block_announces_protocol.take_protocol_handle(), + )) + .chain(notification_protocols.into_iter().map(|mut s| { + ( + notifications::ProtocolConfig { + name: s.protocol_name().clone(), + fallback_names: s.fallback_names().cloned().collect(), + handshake: s + .handshake() + .as_ref() + .map_or(roles.encode(), |h| (*h).to_vec()), + max_notification_size: s.max_notification_size(), + }, + s.take_protocol_handle(), + ) })), ), installed_protocols, diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 7e56793939b55..ae3be1b7cf830 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -17,8 +17,10 @@ // along with this program. If not, see . use crate::{ - protocol::notifications::handler::{ - self, NotificationsSink, NotifsHandler, NotifsHandlerIn, NotifsHandlerOut, + config::ProtocolHandlePair, + protocol::notifications::{ + handler::{self, NotificationsSink, NotifsHandlerIn, NotifsHandlerOut, NotifsHandlerProto}, + service::{NotificationCommand, ProtocolHandle}, }, types::ProtocolName, }; @@ -38,8 +40,11 @@ use libp2p::{ use log::{error, trace, warn}; use parking_lot::RwLock; use rand::distributions::{Distribution as _, Uniform}; -use sc_peerset::DropReason; use smallvec::SmallVec; +use tokio_stream::StreamMap; + +use sc_peerset::DropReason; + use std::{ cmp, collections::{hash_map::Entry, VecDeque}, @@ -107,6 +112,12 @@ pub struct Notifications { /// Notification protocols. Entries never change after initialization. notif_protocols: Vec, + /// Protocol handles. + protocol_handles: Vec, + + // Command streams. + command_streams: StreamMap + Send>>, + /// Receiver for instructions about who to connect to or disconnect from. peerset: sc_peerset::Peerset, @@ -360,21 +371,37 @@ impl Notifications { /// Creates a `CustomProtos`. pub fn new( peerset: sc_peerset::Peerset, - notif_protocols: impl Iterator, + notif_protocols: impl Iterator, ) -> Self { - let notif_protocols = notif_protocols - .map(|cfg| handler::ProtocolConfig { - name: cfg.name, - fallback_names: cfg.fallback_names, - handshake: Arc::new(RwLock::new(cfg.handshake)), - max_notification_size: cfg.max_notification_size, + let (notif_protocols, protocol_handle_pairs): (Vec<_>, Vec<_>) = notif_protocols + .map(|(cfg, protocol_handle_pair)| { + ( + handler::ProtocolConfig { + name: cfg.name, + fallback_names: cfg.fallback_names, + handshake: Arc::new(RwLock::new(cfg.handshake)), + max_notification_size: cfg.max_notification_size, + }, + protocol_handle_pair, + ) }) - .collect::>(); - + .unzip(); assert!(!notif_protocols.is_empty()); + let (protocol_handles, command_streams): (Vec<_>, Vec<_>) = protocol_handle_pairs + .into_iter() + .enumerate() + .map(|(set_id, protocol_handle_pair)| { + let (protocol_handle, command_stream) = protocol_handle_pair.split(); + + (protocol_handle, (set_id, command_stream)) + }) + .unzip(); + Self { notif_protocols, + protocol_handles, + command_streams: StreamMap::from_iter(command_streams.into_iter()), peerset, peers: FnvHashMap::default(), delays: Default::default(), diff --git a/client/network/src/protocol/notifications/service.rs b/client/network/src/protocol/notifications/service.rs index 26a287406e77c..7d7b0e5a65205 100644 --- a/client/network/src/protocol/notifications/service.rs +++ b/client/network/src/protocol/notifications/service.rs @@ -217,7 +217,7 @@ impl ProtocolHandlePair { /// Split [`ProtocolHandlePair`] into a handle which allows it to send events to the protocol /// into a stream of commands received from the protocol. - pub fn split(self) -> (ProtocolHandle, Box>) { + pub fn split(self) -> (ProtocolHandle, Box + Send>) { (ProtocolHandle::new(self.0), Box::new(tokio_stream::wrappers::ReceiverStream::new(self.1))) } } diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index a2ad85ff77bff..2b8c7a8e986ab 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -39,7 +39,6 @@ use sc_network::{ config::{ FullNetworkConfiguration, NonDefaultSetConfig, ProtocolId, SyncMode as SyncOperationMode, }, - event::Event, types::ProtocolName, utils::LruHashSet, NotificationService, NotificationsSink, From 9685625dcef81fffc3a308f4121ae08ea23b4646 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Thu, 16 Mar 2023 09:06:45 +0200 Subject: [PATCH 07/43] Poll commands from protocols in `Notifications` Substream open/close functionality is not supported so they're left as `todo!()s`. Remove `SendNotification` command as sending notifications is now supported by using the installed notification sink received in `NotificationStreamOpened` event. --- .../src/protocol/notifications/behaviour.rs | 22 ++++++++++++++++++- .../src/protocol/notifications/service.rs | 15 ++++++------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index ae3be1b7cf830..a1deb36cf719d 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -116,7 +116,7 @@ pub struct Notifications { protocol_handles: Vec, // Command streams. - command_streams: StreamMap + Send>>, + command_streams: StreamMap + Send + Unpin>>, /// Receiver for instructions about who to connect to or disconnect from. peerset: sc_peerset::Peerset, @@ -2042,6 +2042,26 @@ impl NetworkBehaviour for Notifications { } } + // poll commands from protocols + loop { + match futures::Stream::poll_next(Pin::new(&mut self.command_streams), cx) { + Poll::Ready(Some((set_id, command))) => match command { + NotificationCommand::SetHandshake(handshake) => { + self.set_notif_protocol_handshake(set_id.into(), handshake); + }, + NotificationCommand::OpenSubstream(_peer) | + NotificationCommand::CloseSubstream(_peer) => { + todo!("substream control not implemented"); + }, + }, + Poll::Ready(None) => { + error!(target: "sub-libp2p", "Protocol command streams have been shut down"); + break + }, + Poll::Pending => break, + } + } + while let Poll::Ready(Some((delay_id, peer_id, set_id))) = Pin::new(&mut self.delays).poll_next(cx) { diff --git a/client/network/src/protocol/notifications/service.rs b/client/network/src/protocol/notifications/service.rs index 7d7b0e5a65205..b240d1ebf7dca 100644 --- a/client/network/src/protocol/notifications/service.rs +++ b/client/network/src/protocol/notifications/service.rs @@ -86,8 +86,7 @@ enum InnerNotificationEvent { /// Notification commands. /// -/// Commands sent by the notifications protocol to `Notifications` -/// in order to modify connectivity state and communicate with the remote peer. +/// Sent by the installed protocols to `Notifications` to open/close/modify substreams. pub enum NotificationCommand { /// Instruct `Notifications` to open a substream to peer. OpenSubstream(PeerId), @@ -95,9 +94,6 @@ pub enum NotificationCommand { /// Instruct `Notifications` to close the substream to peer. CloseSubstream(PeerId), - /// Send notification to peer. - SendNotification(PeerId, Vec), - /// Set handshake for the notifications protocol. SetHandshake(Vec), } @@ -158,6 +154,7 @@ impl NotificationService for NotificationHandle { ) -> Result<(), error::Error> { self.peers .get(&peer) + // TODO: check what the current implementation does in case the peer doesn't exist .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? .reserve_notification() .await @@ -215,9 +212,11 @@ impl ProtocolHandlePair { Self(tx, rx) } - /// Split [`ProtocolHandlePair`] into a handle which allows it to send events to the protocol - /// into a stream of commands received from the protocol. - pub fn split(self) -> (ProtocolHandle, Box + Send>) { + /// Consume `self` and split [`ProtocolHandlePair`] into a handle which allows it to send events + /// to the protocol into a stream of commands received from the protocol. + pub fn split( + self, + ) -> (ProtocolHandle, Box + Send + Unpin>) { (ProtocolHandle::new(self.0), Box::new(tokio_stream::wrappers::ReceiverStream::new(self.1))) } } From b93cf904e8d06b1e1992b34e7ee9cde4c1e2963b Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Mon, 20 Mar 2023 16:01:41 +0200 Subject: [PATCH 08/43] Start using `tracing_unbounded` for `Notifications` -> protocols channel --- client/network/src/lib.rs | 2 +- .../src/protocol/notifications/behaviour.rs | 136 ++++++++++-------- .../src/protocol/notifications/service.rs | 97 +++++-------- .../src/protocol/notifications/tests.rs | 18 ++- 4 files changed, 125 insertions(+), 128 deletions(-) diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index 22d7040e04172..3ab0fd061b220 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -244,7 +244,6 @@ mod behaviour; mod protocol; -mod service; pub mod config; pub mod discovery; @@ -253,6 +252,7 @@ pub mod event; pub mod network_state; pub mod peer_info; pub mod request_responses; +pub mod service; pub mod transport; pub mod types; pub mod utils; diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index a1deb36cf719d..ca34b263c0e98 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -2179,7 +2179,13 @@ mod tests { } } - fn development_notifs() -> (Notifications, sc_peerset::PeersetHandle) { + fn development_notifs() -> ( + Notifications, + sc_peerset::PeersetHandle, + Box, + ) { + let (protocol_handle_pair, notif_service) = + crate::protocol::notifications::service::notification_service(); let (peerset, peerset_handle) = { let mut sets = Vec::with_capacity(1); @@ -2197,20 +2203,24 @@ mod tests { ( Notifications::new( peerset, - iter::once(ProtocolConfig { - name: "/foo".into(), - fallback_names: Vec::new(), - handshake: vec![1, 2, 3, 4], - max_notification_size: u64::MAX, - }), + iter::once(( + ProtocolConfig { + name: "/foo".into(), + fallback_names: Vec::new(), + handshake: vec![1, 2, 3, 4], + max_notification_size: u64::MAX, + }, + protocol_handle_pair, + )), ), peerset_handle, + notif_service, ) } #[test] fn update_handshake() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let inner = notif.notif_protocols.get_mut(0).unwrap().handshake.read().clone(); assert_eq!(inner, vec![1, 2, 3, 4]); @@ -2225,14 +2235,14 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn update_unknown_handshake() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); notif.set_notif_protocol_handshake(1337.into(), vec![5, 6, 7, 8]); } #[test] fn disconnect_backoff_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); notif.peers.insert( @@ -2249,7 +2259,7 @@ mod tests { #[test] fn disconnect_pending_request() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); notif.peers.insert( @@ -2266,7 +2276,7 @@ mod tests { #[test] fn disconnect_requested_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); notif.peers.insert((peer, 0.into()), PeerState::Requested); @@ -2277,7 +2287,7 @@ mod tests { #[test] fn disconnect_disabled_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); notif.peers.insert( (peer, 0.into()), @@ -2293,7 +2303,7 @@ mod tests { #[test] fn remote_opens_connection_and_substream() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let connected = ConnectedPoint::Listener { @@ -2343,7 +2353,7 @@ mod tests { #[tokio::test] async fn disconnect_remote_substream_before_handled_by_peerset() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let connected = ConnectedPoint::Listener { @@ -2379,7 +2389,7 @@ mod tests { #[test] fn peerset_report_connect_backoff() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let set_id = sc_peerset::SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -2444,7 +2454,7 @@ mod tests { #[test] fn peerset_connect_incoming() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -2478,7 +2488,7 @@ mod tests { #[test] fn peerset_disconnect_disable_pending_enable() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let set_id = sc_peerset::SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -2525,7 +2535,7 @@ mod tests { #[test] fn peerset_disconnect_enabled() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -2559,7 +2569,7 @@ mod tests { #[test] fn peerset_disconnect_requested() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = sc_peerset::SetId::from(0); @@ -2574,7 +2584,7 @@ mod tests { #[test] fn peerset_disconnect_pending_request() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let set_id = sc_peerset::SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -2627,7 +2637,7 @@ mod tests { #[test] fn peerset_accept_peer_not_alive() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -2674,7 +2684,7 @@ mod tests { #[test] fn secondary_connection_peer_state_incoming() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -2729,7 +2739,7 @@ mod tests { #[test] fn close_connection_for_disabled_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -2763,7 +2773,7 @@ mod tests { #[test] fn close_connection_for_incoming_peer_one_connection() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -2808,7 +2818,7 @@ mod tests { #[test] fn close_connection_for_incoming_peer_two_connections() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let conn1 = ConnectionId::new_unchecked(1); @@ -2877,7 +2887,7 @@ mod tests { #[test] fn connection_and_substream_open() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -2929,7 +2939,7 @@ mod tests { #[test] fn connection_closed_sink_replaced() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -3025,7 +3035,7 @@ mod tests { #[test] fn dial_failure_for_requested_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = sc_peerset::SetId::from(0); @@ -3048,7 +3058,7 @@ mod tests { #[tokio::test] async fn write_notification() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -3097,7 +3107,7 @@ mod tests { #[test] fn peerset_report_connect_backoff_expired() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let set_id = sc_peerset::SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3145,7 +3155,7 @@ mod tests { #[test] fn peerset_report_disconnect_disabled() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = sc_peerset::SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -3171,7 +3181,7 @@ mod tests { #[test] fn peerset_report_disconnect_backoff() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let set_id = sc_peerset::SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3217,7 +3227,7 @@ mod tests { #[test] fn peer_is_backed_off_if_both_connections_get_closed_while_peer_is_disabled_with_back_off() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let set_id = sc_peerset::SetId::from(0); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); @@ -3290,7 +3300,7 @@ mod tests { #[test] fn inject_connection_closed_incoming_with_backoff() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = sc_peerset::SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -3341,7 +3351,7 @@ mod tests { #[test] fn two_connections_inactive_connection_gets_closed_peer_state_is_still_incoming() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -3396,7 +3406,7 @@ mod tests { #[test] fn two_connections_active_connection_gets_closed_peer_state_is_disabled() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -3454,7 +3464,7 @@ mod tests { #[test] fn inject_connection_closed_for_active_connection() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -3522,7 +3532,7 @@ mod tests { #[test] fn inject_dial_failure_for_pending_request() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let set_id = sc_peerset::SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3585,7 +3595,7 @@ mod tests { #[test] fn peerstate_incoming_open_desired_by_remote() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = sc_peerset::SetId::from(0); let conn1 = ConnectionId::new_unchecked(0); @@ -3639,7 +3649,7 @@ mod tests { #[tokio::test] async fn remove_backoff_peer_after_timeout() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = sc_peerset::SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -3717,7 +3727,7 @@ mod tests { #[tokio::test] async fn reschedule_disabled_pending_enable_when_connection_not_closed() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -3833,7 +3843,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn peerset_report_connect_with_enabled_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -3884,7 +3894,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn peerset_report_connect_with_disabled_pending_enable_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let set_id = sc_peerset::SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3926,7 +3936,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn peerset_report_connect_with_requested_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = sc_peerset::SetId::from(0); @@ -3941,7 +3951,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn peerset_report_connect_with_pending_requested() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let set_id = sc_peerset::SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3994,7 +4004,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn peerset_report_disconnect_with_incoming_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = sc_peerset::SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -4030,7 +4040,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn peerset_report_accept_incoming_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -4071,7 +4081,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn peerset_report_accept_not_incoming_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -4120,7 +4130,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_connection_closed_non_existent_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let endpoint = ConnectedPoint::Listener { local_addr: Multiaddr::empty(), @@ -4140,7 +4150,7 @@ mod tests { #[test] fn disconnect_non_existent_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = sc_peerset::SetId::from(0); @@ -4152,7 +4162,7 @@ mod tests { #[test] fn accept_non_existent_connection() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); notif.peerset_report_accept(0.into()); @@ -4162,7 +4172,7 @@ mod tests { #[test] fn reject_non_existent_connection() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); notif.peerset_report_reject(0.into()); @@ -4172,7 +4182,7 @@ mod tests { #[test] fn reject_non_active_connection() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -4210,7 +4220,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn reject_non_existent_peer_but_alive_connection() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -4250,7 +4260,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_non_existent_connection_closed_for_incoming_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -4293,7 +4303,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_non_existent_connection_closed_for_disabled_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let set_id = sc_peerset::SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -4328,7 +4338,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_non_existent_connection_closed_for_disabled_pending_enable() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let set_id = sc_peerset::SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -4379,7 +4389,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_connection_closed_for_incoming_peer_state_mismatch() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -4423,7 +4433,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_connection_closed_for_enabled_state_mismatch() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = sc_peerset::SetId::from(0); @@ -4470,7 +4480,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_connection_closed_for_backoff_peer() { - let (mut notif, _peerset) = development_notifs(); + let (mut notif, _peerset, _notif_service) = development_notifs(); let set_id = sc_peerset::SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -4524,8 +4534,8 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn open_result_ok_non_existent_peer() { - let (mut notif, _peerset) = development_notifs(); - let conn = ConnectionId::new_unchecked(0); + let (mut notif, _peerset, _notif_service) = development_notifs(); + let conn = ConnectionId::new(); let connected = ConnectedPoint::Listener { local_addr: Multiaddr::empty(), send_back_addr: Multiaddr::empty(), diff --git a/client/network/src/protocol/notifications/service.rs b/client/network/src/protocol/notifications/service.rs index b240d1ebf7dca..4e746d1c37eff 100644 --- a/client/network/src/protocol/notifications/service.rs +++ b/client/network/src/protocol/notifications/service.rs @@ -34,11 +34,13 @@ use libp2p::PeerId; use tokio::sync::{mpsc, oneshot}; use sc_network_common::role::ObservedRole; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use std::{collections::HashMap, fmt::Debug}; /// Inner notification event to deal with `NotificationsSinks` without exposing that /// implementation detail to [`NotificationService`] consumers. +#[derive(Debug)] enum InnerNotificationEvent { /// Validate inbound substream. ValidateInboundSubstream { @@ -105,7 +107,7 @@ pub struct NotificationHandle { tx: mpsc::Sender, /// RX channel for receiving events from `Notifications`. - rx: mpsc::Receiver, + rx: TracingUnboundedReceiver, /// Connected peers. peers: HashMap, @@ -115,7 +117,7 @@ impl NotificationHandle { /// Create new [`NotificationHandle`]. fn new( tx: mpsc::Sender, - rx: mpsc::Receiver, + rx: TracingUnboundedReceiver, ) -> Self { Self { tx, rx, peers: HashMap::new() } } @@ -170,7 +172,7 @@ impl NotificationService for NotificationHandle { /// Get next event from the `Notifications` event stream. async fn next_event(&mut self) -> Option { - match self.rx.recv().await? { + match self.rx.next().await? { InnerNotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }), InnerNotificationEvent::NotificationStreamOpened { @@ -199,14 +201,14 @@ impl NotificationService for NotificationHandle { /// Channel pair which allows `Notifications` to interact with a protocol. #[derive(Debug)] pub struct ProtocolHandlePair( - mpsc::Sender, + TracingUnboundedSender, mpsc::Receiver, ); impl ProtocolHandlePair { /// Create new [`ProtocolHandlePair`]. fn new( - tx: mpsc::Sender, + tx: TracingUnboundedSender, rx: mpsc::Receiver, ) -> Self { Self(tx, rx) @@ -226,11 +228,11 @@ impl ProtocolHandlePair { #[derive(Debug)] pub struct ProtocolHandle { /// TX channel for sending events to protocol. - tx: mpsc::Sender, + tx: TracingUnboundedSender, } impl ProtocolHandle { - fn new(tx: mpsc::Sender) -> Self { + fn new(tx: TracingUnboundedSender) -> Self { Self { tx } } @@ -239,7 +241,7 @@ impl ProtocolHandle { /// /// Return `oneshot::Receiver` which allows `Notifications` to poll for the validation result /// from protocol. - async fn report_incoming_substream( + pub fn report_incoming_substream( &self, peer: PeerId, handshake: Vec, @@ -247,15 +249,18 @@ impl ProtocolHandle { let (result_tx, rx) = oneshot::channel(); self.tx - .send(InnerNotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) - .await + .unbounded_send(InnerNotificationEvent::ValidateInboundSubstream { + peer, + handshake, + result_tx, + }) .map(|_| rx) .map_err(|_| ()) } /// Report to the protocol that a substream has been opened and that it can now use the handle /// to send notifications to the remote peer. - async fn report_substream_opened( + pub fn report_substream_opened( &self, peer: PeerId, role: ObservedRole, @@ -263,33 +268,30 @@ impl ProtocolHandle { sink: NotificationsSink, ) -> Result<(), ()> { self.tx - .send(InnerNotificationEvent::NotificationStreamOpened { + .unbounded_send(InnerNotificationEvent::NotificationStreamOpened { peer, role, negotiated_fallback, sink, }) - .await .map_err(|_| ()) } /// Substream was closed. - async fn report_substream_closed(&mut self, peer: PeerId) -> Result<(), ()> { + pub fn report_substream_closed(&mut self, peer: PeerId) -> Result<(), ()> { self.tx - .send(InnerNotificationEvent::NotificationStreamClosed { peer }) - .await + .unbounded_send(InnerNotificationEvent::NotificationStreamClosed { peer }) .map_err(|_| ()) } /// Notification was received from the substream. - async fn report_notification_received( + pub fn report_notification_received( &mut self, peer: PeerId, notification: Vec, ) -> Result<(), ()> { self.tx - .send(InnerNotificationEvent::NotificationReceived { peer, notification }) - .await + .unbounded_send(InnerNotificationEvent::NotificationReceived { peer, notification }) .map_err(|_| ()) } } @@ -299,7 +301,7 @@ impl ProtocolHandle { /// Handle pair allows `Notifications` and the protocol to communicate with each other directly. pub fn notification_service() -> (ProtocolHandlePair, Box) { let (cmd_tx, cmd_rx) = mpsc::channel(64); // TODO: zzz - let (event_tx, event_rx) = mpsc::channel(64); // TODO: zzz + let (event_tx, event_rx) = tracing_unbounded("mpsc_transactions_handler", 100_000); (ProtocolHandlePair::new(event_tx, cmd_rx), Box::new(NotificationHandle::new(cmd_tx, event_rx))) } @@ -315,7 +317,7 @@ mod tests { let (mut handle, stream) = proto.split(); let peer_id = PeerId::random(); - let rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + let rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = notif.next_event().await @@ -337,10 +339,7 @@ mod tests { let (mut handle, stream) = proto.split(); let peer_id = PeerId::random(); - handle - .report_substream_opened(peer_id, ObservedRole::Full, None, sink) - .await - .unwrap(); + handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, @@ -364,7 +363,7 @@ mod tests { let peer_id = PeerId::random(); // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = notif.next_event().await @@ -378,10 +377,7 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, ObservedRole::Full, None, sink) - .await - .unwrap(); + handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, @@ -411,7 +407,7 @@ mod tests { let peer_id = PeerId::random(); // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = notif.next_event().await @@ -425,10 +421,7 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, ObservedRole::Full, None, sink) - .await - .unwrap(); + handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, @@ -490,7 +483,7 @@ mod tests { let peer_id = PeerId::random(); // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = notif.next_event().await @@ -504,10 +497,7 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, ObservedRole::Full, None, sink) - .await - .unwrap(); + handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, @@ -523,7 +513,7 @@ mod tests { } // notification is received - handle.report_notification_received(peer_id, vec![1, 3, 3, 8]).await.unwrap(); + handle.report_notification_received(peer_id, vec![1, 3, 3, 8]).unwrap(); if let Some(NotificationEvent::NotificationReceived { peer, notification }) = notif.next_event().await @@ -543,7 +533,7 @@ mod tests { let peer_id = PeerId::random(); // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = notif.next_event().await @@ -557,10 +547,7 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, ObservedRole::Full, None, sink) - .await - .unwrap(); + handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, @@ -608,7 +595,7 @@ mod tests { let peer_id = PeerId::random(); // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = notif.next_event().await @@ -622,10 +609,7 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, ObservedRole::Full, None, sink) - .await - .unwrap(); + handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, @@ -642,7 +626,7 @@ mod tests { // report that a substream has been closed but don't poll `notif` to receive this // information - handle.report_substream_closed(peer_id).await.unwrap(); + handle.report_substream_closed(peer_id).unwrap(); drop(sync_rx); // as per documentation, error is not reported but the notification is silently dropped @@ -657,7 +641,7 @@ mod tests { let peer_id = PeerId::random(); // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).await.unwrap(); + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = notif.next_event().await @@ -671,10 +655,7 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, ObservedRole::Full, None, sink) - .await - .unwrap(); + handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, @@ -691,7 +672,7 @@ mod tests { // report that a substream has been closed but don't poll `notif` to receive this // information - handle.report_substream_closed(peer_id).await.unwrap(); + handle.report_substream_closed(peer_id).unwrap(); drop(async_rx); // as per documentation, error is not reported but the notification is silently dropped diff --git a/client/network/src/protocol/notifications/tests.rs b/client/network/src/protocol/notifications/tests.rs index d13a4fcfa3809..ac11f6f03112a 100644 --- a/client/network/src/protocol/notifications/tests.rs +++ b/client/network/src/protocol/notifications/tests.rs @@ -78,16 +78,21 @@ fn build_nodes() -> (Swarm, Swarm) { reserved_only: false, }], }); + let (protocol_handle_pair, _notif_service) = + crate::protocol::notifications::service::notification_service(); let behaviour = CustomProtoWithAddr { inner: Notifications::new( peerset, - iter::once(ProtocolConfig { - name: "/foo".into(), - fallback_names: Vec::new(), - handshake: Vec::new(), - max_notification_size: 1024 * 1024, - }), + iter::once(( + ProtocolConfig { + name: "/foo".into(), + fallback_names: Vec::new(), + handshake: Vec::new(), + max_notification_size: 1024 * 1024, + }, + protocol_handle_pair, + )), ), addrs: addrs .iter() @@ -331,6 +336,7 @@ fn reconnect_after_disconnect() { } }; + // TODO: rewrite these using `NotificationService` match event { SwarmEvent::Behaviour(NotificationsOut::CustomProtocolOpen { .. }) | SwarmEvent::Behaviour(NotificationsOut::CustomProtocolClosed { .. }) => panic!(), From 8cc85c6eb91a2fe483f906ae52cda9b7016cca48 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Mon, 20 Mar 2023 16:04:16 +0200 Subject: [PATCH 09/43] Send `NotificationEvent`s from `Notifications` The role should not be part of `NotificationStreamOpened` as `Notifications` has no knowledge of that and the protocols that are interested in that information should query it from `Peerset` (when the support for that is added) --- .../src/protocol/notifications/behaviour.rs | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index ca34b263c0e98..ac07ec94831b4 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -43,6 +43,7 @@ use rand::distributions::{Distribution as _, Uniform}; use smallvec::SmallVec; use tokio_stream::StreamMap; +use sc_network_common::role::ObservedRole; use sc_peerset::DropReason; use std::{ @@ -1602,6 +1603,7 @@ impl NetworkBehaviour for Notifications { trace!(target: "sub-libp2p", "PSM <= Incoming({}, {:?}).", peer_id, incoming_id); self.peerset.incoming(set_id, peer_id, incoming_id); + // TODO: report opened substream to the protocol. self.incoming.push(IncomingPeer { peer_id, set_id, @@ -1843,11 +1845,19 @@ impl NetworkBehaviour for Notifications { let event = NotificationsOut::CustomProtocolOpen { peer_id, set_id, - negotiated_fallback, + negotiated_fallback: negotiated_fallback.clone(), received_handshake, notifications_sink: notifications_sink.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); + let _ = self.protocol_handles[protocol_index] + .report_substream_opened( + peer_id, + ObservedRole::Full, /* TODO: this needs to be queried by + * the protocol from `Peerset` */ + negotiated_fallback, + notifications_sink.clone(), + ); } *connec_state = ConnectionState::Open(notifications_sink); } else if let Some((_, connec_state)) = @@ -1992,9 +2002,15 @@ impl NetworkBehaviour for Notifications { peer_id, set_id, ); - let event = NotificationsOut::Notification { peer_id, set_id, message }; - - self.events.push_back(ToSwarm::GenerateEvent(event)); + let event = NotificationsOut::Notification { + peer_id, + set_id, + message: message.clone(), + }; + self.events.push_back(NetworkBehaviourAction::GenerateEvent(event)); + // TODO: error handling + let _ = self.protocol_handles[protocol_index] + .report_notification_received(peer_id, message.to_vec()); } else { trace!( target: "sub-libp2p", From f4f9090d0adfe952270212376498ecae9436c46c Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Mon, 20 Mar 2023 16:03:47 +0200 Subject: [PATCH 10/43] Start using `NotificationService` for transactions protocol --- client/network/transactions/src/lib.rs | 107 +++++++++++-------------- 1 file changed, 45 insertions(+), 62 deletions(-) diff --git a/client/network/transactions/src/lib.rs b/client/network/transactions/src/lib.rs index 7faf2b893a590..f321f0de371b0 100644 --- a/client/network/transactions/src/lib.rs +++ b/client/network/transactions/src/lib.rs @@ -38,9 +38,10 @@ use sc_network::{ config::{NonDefaultSetConfig, NonReservedPeerMode, ProtocolId, SetConfig}, error, event::Event, + service::traits::{NotificationEvent, NotificationService}, types::ProtocolName, utils::{interval, LruHashSet}, - NetworkEventStream, NetworkNotification, NetworkPeers, NotificationService, + NetworkEventStream, NetworkNotification, NetworkPeers, }; use sc_network_common::{ role::ObservedRole, @@ -174,13 +175,12 @@ impl TransactionsHandlerPrototype { transaction_pool: Arc>, metrics_registry: Option<&Registry>, ) -> error::Result<(TransactionsHandler, TransactionsHandlerController)> { - let net_event_stream = network.event_stream("transactions-handler-net"); let sync_event_stream = sync.event_stream("transactions-handler-sync"); let (to_handler, from_controller) = tracing_unbounded("mpsc_transactions_handler", 100_000); let handler = TransactionsHandler { protocol_name: self.protocol_name, - _notification_handle: self.notification_handle, + notification_handle: self.notification_handle, propagate_timeout: (Box::pin(interval(PROPAGATE_TIMEOUT)) as Pin + Send>>) .fuse(), @@ -188,7 +188,6 @@ impl TransactionsHandlerPrototype { pending_transactions_peers: HashMap::new(), network, sync, - net_event_stream: net_event_stream.fuse(), sync_event_stream: sync_event_stream.fuse(), peers: HashMap::new(), transaction_pool, @@ -255,8 +254,6 @@ pub struct TransactionsHandler< network: N, /// Syncing service. sync: S, - /// Stream of networking events. - net_event_stream: stream::Fuse + Send>>>, /// Receiver for syncing-related events. sync_event_stream: stream::Fuse + Send>>>, // All connected peers @@ -266,7 +263,7 @@ pub struct TransactionsHandler< /// Prometheus metrics. metrics: Option, /// Handle that is used to communicate with `sc_network::Notifications`. - _notification_handle: Box, + notification_handle: Box, } /// Peer information @@ -299,14 +296,6 @@ where warn!(target: "sub-libp2p", "Inconsistent state, no peers for pending transaction!"); } }, - network_event = self.net_event_stream.next() => { - if let Some(network_event) = network_event { - self.handle_network_event(network_event).await; - } else { - // Networking has seemingly closed. Closing as well. - return; - } - }, sync_event = self.sync_event_stream.next() => { if let Some(sync_event) = sync_event { self.handle_sync_event(sync_event); @@ -321,10 +310,49 @@ where ToHandler::PropagateTransactions => self.propagate_transactions(), } }, + event = self.notification_handle.next_event().fuse() => { + if let Some(event) = event { + self.handle_notification_event(event) + } else { + // `Notifications` has seemingly closed. Closing as well. + return + } + } } } } + fn handle_notification_event(&mut self, event: NotificationEvent) { + match event { + NotificationEvent::NotificationStreamOpened { peer, role, .. } => { + let _was_in = self.peers.insert( + peer, + Peer { + known_transactions: LruHashSet::new( + NonZeroUsize::new(MAX_KNOWN_TRANSACTIONS).expect("Constant is nonzero"), + ), + role, + }, + ); + debug_assert!(_was_in.is_none()); + }, + NotificationEvent::NotificationStreamClosed { peer } => { + let _peer = self.peers.remove(&peer); + debug_assert!(_peer.is_some()); + }, + NotificationEvent::NotificationReceived { peer, notification } => { + if let Ok(m) = + as Decode>::decode(&mut notification.as_ref()) + { + self.on_transactions(peer, m); + } else { + warn!(target: "sub-libp2p", "Failed to decode transactions list"); + } + }, + event => log::debug!(target: "sync", "ignoring {event:?}"), + } + } + fn handle_sync_event(&mut self, event: SyncEvent) { match event { SyncEvent::PeerConnected(remote) => { @@ -347,51 +375,6 @@ where } } - async fn handle_network_event(&mut self, event: Event) { - match event { - Event::Dht(_) => {}, - Event::NotificationStreamOpened { remote, protocol, role, .. } - if protocol == self.protocol_name => - { - let _was_in = self.peers.insert( - remote, - Peer { - known_transactions: LruHashSet::new( - NonZeroUsize::new(MAX_KNOWN_TRANSACTIONS).expect("Constant is nonzero"), - ), - role, - }, - ); - debug_assert!(_was_in.is_none()); - }, - Event::NotificationStreamClosed { remote, protocol } - if protocol == self.protocol_name => - { - let _peer = self.peers.remove(&remote); - debug_assert!(_peer.is_some()); - }, - - Event::NotificationsReceived { remote, messages } => { - for (protocol, message) in messages { - if protocol != self.protocol_name { - continue - } - - if let Ok(m) = - as Decode>::decode(&mut message.as_ref()) - { - self.on_transactions(remote, m); - } else { - warn!(target: "sub-libp2p", "Failed to decode transactions list"); - } - } - }, - - // Not our concern. - Event::NotificationStreamOpened { .. } | Event::NotificationStreamClosed { .. } => {}, - } - } - /// Called when peer sends us new transactions fn on_transactions(&mut self, who: PeerId, transactions: Transactions) { // Accept transactions only when node is not major syncing @@ -483,8 +466,8 @@ where propagated_to.entry(hash).or_default().push(who.to_base58()); } trace!(target: "sync", "Sending {} transactions to {}", to_send.len(), who); - self.network - .write_notification(*who, self.protocol_name.clone(), to_send.encode()); + // TODO: don't ignore error + let _ = self.notification_handle.send_sync_notification(who, to_send.encode()); } } From 796f8263c12c6df50b9d47c7e99902217c59a3c4 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Wed, 22 Mar 2023 08:48:38 +0200 Subject: [PATCH 11/43] Implement `clone()` for `NotificationService` --- client/network/src/config.rs | 3 +- .../src/protocol/notifications/behaviour.rs | 2 +- .../src/protocol/notifications/handler.rs | 1 - .../src/protocol/notifications/service.rs | 417 +++++++++++++++--- .../src/protocol/notifications/tests.rs | 2 +- client/network/src/service/traits.rs | 4 + 6 files changed, 367 insertions(+), 62 deletions(-) diff --git a/client/network/src/config.rs b/client/network/src/config.rs index edde869f9f18a..57960822ff75f 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -531,7 +531,8 @@ impl NonDefaultSetConfig { handshake: Option, set_config: SetConfig, ) -> (Self, Box) { - let (protocol_handle_pair, notification_handle) = notification_service(); + let (protocol_handle_pair, notification_handle) = + notification_service(protocol_name.clone()); ( Self { protocol_name, diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index ac07ec94831b4..99dd01819e79f 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -2201,7 +2201,7 @@ mod tests { Box, ) { let (protocol_handle_pair, notif_service) = - crate::protocol::notifications::service::notification_service(); + crate::protocol::notifications::service::notification_service("/proto/1".into()); let (peerset, peerset_handle) = { let mut sets = Vec::with_capacity(1); diff --git a/client/network/src/protocol/notifications/handler.rs b/client/network/src/protocol/notifications/handler.rs index 260a38557421a..cee75398c7ec7 100644 --- a/client/network/src/protocol/notifications/handler.rs +++ b/client/network/src/protocol/notifications/handler.rs @@ -402,7 +402,6 @@ impl NotificationsSink { tx.try_send(NotificationsSinkMessage::Notification { message: message.into() }); if result.is_err() { - println!("error happened"); // Cloning the `mpsc::Sender` guarantees the allocation of an extra spot in the // buffer, and therefore `try_send` will succeed. let _result2 = tx.clone().try_send(NotificationsSinkMessage::ForceClose); diff --git a/client/network/src/protocol/notifications/service.rs b/client/network/src/protocol/notifications/service.rs index 4e746d1c37eff..55b5cbe38ddfd 100644 --- a/client/network/src/protocol/notifications/service.rs +++ b/client/network/src/protocol/notifications/service.rs @@ -20,6 +20,8 @@ #![allow(unused)] +// TODO: remove allow(unused) + use crate::{ error, protocol::notifications::handler::{ @@ -29,14 +31,28 @@ use crate::{ types::ProtocolName, }; -use futures::{stream::Stream, SinkExt, StreamExt}; +use futures::{ + stream::{FuturesUnordered, Stream}, + SinkExt, StreamExt, +}; use libp2p::PeerId; use tokio::sync::{mpsc, oneshot}; +use tokio_stream::wrappers::ReceiverStream; use sc_network_common::role::ObservedRole; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; -use std::{collections::HashMap, fmt::Debug}; +use std::{ + collections::HashMap, + fmt::Debug, + sync::{Arc, Mutex}, +}; + +/// Logging target for the file. +const LOG_TARGET: &str = "notification-service"; + +/// Type representing subscribers of a notification protocol. +type Subscribers = Arc>>>; /// Inner notification event to deal with `NotificationsSinks` without exposing that /// implementation detail to [`NotificationService`] consumers. @@ -103,12 +119,18 @@ pub enum NotificationCommand { /// Handle that is passed on to the notifications protocol. #[derive(Debug)] pub struct NotificationHandle { + /// Protocol name. + protocol: ProtocolName, + /// TX channel for sending commands to `Notifications`. tx: mpsc::Sender, /// RX channel for receiving events from `Notifications`. rx: TracingUnboundedReceiver, + /// All subscribers of `NotificationEvent`s. + subscribers: Subscribers, + /// Connected peers. peers: HashMap, } @@ -116,22 +138,24 @@ pub struct NotificationHandle { impl NotificationHandle { /// Create new [`NotificationHandle`]. fn new( + protocol: ProtocolName, tx: mpsc::Sender, rx: TracingUnboundedReceiver, + subscribers: Arc>>>, ) -> Self { - Self { tx, rx, peers: HashMap::new() } + Self { protocol, tx, rx, subscribers, peers: HashMap::new() } } } #[async_trait::async_trait] impl NotificationService for NotificationHandle { /// Instruct `Notifications` to open a new substream for `peer`. - async fn open_substream(&mut self, peer: PeerId) -> Result<(), ()> { + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { todo!("support for opening substreams not implemented yet"); } /// Instruct `Notifications` to close substream for `peer`. - async fn close_substream(&mut self, peer: PeerId) -> Result<(), ()> { + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { todo!("support for closing substreams not implemented yet"); } @@ -141,8 +165,11 @@ impl NotificationService for NotificationHandle { peer: &PeerId, notification: Vec, ) -> Result<(), error::Error> { + log::trace!(target: LOG_TARGET, "{}: send sync notification to {peer:?}", self.protocol); + self.peers .get(&peer) + // TODO: check what the current implementation does in case the peer doesn't exist .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? .send_sync_notification(notification); Ok(()) @@ -154,6 +181,8 @@ impl NotificationService for NotificationHandle { peer: &PeerId, notification: Vec, ) -> Result<(), error::Error> { + log::trace!(target: LOG_TARGET, "{}: send async notification to {peer:?}", self.protocol); + self.peers .get(&peer) // TODO: check what the current implementation does in case the peer doesn't exist @@ -167,6 +196,8 @@ impl NotificationService for NotificationHandle { /// Set handshake for the notification protocol replacing the old handshake. async fn set_hanshake(&mut self, handshake: Vec) -> Result<(), ()> { + log::trace!(target: LOG_TARGET, "{}: set handshake to {handshake:?}", self.protocol); + self.tx.send(NotificationCommand::SetHandshake(handshake)).await.map_err(|_| ()) } @@ -196,22 +227,44 @@ impl NotificationService for NotificationHandle { Some(NotificationEvent::NotificationReceived { peer, notification }), } } + + fn clone(&mut self) -> Result, ()> { + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + let (event_tx, event_rx) = + tracing_unbounded("mpsc-notification-to-protocol-clonable", 100_000); + subscribers.push(event_tx); + + Ok(Box::new(NotificationHandle { + protocol: self.protocol.clone(), + tx: self.tx.clone(), + rx: event_rx, + peers: self.peers.clone(), + subscribers: self.subscribers.clone(), + })) + } } /// Channel pair which allows `Notifications` to interact with a protocol. #[derive(Debug)] -pub struct ProtocolHandlePair( - TracingUnboundedSender, - mpsc::Receiver, -); +pub struct ProtocolHandlePair { + /// Protocol name. + protocol: ProtocolName, + + /// Subscribers of the notification protocol events. + subscribers: Subscribers, + + // Receiver for notification commands received from the protocol implementation. + rx: mpsc::Receiver, +} impl ProtocolHandlePair { /// Create new [`ProtocolHandlePair`]. fn new( - tx: TracingUnboundedSender, + protocol: ProtocolName, + subscribers: Subscribers, rx: mpsc::Receiver, ) -> Self { - Self(tx, rx) + Self { protocol, subscribers, rx } } /// Consume `self` and split [`ProtocolHandlePair`] into a handle which allows it to send events @@ -219,7 +272,10 @@ impl ProtocolHandlePair { pub fn split( self, ) -> (ProtocolHandle, Box + Send + Unpin>) { - (ProtocolHandle::new(self.0), Box::new(tokio_stream::wrappers::ReceiverStream::new(self.1))) + ( + ProtocolHandle::new(self.protocol, self.subscribers), + Box::new(ReceiverStream::new(self.rx)), + ) } } @@ -227,13 +283,16 @@ impl ProtocolHandlePair { /// with the protocol. #[derive(Debug)] pub struct ProtocolHandle { - /// TX channel for sending events to protocol. - tx: TracingUnboundedSender, + /// Protocol name. + protocol: ProtocolName, + + /// Subscribers of the notification protocol. + subscribers: Subscribers, } impl ProtocolHandle { - fn new(tx: TracingUnboundedSender) -> Self { - Self { tx } + fn new(protocol: ProtocolName, subscribers: Subscribers) -> Self { + Self { protocol, subscribers } } /// Report to the protocol that a substream has been opened and it must be validated by the @@ -246,16 +305,60 @@ impl ProtocolHandle { peer: PeerId, handshake: Vec, ) -> Result, ()> { - let (result_tx, rx) = oneshot::channel(); + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; - self.tx - .unbounded_send(InnerNotificationEvent::ValidateInboundSubstream { - peer, - handshake, - result_tx, + log::trace!( + target: LOG_TARGET, + "{}: report incoming substream for {peer}, handshake {handshake:?}", + self.protocol + ); + + // if there is only one subscriber, `Notifications` can wait directly on the + // `oneshot::channel()` RX pair without indirection + if subscribers.len() == 1 { + let (result_tx, rx) = oneshot::channel(); + return subscribers[0] + .unbounded_send(InnerNotificationEvent::ValidateInboundSubstream { + peer, + handshake, + result_tx, + }) + .map(|_| rx) + .map_err(|_| ()) + } + + // if there are multiple subscribers, create a task which waits for + // all of them to finish and returns the combined result to `Notifications` + let mut results: FuturesUnordered<_> = subscribers + .iter() + .filter_map(|subscriber| { + let (result_tx, rx) = oneshot::channel(); + + subscriber + .unbounded_send(InnerNotificationEvent::ValidateInboundSubstream { + peer, + handshake: handshake.clone(), + result_tx, + }) + .is_ok() + .then_some(rx) }) - .map(|_| rx) - .map_err(|_| ()) + .collect(); + + let (tx, rx) = oneshot::channel(); + tokio::spawn(async move { + while let Some(event) = results.next().await { + match event { + Err(_) | Ok(ValidationResult::Reject) => + return tx.send(ValidationResult::Reject), + Ok(ValidationResult::Accept) => {}, + } + } + + return tx.send(ValidationResult::Accept) + }); + + Ok(rx) } /// Report to the protocol that a substream has been opened and that it can now use the handle @@ -267,21 +370,37 @@ impl ProtocolHandle { negotiated_fallback: Option, sink: NotificationsSink, ) -> Result<(), ()> { - self.tx - .unbounded_send(InnerNotificationEvent::NotificationStreamOpened { - peer, - role, - negotiated_fallback, - sink, - }) - .map_err(|_| ()) + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + + log::trace!(target: LOG_TARGET, "{}: substream opened for {peer:?}", self.protocol); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationStreamOpened { + peer, + role: role.clone(), + negotiated_fallback: negotiated_fallback.clone(), + sink: sink.clone(), + }) + .is_ok() + }); + + Ok(()) } /// Substream was closed. pub fn report_substream_closed(&mut self, peer: PeerId) -> Result<(), ()> { - self.tx - .unbounded_send(InnerNotificationEvent::NotificationStreamClosed { peer }) - .map_err(|_| ()) + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + + log::trace!(target: LOG_TARGET, "{}: substream closed for {peer:?}", self.protocol); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationStreamClosed { peer }) + .is_ok() + }); + + Ok(()) } /// Notification was received from the substream. @@ -290,20 +409,37 @@ impl ProtocolHandle { peer: PeerId, notification: Vec, ) -> Result<(), ()> { - self.tx - .unbounded_send(InnerNotificationEvent::NotificationReceived { peer, notification }) - .map_err(|_| ()) + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + + log::trace!(target: LOG_TARGET, "{}: substream closed for {peer:?}", self.protocol); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationReceived { + peer, + notification: notification.clone(), + }) + .is_ok() + }); + + Ok(()) } } /// Create new (protocol, notification) handle pair. /// /// Handle pair allows `Notifications` and the protocol to communicate with each other directly. -pub fn notification_service() -> (ProtocolHandlePair, Box) { +pub fn notification_service( + protocol: ProtocolName, +) -> (ProtocolHandlePair, Box) { let (cmd_tx, cmd_rx) = mpsc::channel(64); // TODO: zzz - let (event_tx, event_rx) = tracing_unbounded("mpsc_transactions_handler", 100_000); + let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); + let subscribers = Arc::new(Mutex::new(vec![event_tx])); - (ProtocolHandlePair::new(event_tx, cmd_rx), Box::new(NotificationHandle::new(cmd_tx, event_rx))) + ( + ProtocolHandlePair::new(protocol.clone(), subscribers.clone(), cmd_rx), + Box::new(NotificationHandle::new(protocol.clone(), cmd_tx, event_rx, subscribers)), + ) } #[cfg(test)] @@ -313,7 +449,7 @@ mod tests { #[tokio::test] async fn validate_and_accept_substream() { - let (proto, mut notif) = notification_service(); + let (proto, mut notif) = notification_service("/proto/1".into()); let (mut handle, stream) = proto.split(); let peer_id = PeerId::random(); @@ -324,7 +460,7 @@ mod tests { { assert_eq!(peer_id, peer); assert_eq!(handshake, vec![1, 3, 3, 7]); - result_tx.send(ValidationResult::Accept); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); } else { panic!("invalid event received"); } @@ -334,7 +470,7 @@ mod tests { #[tokio::test] async fn substream_opened() { - let (proto, mut notif) = notification_service(); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, _) = NotificationsSink::new(PeerId::random()); let (mut handle, stream) = proto.split(); @@ -357,7 +493,7 @@ mod tests { #[tokio::test] async fn send_sync_notification() { - let (proto, mut notif) = notification_service(); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, stream) = proto.split(); let peer_id = PeerId::random(); @@ -370,7 +506,7 @@ mod tests { { assert_eq!(peer_id, peer); assert_eq!(handshake, vec![1, 3, 3, 7]); - result_tx.send(ValidationResult::Accept); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); } else { panic!("invalid event received"); } @@ -401,7 +537,7 @@ mod tests { #[tokio::test] async fn send_async_notification() { - let (proto, mut notif) = notification_service(); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); let (mut handle, stream) = proto.split(); let peer_id = PeerId::random(); @@ -414,7 +550,7 @@ mod tests { { assert_eq!(peer_id, peer); assert_eq!(handshake, vec![1, 3, 3, 7]); - result_tx.send(ValidationResult::Accept); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); } else { panic!("invalid event received"); } @@ -445,7 +581,7 @@ mod tests { #[tokio::test] async fn send_sync_notification_to_non_existent_peer() { - let (proto, mut notif) = notification_service(); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, stream) = proto.split(); let peer = PeerId::random(); @@ -461,7 +597,7 @@ mod tests { #[tokio::test] async fn send_async_notification_to_non_existent_peer() { - let (proto, mut notif) = notification_service(); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, stream) = proto.split(); let peer = PeerId::random(); @@ -477,7 +613,7 @@ mod tests { #[tokio::test] async fn receive_notification() { - let (proto, mut notif) = notification_service(); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, stream) = proto.split(); let peer_id = PeerId::random(); @@ -490,7 +626,7 @@ mod tests { { assert_eq!(peer_id, peer); assert_eq!(handshake, vec![1, 3, 3, 7]); - result_tx.send(ValidationResult::Accept); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); } else { panic!("invalid event received"); } @@ -527,7 +663,7 @@ mod tests { #[tokio::test] async fn backpressure_works() { - let (proto, mut notif) = notification_service(); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); let (mut handle, stream) = proto.split(); let peer_id = PeerId::random(); @@ -540,7 +676,7 @@ mod tests { { assert_eq!(peer_id, peer); assert_eq!(handshake, vec![1, 3, 3, 7]); - result_tx.send(ValidationResult::Accept); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); } else { panic!("invalid event received"); } @@ -589,7 +725,7 @@ mod tests { #[tokio::test] async fn peer_disconnects_then_sync_notification_is_sent() { - let (proto, mut notif) = notification_service(); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, stream) = proto.split(); let peer_id = PeerId::random(); @@ -602,7 +738,7 @@ mod tests { { assert_eq!(peer_id, peer); assert_eq!(handshake, vec![1, 3, 3, 7]); - result_tx.send(ValidationResult::Accept); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); } else { panic!("invalid event received"); } @@ -635,7 +771,7 @@ mod tests { #[tokio::test] async fn peer_disconnects_then_async_notification_is_sent() { - let (proto, mut notif) = notification_service(); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); let (mut handle, stream) = proto.split(); let peer_id = PeerId::random(); @@ -648,7 +784,7 @@ mod tests { { assert_eq!(peer_id, peer); assert_eq!(handshake, vec![1, 3, 3, 7]); - result_tx.send(ValidationResult::Accept); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); } else { panic!("invalid event received"); } @@ -683,4 +819,169 @@ mod tests { panic!("invalid state after calling `send_async_notificatio()` on closed connection") } } + + #[tokio::test] + async fn cloned_service_opening_substream_works() { + let (proto, mut notif1) = notification_service("/proto/1".into()); + let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, stream) = proto.split(); + let mut notif2 = notif1.clone().unwrap(); + let peer_id = PeerId::random(); + + // validate inbound substream + let mut result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + // verify that `stream1` also gets the event + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif1.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + + // verify that because only one listener has thus far send their result, the result is + // pending + assert!(result_rx.try_recv().is_err()); + + // verify that `stream2` also gets the event + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif2.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + result_tx.send(ValidationResult::Accept); + } else { + panic!("invalid event received"); + } + + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + } + + #[tokio::test] + async fn cloned_service_one_service_rejects_substream() { + let (proto, mut notif1) = notification_service("/proto/1".into()); + let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, stream) = proto.split(); + let mut notif2 = notif1.clone().unwrap(); + let mut notif3 = notif2.clone().unwrap(); + let peer_id = PeerId::random(); + + // validate inbound substream + let mut result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + for notif in vec![&mut notif1, &mut notif2] { + if let Some(NotificationEvent::ValidateInboundSubstream { + peer, + handshake, + result_tx, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + } + + // `notif3` has not yet sent their validation result + assert!(result_rx.try_recv().is_err()); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif3.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Reject).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Reject); + } + + #[tokio::test] + async fn cloned_service_opening_substream_sending_and_receiving_notifications_work() { + let (proto, mut notif1) = notification_service("/proto/1".into()); + let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, stream) = proto.split(); + let mut notif2 = notif1.clone().unwrap(); + let mut notif3 = notif1.clone().unwrap(); + let peer_id = PeerId::random(); + + // validate inbound substream + let mut result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + // accept the inbound substream for all services + if let Some(NotificationEvent::ValidateInboundSubstream { + peer, + handshake, + result_tx, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that then notification stream has been opened + handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + } else { + panic!("invalid event received"); + } + } + // receive a notification from peer and verify all services receive it + handle.report_notification_received(peer_id, vec![1, 3, 3, 8]).unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + if let Some(NotificationEvent::NotificationReceived { peer, notification }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(notification, vec![1, 3, 3, 8]); + } else { + panic!("invalid event received"); + } + } + + for (i, notif) in vec![&mut notif1, &mut notif2, &mut notif3].iter().enumerate() { + // send notification from each service and verify peer receives it + notif.send_sync_notification(&peer_id, vec![1, 3, 3, i as u8]); + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, i as u8] }) + ); + } + + // close the substream for peer and verify all services receive the event + handle.report_substream_closed(peer_id).unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + if let Some(NotificationEvent::NotificationStreamClosed { peer }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + } else { + panic!("invalid event received"); + } + } + } } diff --git a/client/network/src/protocol/notifications/tests.rs b/client/network/src/protocol/notifications/tests.rs index ac11f6f03112a..085fcb489977a 100644 --- a/client/network/src/protocol/notifications/tests.rs +++ b/client/network/src/protocol/notifications/tests.rs @@ -79,7 +79,7 @@ fn build_nodes() -> (Swarm, Swarm) { }], }); let (protocol_handle_pair, _notif_service) = - crate::protocol::notifications::service::notification_service(); + crate::protocol::notifications::service::notification_service("/foo".into()); let behaviour = CustomProtoWithAddr { inner: Notifications::new( diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 17aacbe3d3b05..22c4a745b73c4 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -758,4 +758,8 @@ pub trait NotificationService: Debug + Send { /// Get next event from the `Notifications` event stream. async fn next_event(&mut self) -> Option; + + /// Make a copy of the object so it can be shared between protocol components + /// who wish to have access to the same underlying notification protocol. + fn clone(&mut self) -> Result, ()>; } From 00a093ae0eb5eb64f374c709cf3507982504e437 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 11 Apr 2023 14:28:45 +0300 Subject: [PATCH 12/43] Fix `BEEFY`, `NetworkGossip` and `GRANDPA` tests --- Cargo.lock | 1 + bin/node-template/node/src/service.rs | 1 + bin/node/cli/src/service.rs | 1 + client/consensus/beefy/src/lib.rs | 6 +- client/consensus/beefy/src/tests.rs | 15 +- client/consensus/beefy/src/worker.rs | 3 + .../grandpa/src/communication/mod.rs | 4 +- .../grandpa/src/communication/tests.rs | 212 ++++++++++------ client/consensus/grandpa/src/lib.rs | 4 + client/consensus/grandpa/src/observer.rs | 23 +- client/consensus/grandpa/src/tests.rs | 82 ++++++- client/network-gossip/Cargo.toml | 1 + client/network-gossip/src/bridge.rs | 226 +++++++++++------- .../src/protocol/notifications/service.rs | 76 +++--- client/network/src/service/signature.rs | 2 + client/network/src/service/traits.rs | 2 + client/network/sync/src/lib.rs | 2 +- client/network/transactions/src/lib.rs | 1 - 18 files changed, 433 insertions(+), 229 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6464d6ffe3a83..aa7c98292537f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9528,6 +9528,7 @@ name = "sc-network-gossip" version = "0.10.0-dev" dependencies = [ "ahash 0.8.3", + "async-trait", "futures", "futures-timer", "libp2p", diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 3a0f5ee08fcca..748b77ad612eb 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -296,6 +296,7 @@ pub fn new_full(config: Configuration) -> Result { link: grandpa_link, network, sync: Arc::new(sync_service), + notification_handle: grandpa_notification_handle, voting_rule: sc_consensus_grandpa::VotingRulesBuilder::default().build(), prometheus_registry, shared_voter_state: SharedVoterState::empty(), diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 5c99a14627bfc..f64219b689680 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -550,6 +550,7 @@ pub fn new_full_base( link: grandpa_link, network: network.clone(), sync: Arc::new(sync_service.clone()), + notification_handle: grandpa_notification_handle, telemetry: telemetry.as_ref().map(|x| x.handle()), voting_rule: grandpa::VotingRulesBuilder::default().build(), prometheus_registry: prometheus_registry.clone(), diff --git a/client/consensus/beefy/src/lib.rs b/client/consensus/beefy/src/lib.rs index d3e5e4bc68936..965fb7fc9d933 100644 --- a/client/consensus/beefy/src/lib.rs +++ b/client/consensus/beefy/src/lib.rs @@ -38,7 +38,7 @@ use parking_lot::Mutex; use prometheus::Registry; use sc_client_api::{Backend, BlockBackend, BlockchainEvents, FinalityNotifications, Finalizer}; use sc_consensus::BlockImport; -use sc_network::{NetworkRequest, ProtocolName}; +use sc_network::{NetworkRequest, NotificationService, ProtocolName}; use sc_network_gossip::{GossipEngine, Network as GossipNetwork, Syncing as GossipSyncing}; use sp_api::{HeaderT, NumberFor, ProvideRuntimeApi}; use sp_blockchain::{ @@ -180,6 +180,8 @@ pub struct BeefyNetworkParams { pub network: Arc, /// Syncing service implementing a sync oracle and an event stream for peers. pub sync: Arc, + /// Handle for receiving notification events. + pub notification_handle: Box, /// Chain specific BEEFY gossip protocol name. See /// [`communication::beefy_protocol_name::gossip_protocol_name`]. pub gossip_protocol_name: ProtocolName, @@ -245,6 +247,7 @@ pub async fn start_beefy_gadget( let BeefyNetworkParams { network, sync, + notification_handle, gossip_protocol_name, justifications_protocol_name, .. @@ -259,6 +262,7 @@ pub async fn start_beefy_gadget( let mut gossip_engine = GossipEngine::new( network.clone(), sync.clone(), + notification_handle, gossip_protocol_name, gossip_validator.clone(), None, diff --git a/client/consensus/beefy/src/tests.rs b/client/consensus/beefy/src/tests.rs index 288a9fde5b817..3f83117a0052e 100644 --- a/client/consensus/beefy/src/tests.rs +++ b/client/consensus/beefy/src/tests.rs @@ -70,7 +70,7 @@ use substrate_test_runtime_client::{BlockBuilderExt, ClientExt}; use tokio::time::Duration; const GENESIS_HASH: H256 = H256::zero(); -fn beefy_gossip_proto_name() -> ProtocolName { +pub(crate) fn beefy_gossip_proto_name() -> ProtocolName { gossip_protocol_name(GENESIS_HASH, None) } @@ -369,6 +369,7 @@ async fn voter_init_setup( let mut gossip_engine = sc_network_gossip::GossipEngine::new( net.peer(0).network_service().clone(), net.peer(0).sync_service().clone(), + net.peer(0).take_notification_handle(&beefy_gossip_proto_name()).unwrap(), "/beefy/whatever", gossip_validator, None, @@ -389,6 +390,14 @@ where { let tasks = FuturesUnordered::new(); + let mut notification_handles = peers + .iter() + .map(|(peer_id, _, _)| { + let peer = &mut net.peers[*peer_id]; + (*peer_id, peer.take_notification_handle(&beefy_gossip_proto_name()).unwrap()) + }) + .collect::>(); + for (peer_id, key, api) in peers.into_iter() { let peer = &net.peers[peer_id]; @@ -406,6 +415,7 @@ where let network_params = crate::BeefyNetworkParams { network: peer.network_service().clone(), sync: peer.sync_service().clone(), + notification_handle: notification_handles.remove(&peer_id).unwrap(), gossip_protocol_name: beefy_gossip_proto_name(), justifications_protocol_name: on_demand_justif_handler.protocol_name(), _phantom: PhantomData, @@ -1266,7 +1276,7 @@ async fn gossipped_finality_proofs() { let api = Arc::new(TestApi::with_validator_set(&validator_set)); let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect(); - let charlie = &net.peers[2]; + let charlie = &mut net.peers[2]; let known_peers = Arc::new(Mutex::new(KnownPeers::::new())); // Charlie will run just the gossip engine and not the full voter. let (gossip_validator, _) = GossipValidator::new(known_peers); @@ -1279,6 +1289,7 @@ async fn gossipped_finality_proofs() { let mut charlie_gossip_engine = sc_network_gossip::GossipEngine::new( charlie.network_service().clone(), charlie.sync_service().clone(), + charlie.take_notification_handle(&beefy_gossip_proto_name()).unwrap(), beefy_gossip_proto_name(), charlie_gossip_validator.clone(), None, diff --git a/client/consensus/beefy/src/worker.rs b/client/consensus/beefy/src/worker.rs index c05de197d58fd..b87d67d371a0b 100644 --- a/client/consensus/beefy/src/worker.rs +++ b/client/consensus/beefy/src/worker.rs @@ -1089,12 +1089,15 @@ pub(crate) mod tests { let api = Arc::new(TestApi::with_validator_set(&genesis_validator_set)); let network = peer.network_service().clone(); let sync = peer.sync_service().clone(); + let notification_handle = + peer.take_notification_handle(&crate::tests::beefy_gossip_proto_name()).unwrap(); let known_peers = Arc::new(Mutex::new(KnownPeers::new())); let (gossip_validator, gossip_report_stream) = GossipValidator::new(known_peers.clone()); let gossip_validator = Arc::new(gossip_validator); let gossip_engine = GossipEngine::new( network.clone(), sync.clone(), + notification_handle, "/beefy/1", gossip_validator.clone(), None, diff --git a/client/consensus/grandpa/src/communication/mod.rs b/client/consensus/grandpa/src/communication/mod.rs index 9d90035d71cbb..23cc1ecd7d470 100644 --- a/client/consensus/grandpa/src/communication/mod.rs +++ b/client/consensus/grandpa/src/communication/mod.rs @@ -46,7 +46,7 @@ use finality_grandpa::{ Message::{Precommit, Prevote, PrimaryPropose}, }; use parity_scale_codec::{Decode, Encode}; -use sc_network::{NetworkBlock, NetworkSyncForkRequest, ReputationChange}; +use sc_network::{NetworkBlock, NetworkSyncForkRequest, NotificationService, ReputationChange}; use sc_network_gossip::{GossipEngine, Network as GossipNetwork}; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO}; use sp_keystore::KeystorePtr; @@ -247,6 +247,7 @@ impl, S: Syncing> NetworkBridge { pub(crate) fn new( service: N, sync: S, + notification_handle: Box, config: crate::Config, set_state: crate::environment::SharedVoterSetState, prometheus_registry: Option<&Registry>, @@ -260,6 +261,7 @@ impl, S: Syncing> NetworkBridge { let gossip_engine = Arc::new(Mutex::new(GossipEngine::new( service.clone(), sync.clone(), + notification_handle, protocol, validator.clone(), prometheus_registry, diff --git a/client/consensus/grandpa/src/communication/tests.rs b/client/consensus/grandpa/src/communication/tests.rs index f97b1f1e88181..f95559d80a5e7 100644 --- a/client/consensus/grandpa/src/communication/tests.rs +++ b/client/consensus/grandpa/src/communication/tests.rs @@ -28,6 +28,7 @@ use parity_scale_codec::Encode; use sc_network::{ config::{MultiaddrWithPeerId, Role}, event::Event as NetworkEvent, + service::traits::{NotificationEvent, NotificationService}, types::ProtocolName, Multiaddr, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, NetworkSyncForkRequest, NotificationSenderError, NotificationSenderT as NotificationSender, @@ -219,10 +220,65 @@ impl NetworkSyncForkRequest> for TestSync { fn set_sync_fork_request(&self, _peers: Vec, _hash: Hash, _number: NumberFor) {} } +#[derive(Debug)] +pub(crate) struct TestNotificationService { + rx: TracingUnboundedReceiver, +} + +#[async_trait::async_trait] +impl NotificationService for TestNotificationService { + /// Instruct `Notifications` to open a new substream for `peer`. + /// + /// `dial_if_disconnected` informs `Notifications` whether to dial + // the peer if there is currently no active connection to it. + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Instruct `Notifications` to close substream for `peer`. + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification( + &self, + _peer: &PeerId, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + // TODO: this needs to be implemented + unimplemented!(); + } + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + async fn send_async_notification( + &self, + _peer: &PeerId, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_hanshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option { + self.rx.next().await + } + + fn clone(&mut self) -> Result, ()> { + unimplemented!(); + } +} + pub(crate) struct Tester { pub(crate) net_handle: super::NetworkBridge, gossip_validator: Arc>, pub(crate) events: TracingUnboundedReceiver, + pub(crate) notification_tx: TracingUnboundedSender, } impl Tester { @@ -287,6 +343,9 @@ fn voter_set_state() -> SharedVoterSetState { // needs to run in a tokio runtime. pub(crate) fn make_test_network() -> (impl Future, TestNetwork) { let (tx, rx) = tracing_unbounded("test", 100_000); + let (notification_tx, notification_rx) = tracing_unbounded("test-notification", 100_000); + + let notification_handle = TestNotificationService { rx: notification_rx }; let net = TestNetwork { sender: tx }; let sync = TestSync {}; @@ -301,14 +360,22 @@ pub(crate) fn make_test_network() -> (impl Future, TestNetwork) } } - let bridge = - super::NetworkBridge::new(net.clone(), sync, config(), voter_set_state(), None, None); + let bridge = super::NetworkBridge::new( + net.clone(), + sync, + Box::new(notification_handle), + config(), + voter_set_state(), + None, + None, + ); ( futures::future::ready(Tester { gossip_validator: bridge.validator.clone(), net_handle: bridge, events: rx, + notification_tx, }), net, ) @@ -393,63 +460,60 @@ fn good_commit_leads_to_relay() { let commit_to_send = encoded_commit.clone(); let network_bridge = tester.net_handle.clone(); - // asking for global communication will cause the test network - // to send us an event asking us for a stream. use it to - // send a message. + // `NetworkBridge` will be operational as soon as it's created and it's + // waiting for events from the network. Send it events that inform that + // a notification stream was opened and that a notification was received. + // + // since each protocol has its own notification stream, events need not be filtered. let sender_id = id; - let send_message = tester.filter_network_events(move |event| match event { - Event::EventStream(sender) => { - // Add the sending peer and send the commit - let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { - remote: sender_id, - protocol: grandpa_protocol_name::NAME.into(), + + let send_message = async move { + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationStreamOpened { + peer: sender_id, + role: ObservedRole::Full, negotiated_fallback: None, + }, + ); + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationReceived { + peer: sender_id, + notification: commit_to_send.clone(), + }, + ); + + // Add a random peer which will be the recipient of this message + let receiver_id = PeerId::random(); + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationStreamOpened { + peer: receiver_id, role: ObservedRole::Full, - received_handshake: vec![], + negotiated_fallback: None, + }, + ); + + // Announce its local set has being on the current set id through a neighbor + // packet, otherwise it won't be eligible to receive the commit + let _ = { + let update = gossip::VersionedNeighborPacket::V1(gossip::NeighborPacket { + round: Round(round), + set_id: SetId(set_id), + commit_finalized_height: 1, }); - let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived { - remote: sender_id, - messages: vec![( - grandpa_protocol_name::NAME.into(), - commit_to_send.clone().into(), - )], - }); + let msg = gossip::GossipMessage::::Neighbor(update); - // Add a random peer which will be the recipient of this message - let receiver_id = PeerId::random(); - let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { - remote: receiver_id, - protocol: grandpa_protocol_name::NAME.into(), - negotiated_fallback: None, - role: ObservedRole::Full, - received_handshake: vec![], - }); + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationReceived { + peer: receiver_id, + notification: msg.encode(), + }, + ); + }; - // Announce its local set has being on the current set id through a neighbor - // packet, otherwise it won't be eligible to receive the commit - let _ = { - let update = gossip::VersionedNeighborPacket::V1(gossip::NeighborPacket { - round: Round(round), - set_id: SetId(set_id), - commit_finalized_height: 1, - }); - - let msg = gossip::GossipMessage::::Neighbor(update); - - sender.unbounded_send(NetworkEvent::NotificationsReceived { - remote: receiver_id, - messages: vec![( - grandpa_protocol_name::NAME.into(), - msg.encode().into(), - )], - }) - }; - - true - }, - _ => false, - }); + tester + } + .boxed(); // when the commit comes in, we'll tell the callback it was good. let handle_commit = commits_in.into_future().map(|(item, _)| match item.unwrap() { @@ -545,31 +609,31 @@ fn bad_commit_leads_to_report() { let commit_to_send = encoded_commit.clone(); let network_bridge = tester.net_handle.clone(); - // asking for global communication will cause the test network - // to send us an event asking us for a stream. use it to - // send a message. + // `NetworkBridge` will be operational as soon as it's created and it's + // waiting for events from the network. Send it events that inform that + // a notification stream was opened and that a notification was received. + // + // since each protocol has its own notification stream, events need not be filtered. let sender_id = id; - let send_message = tester.filter_network_events(move |event| match event { - Event::EventStream(sender) => { - let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { - remote: sender_id, - protocol: grandpa_protocol_name::NAME.into(), - negotiated_fallback: None, + + let send_message = async move { + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationStreamOpened { + peer: sender_id, role: ObservedRole::Full, - received_handshake: vec![], - }); - let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived { - remote: sender_id, - messages: vec![( - grandpa_protocol_name::NAME.into(), - commit_to_send.clone().into(), - )], - }); + negotiated_fallback: None, + }, + ); + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationReceived { + peer: sender_id, + notification: commit_to_send.clone(), + }, + ); - true - }, - _ => false, - }); + tester + } + .boxed(); // when the commit comes in, we'll tell the callback it was bad. let handle_commit = commits_in.into_future().map(|(item, _)| match item.unwrap() { diff --git a/client/consensus/grandpa/src/lib.rs b/client/consensus/grandpa/src/lib.rs index 90651c3c22cd6..66423d59f5d53 100644 --- a/client/consensus/grandpa/src/lib.rs +++ b/client/consensus/grandpa/src/lib.rs @@ -680,6 +680,8 @@ pub struct GrandpaParams { pub network: N, /// Event stream for syncing-related events. pub sync: S, + /// Handle for interacting with `Notifications`. + pub notification_handle: Box, /// A voting rule used to potentially restrict target votes. pub voting_rule: VR, /// The prometheus metrics registry. @@ -733,6 +735,7 @@ where link, network, sync, + notification_handle, voting_rule, prometheus_registry, shared_voter_state, @@ -758,6 +761,7 @@ where let network = NetworkBridge::new( network, sync, + notification_handle, config.clone(), persistent_data.set_state.clone(), prometheus_registry.as_ref(), diff --git a/client/consensus/grandpa/src/observer.rs b/client/consensus/grandpa/src/observer.rs index 8541baa822bb4..d8503b653a3e2 100644 --- a/client/consensus/grandpa/src/observer.rs +++ b/client/consensus/grandpa/src/observer.rs @@ -28,6 +28,7 @@ use futures::prelude::*; use log::{debug, info, warn}; use sc_client_api::backend::Backend; +use sc_network::NotificationService; use sc_telemetry::TelemetryHandle; use sc_utils::mpsc::TracingUnboundedReceiver; use sp_blockchain::HeaderMetadata; @@ -168,6 +169,7 @@ pub fn run_grandpa_observer( link: LinkHalf, network: N, sync: S, + notification_handle: Box, ) -> sp_blockchain::Result + Send> where BE: Backend + Unpin + 'static, @@ -189,6 +191,7 @@ where let network = NetworkBridge::new( network, sync, + notification_handle, config.clone(), persistent_data.set_state.clone(), None, @@ -414,14 +417,14 @@ mod tests { use futures::executor; - /// Ensure `Future` implementation of `ObserverWork` is polling its `NetworkBridge`. Regression - /// test for bug introduced in d4fbb897c and fixed in b7af8b339. + /// Ensure `Future` implementation of `ObserverWork` is polling its `NetworkBridge`. + /// Regression test for bug introduced in d4fbb897c and fixed in b7af8b339. /// - /// When polled, `NetworkBridge` forwards reputation change requests from the `GossipValidator` - /// to the underlying `dyn Network`. This test triggers a reputation change by calling - /// `GossipValidator::validate` with an invalid gossip message. After polling the `ObserverWork` - /// which should poll the `NetworkBridge`, the reputation change should be forwarded to the test - /// network. + /// When polled, `NetworkBridge` forwards reputation change requests from the + /// `GossipValidator` to the underlying `dyn Network`. This test triggers a reputation change + /// by calling `GossipValidator::validate` with an invalid gossip message. After polling the + /// `ObserverWork` which should poll the `NetworkBridge`, the reputation change should be + /// forwarded to the test network. #[test] fn observer_work_polls_underlying_network_bridge() { // Create a test network. @@ -463,12 +466,6 @@ mod tests { // validator to the test network. assert!(observer.now_or_never().is_none()); - // Ignore initial event stream request by gossip engine. - match tester.events.next().now_or_never() { - Some(Some(Event::EventStream(_))) => {}, - _ => panic!("expected event stream request"), - }; - assert_matches!(tester.events.next().now_or_never(), Some(Some(Event::Report(_, _)))); }); } diff --git a/client/consensus/grandpa/src/tests.rs b/client/consensus/grandpa/src/tests.rs index c46e249be485c..e111948c909f3 100644 --- a/client/consensus/grandpa/src/tests.rs +++ b/client/consensus/grandpa/src/tests.rs @@ -313,6 +313,9 @@ fn initialize_grandpa( (net.peers[peer_id].network_service().clone(), link) }; let sync = net.peers[peer_id].sync_service().clone(); + let notification_handle = net.peers[peer_id] + .take_notification_handle(&grandpa_protocol_name::NAME.into()) + .unwrap(); let grandpa_params = GrandpaParams { config: Config { @@ -328,6 +331,7 @@ fn initialize_grandpa( link, network: net_service, sync, + notification_handle, voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -462,6 +466,9 @@ async fn finalize_3_voters_1_full_observer() { let net_service = net.peers[peer_id].network_service().clone(); let sync = net.peers[peer_id].sync_service().clone(); let link = net.peers[peer_id].data.lock().take().expect("link initialized at startup; qed"); + let notification_handle = net.peers[peer_id] + .take_notification_handle(&grandpa_protocol_name::NAME.into()) + .unwrap(); let grandpa_params = GrandpaParams { config: Config { @@ -477,6 +484,7 @@ async fn finalize_3_voters_1_full_observer() { link, network: net_service, sync, + notification_handle, voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -544,14 +552,17 @@ async fn transition_3_voters_twice_1_full_observer() { for (peer_id, local_key) in all_peers.clone().into_iter().enumerate() { let keystore = create_keystore(local_key); - let (net_service, link, sync) = { - let net = net.lock(); + let (net_service, link, sync, notification_handle) = { + let mut net = net.lock(); let link = net.peers[peer_id].data.lock().take().expect("link initialized at startup; qed"); ( net.peers[peer_id].network_service().clone(), link, net.peers[peer_id].sync_service().clone(), + net.peers[peer_id] + .take_notification_handle(&grandpa_protocol_name::NAME.into()) + .unwrap(), ) }; @@ -569,6 +580,7 @@ async fn transition_3_voters_twice_1_full_observer() { link, network: net_service, sync, + notification_handle, voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1003,6 +1015,9 @@ async fn voter_persists_its_votes() { communication::NetworkBridge::new( net.peers[1].network_service().clone(), net.peers[1].sync_service().clone(), + net.peers[1] + .take_notification_handle(&grandpa_protocol_name::NAME.into()) + .unwrap(), config.clone(), set_state, None, @@ -1021,6 +1036,9 @@ async fn voter_persists_its_votes() { (net.peers[0].network_service().clone(), link) }; let sync = net.peers[0].sync_service().clone(); + let notification_handle = net.peers[0] + .take_notification_handle(&grandpa_protocol_name::NAME.into()) + .unwrap(); let grandpa_params = GrandpaParams { config: Config { @@ -1036,6 +1054,7 @@ async fn voter_persists_its_votes() { link, network: net_service, sync, + notification_handle, voting_rule: VotingRulesBuilder::default().build(), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1057,6 +1076,9 @@ async fn voter_persists_its_votes() { net.add_authority_peer(); let net_service = net.peers[2].network_service().clone(); let sync = net.peers[2].sync_service().clone(); + let notification_handle = net.peers[2] + .take_notification_handle(&grandpa_protocol_name::NAME.into()) + .unwrap(); // but we'll reuse the client from the first peer (alice_voter1) // since we want to share the same database, so that we can // read the persisted state after aborting alice_voter1. @@ -1079,6 +1101,7 @@ async fn voter_persists_its_votes() { link, network: net_service, sync, + notification_handle, voting_rule: VotingRulesBuilder::default().build(), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1227,6 +1250,9 @@ async fn finalize_3_voters_1_light_observer() { let mut net = GrandpaTestNet::new(TestApi::new(voters), 3, 1); let voters = initialize_grandpa(&mut net, authorities); + let notification_service = net.peers[3] + .take_notification_handle(&grandpa_protocol_name::NAME.into()) + .unwrap(); let observer = observer::run_grandpa_observer( Config { gossip_duration: TEST_GOSSIP_DURATION, @@ -1241,6 +1267,7 @@ async fn finalize_3_voters_1_light_observer() { net.peers[3].data.lock().take().expect("link initialized at startup; qed"), net.peers[3].network_service().clone(), net.peers[3].sync_service().clone(), + notification_service, ) .unwrap(); net.peer(0).push_blocks(20, false); @@ -1289,6 +1316,10 @@ async fn voter_catches_up_to_latest_round_when_behind() { link, network: net.peer(peer_id).network_service().clone(), sync: net.peer(peer_id).sync_service().clone(), + notification_handle: net + .peer(peer_id) + .take_notification_handle(&grandpa_protocol_name::NAME.into()) + .unwrap(), voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1378,6 +1409,7 @@ fn test_environment_with_select_chain( keystore: Option, network_service: N, sync_service: S, + notification_service: Box, select_chain: SC, voting_rule: VR, ) -> TestEnvironment @@ -1402,6 +1434,7 @@ where let network = NetworkBridge::new( network_service.clone(), sync_service, + notification_service, config.clone(), set_state.clone(), None, @@ -1430,6 +1463,7 @@ fn test_environment( keystore: Option, network_service: N, sync_service: S, + notification_service: Box, voting_rule: VR, ) -> TestEnvironment, VR> where @@ -1442,6 +1476,7 @@ where keystore, network_service, sync_service, + notification_service, link.select_chain.clone(), voting_rule, ) @@ -1458,14 +1493,22 @@ async fn grandpa_environment_respects_voting_rules() { let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let mut notification_service = + peer.take_notification_handle(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); // add 21 blocks let hashes = peer.push_blocks(21, false); // create an environment with no voting rule restrictions - let unrestricted_env = - test_environment(&link, None, network_service.clone(), sync_service.clone(), ()); + let unrestricted_env = test_environment( + &link, + None, + network_service.clone(), + sync_service.clone(), + notification_service.clone().unwrap(), + (), + ); // another with 3/4 unfinalized chain voting rule restriction let three_quarters_env = test_environment( @@ -1473,6 +1516,7 @@ async fn grandpa_environment_respects_voting_rules() { None, network_service.clone(), sync_service.clone(), + notification_service.clone().unwrap(), voting_rule::ThreeQuartersOfTheUnfinalizedChain, ); @@ -1483,6 +1527,7 @@ async fn grandpa_environment_respects_voting_rules() { None, network_service.clone(), sync_service, + notification_service, VotingRulesBuilder::default().build(), ); @@ -1576,6 +1621,8 @@ async fn grandpa_environment_passes_actual_best_block_to_voting_rules() { let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let notification_service = + peer.take_notification_handle(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let client = peer.client().as_client().clone(); let select_chain = MockSelectChain::default(); @@ -1590,6 +1637,7 @@ async fn grandpa_environment_passes_actual_best_block_to_voting_rules() { None, network_service.clone(), sync_service, + notification_service, select_chain.clone(), voting_rule::BeforeBestBlockBy(5), ); @@ -1637,6 +1685,8 @@ async fn grandpa_environment_checks_if_best_block_is_descendent_of_finality_targ let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let notification_service = + peer.take_notification_handle(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let client = peer.client().as_client().clone(); let select_chain = MockSelectChain::default(); @@ -1646,6 +1696,7 @@ async fn grandpa_environment_checks_if_best_block_is_descendent_of_finality_targ None, network_service.clone(), sync_service.clone(), + notification_service, select_chain.clone(), voting_rule.clone(), ); @@ -1748,11 +1799,19 @@ async fn grandpa_environment_never_overwrites_round_voter_state() { let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let notification_service = + peer.take_notification_handle(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let keystore = create_keystore(peers[0]); - let environment = - test_environment(&link, Some(keystore), network_service.clone(), sync_service, ()); + let environment = test_environment( + &link, + Some(keystore), + network_service.clone(), + sync_service, + notification_service, + (), + ); let round_state = || finality_grandpa::round::State::genesis(Default::default()); let base = || Default::default(); @@ -1954,9 +2013,18 @@ async fn grandpa_environment_doesnt_send_equivocation_reports_for_itself() { let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let notification_service = + peer.take_notification_handle(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let keystore = create_keystore(alice); - test_environment(&link, Some(keystore), network_service.clone(), sync_service, ()) + test_environment( + &link, + Some(keystore), + network_service.clone(), + sync_service, + notification_service, + (), + ) }; let signed_prevote = { diff --git a/client/network-gossip/Cargo.toml b/client/network-gossip/Cargo.toml index 40277c946a1d7..bd7ad2af6446c 100644 --- a/client/network-gossip/Cargo.toml +++ b/client/network-gossip/Cargo.toml @@ -28,6 +28,7 @@ sc-peerset = { version = "4.0.0-dev", path = "../peerset" } sp-runtime = { version = "7.0.0", path = "../../primitives/runtime" } [dev-dependencies] +async-trait = "0.1.57" tokio = "1.22.0" quickcheck = { version = "1.0.3", default-features = false } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 4793d7822ddbe..35ca9f222197a 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -21,7 +21,11 @@ use crate::{ Network, Syncing, Validator, }; -use sc_network::{event::Event, types::ProtocolName}; +use sc_network::{ + service::traits::{NotificationEvent, ValidationResult}, + types::ProtocolName, + NotificationService, +}; use sc_network_common::sync::SyncEvent; use sc_peerset::ReputationChange; @@ -49,10 +53,10 @@ pub struct GossipEngine { periodic_maintenance_interval: futures_timer::Delay, protocol: ProtocolName, - /// Incoming events from the network. - network_event_stream: Pin + Send>>, /// Incoming events from the syncing service. sync_event_stream: Pin + Send>>, + /// Handle for polling notification-related events. + notification_handle: Box, /// Outgoing events to the consumer. message_sinks: HashMap>>, /// Buffered messages (see [`ForwardingState`]). @@ -82,6 +86,7 @@ impl GossipEngine { pub fn new( network: N, sync: S, + notification_handle: Box, protocol: impl Into, validator: Arc>, metrics_registry: Option<&Registry>, @@ -92,17 +97,16 @@ impl GossipEngine { S: Syncing + Send + Clone + 'static, { let protocol = protocol.into(); - let network_event_stream = network.event_stream("network-gossip"); let sync_event_stream = sync.event_stream("network-gossip"); GossipEngine { state_machine: ConsensusGossip::new(validator, protocol.clone(), metrics_registry), network: Box::new(network), sync: Box::new(sync), + notification_handle, periodic_maintenance_interval: futures_timer::Delay::new(PERIODIC_MAINTENANCE_INTERVAL), protocol, - network_event_stream, sync_event_stream, message_sinks: HashMap::new(), forwarding_state: ForwardingState::Idle, @@ -185,46 +189,40 @@ impl Future for GossipEngine { 'outer: loop { match &mut this.forwarding_state { ForwardingState::Idle => { - let net_event_stream = this.network_event_stream.poll_next_unpin(cx); + let next_notification = this.notification_handle.next_event().poll_unpin(cx); let sync_event_stream = this.sync_event_stream.poll_next_unpin(cx); - if net_event_stream.is_pending() && sync_event_stream.is_pending() { + if next_notification.is_pending() && sync_event_stream.is_pending() { break } - match net_event_stream { + match next_notification { Poll::Ready(Some(event)) => match event { - Event::NotificationStreamOpened { remote, protocol, role, .. } => - if protocol == this.protocol { - this.state_machine.new_peer(&mut *this.network, remote, role); - }, - Event::NotificationStreamClosed { remote, protocol } => { - if protocol == this.protocol { - this.state_machine - .peer_disconnected(&mut *this.network, remote); - } + NotificationEvent::ValidateInboundSubstream { + peer, + handshake, + result_tx, + } => { + log::debug!( + target: "gossip", + "accepting inbound substream from {peer}, handshake {handshake:?}" + ); + let _ = result_tx.send(ValidationResult::Accept); }, - Event::NotificationsReceived { remote, messages } => { - let messages = messages - .into_iter() - .filter_map(|(engine, data)| { - if engine == this.protocol { - Some(data.to_vec()) - } else { - None - } - }) - .collect(); - + NotificationEvent::NotificationStreamOpened { peer, role, .. } => { + this.state_machine.new_peer(&mut *this.network, peer, role); + }, + NotificationEvent::NotificationStreamClosed { peer } => { + this.state_machine.peer_disconnected(&mut *this.network, peer); + }, + NotificationEvent::NotificationReceived { peer, notification } => { let to_forward = this.state_machine.on_incoming( &mut *this.network, - remote, - messages, + peer, + vec![notification], ); - this.forwarding_state = ForwardingState::Busy(to_forward.into()); }, - Event::Dht(_) => {}, }, // The network event stream closed. Do the same for [`GossipValidator`]. Poll::Ready(None) => { @@ -333,14 +331,15 @@ mod tests { use super::*; use crate::{multiaddr::Multiaddr, ValidationResult, ValidatorContext}; use futures::{ - channel::mpsc::{unbounded, UnboundedSender}, + channel::mpsc::{unbounded, UnboundedReceiver, UnboundedSender}, executor::{block_on, block_on_stream}, future::poll_fn, }; use quickcheck::{Arbitrary, Gen, QuickCheck}; use sc_network::{ - config::MultiaddrWithPeerId, NetworkBlock, NetworkEventStream, NetworkNotification, - NetworkPeers, NotificationSenderError, NotificationSenderT as NotificationSender, + config::MultiaddrWithPeerId, service::traits::NotificationEvent, Event, NetworkBlock, + NetworkEventStream, NetworkNotification, NetworkPeers, NotificationSenderError, + NotificationSenderT as NotificationSender, NotificationService, }; use sc_network_common::{role::ObservedRole, sync::SyncEventStream}; use sp_runtime::{ @@ -354,14 +353,10 @@ mod tests { use substrate_test_runtime_client::runtime::Block; #[derive(Clone, Default)] - struct TestNetwork { - inner: Arc>, - } + struct TestNetwork {} #[derive(Clone, Default)] - struct TestNetworkInner { - event_senders: Vec>, - } + struct TestNetworkInner {} impl NetworkPeers for TestNetwork { fn set_authorized_peers(&self, _peers: HashSet) { @@ -435,10 +430,7 @@ mod tests { impl NetworkEventStream for TestNetwork { fn event_stream(&self, _name: &'static str) -> Pin + Send>> { - let (tx, rx) = unbounded(); - self.inner.lock().unwrap().event_senders.push(tx); - - Box::pin(rx) + unimplemented!(); } } @@ -510,6 +502,60 @@ mod tests { } } + #[derive(Debug)] + pub(crate) struct TestNotificationService { + rx: UnboundedReceiver, + } + + // TODO: provide implementation + #[async_trait::async_trait] + impl sc_network::service::traits::NotificationService for TestNotificationService { + /// Instruct `Notifications` to open a new substream for `peer`. + /// + /// `dial_if_disconnected` informs `Notifications` whether to dial + // the peer if there is currently no active connection to it. + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Instruct `Notifications` to close substream for `peer`. + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification( + &self, + _peer: &PeerId, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + async fn send_async_notification( + &self, + _peer: &PeerId, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_hanshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option { + self.rx.next().await + } + + fn clone(&mut self) -> Result, ()> { + unimplemented!(); + } + } + struct AllowAll; impl Validator for AllowAll { fn validate( @@ -530,16 +576,19 @@ mod tests { fn returns_when_network_event_stream_closes() { let network = TestNetwork::default(); let sync = Arc::new(TestSync::default()); + let (tx, rx) = unbounded(); + let notification_service = Box::new(TestNotificationService { rx }); let mut gossip_engine = GossipEngine::::new( network.clone(), sync, + notification_service, "/my_protocol", Arc::new(AllowAll {}), None, ); - // Drop network event stream sender side. - drop(network.inner.lock().unwrap().event_senders.pop()); + // drop notification service sender side. + drop(tx); block_on(poll_fn(move |ctx| { if let Poll::Pending = gossip_engine.poll_unpin(ctx) { @@ -559,42 +608,36 @@ mod tests { let remote_peer = PeerId::random(); let network = TestNetwork::default(); let sync = Arc::new(TestSync::default()); + let (mut tx, rx) = unbounded(); + let notification_service = Box::new(TestNotificationService { rx }); let mut gossip_engine = GossipEngine::::new( network.clone(), sync.clone(), + notification_service, protocol.clone(), Arc::new(AllowAll {}), None, ); - let mut event_sender = network.inner.lock().unwrap().event_senders.pop().unwrap(); - // Register the remote peer. - event_sender - .start_send(Event::NotificationStreamOpened { - remote: remote_peer, - protocol: protocol.clone(), - negotiated_fallback: None, - role: ObservedRole::Authority, - received_handshake: vec![], - }) - .expect("Event stream is unbounded; qed."); + tx.send(NotificationEvent::NotificationStreamOpened { + peer: remote_peer, + role: ObservedRole::Authority, + negotiated_fallback: None, + }) + .await + .unwrap(); let messages = vec![vec![1], vec![2]]; - let events = messages - .iter() - .cloned() - .map(|m| Event::NotificationsReceived { - remote: remote_peer, - messages: vec![(protocol.clone(), m.into())], - }) - .collect::>(); // Send first event before subscribing. - event_sender - .start_send(events[0].clone()) - .expect("Event stream is unbounded; qed."); + tx.send(NotificationEvent::NotificationReceived { + peer: remote_peer, + notification: messages[0].clone().into(), + }) + .await + .unwrap(); let mut subscribers = vec![]; for _ in 0..2 { @@ -602,9 +645,12 @@ mod tests { } // Send second event after subscribing. - event_sender - .start_send(events[1].clone()) - .expect("Event stream is unbounded; qed."); + tx.send(NotificationEvent::NotificationReceived { + peer: remote_peer, + notification: messages[1].clone().into(), + }) + .await + .unwrap(); tokio::spawn(gossip_engine); @@ -681,6 +727,8 @@ mod tests { let remote_peer = PeerId::random(); let network = TestNetwork::default(); let sync = Arc::new(TestSync::default()); + let (mut tx, rx) = unbounded(); + let notification_service = Box::new(TestNotificationService { rx }); let num_channels_per_topic = channels.iter().fold( HashMap::new(), @@ -708,6 +756,7 @@ mod tests { let mut gossip_engine = GossipEngine::::new( network.clone(), sync.clone(), + notification_service, protocol.clone(), Arc::new(TestValidator {}), None, @@ -733,22 +782,17 @@ mod tests { } } - let mut event_sender = network.inner.lock().unwrap().event_senders.pop().unwrap(); - // Register the remote peer. - event_sender - .start_send(Event::NotificationStreamOpened { - remote: remote_peer, - protocol: protocol.clone(), - negotiated_fallback: None, - role: ObservedRole::Authority, - received_handshake: vec![], - }) - .expect("Event stream is unbounded; qed."); + tx.start_send(NotificationEvent::NotificationStreamOpened { + peer: remote_peer, + role: ObservedRole::Authority, + negotiated_fallback: None, + }) + .unwrap(); // Send messages into the network event stream. for (i_notification, messages) in notifications.iter().enumerate() { - let messages = messages + let messages: Vec> = messages .into_iter() .enumerate() .map(|(i_message, Message { topic })| { @@ -761,13 +805,17 @@ mod tests { message.push(i_notification.try_into().unwrap()); message.push(i_message.try_into().unwrap()); - (protocol.clone(), message.into()) + message.into() }) .collect(); - event_sender - .start_send(Event::NotificationsReceived { remote: remote_peer, messages }) - .expect("Event stream is unbounded; qed."); + for message in messages { + tx.start_send(NotificationEvent::NotificationReceived { + peer: remote_peer, + notification: message, + }) + .unwrap(); + } } let mut received_msgs_per_topic_all_chan = HashMap::::new(); diff --git a/client/network/src/protocol/notifications/service.rs b/client/network/src/protocol/notifications/service.rs index 55b5cbe38ddfd..8f7e4cc298550 100644 --- a/client/network/src/protocol/notifications/service.rs +++ b/client/network/src/protocol/notifications/service.rs @@ -18,22 +18,16 @@ //! Notification service implementation. -#![allow(unused)] - -// TODO: remove allow(unused) - use crate::{ error, - protocol::notifications::handler::{ - NotificationsSink, NotificationsSinkMessage, ASYNC_NOTIFICATIONS_BUFFER_SIZE, - }, + protocol::notifications::handler::NotificationsSink, service::traits::{NotificationEvent, NotificationService, ValidationResult}, types::ProtocolName, }; use futures::{ stream::{FuturesUnordered, Stream}, - SinkExt, StreamExt, + StreamExt, }; use libp2p::PeerId; use tokio::sync::{mpsc, oneshot}; @@ -305,7 +299,7 @@ impl ProtocolHandle { peer: PeerId, handshake: Vec, ) -> Result, ()> { - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + let subscribers = self.subscribers.lock().map_err(|_| ())?; log::trace!( target: LOG_TARGET, @@ -314,7 +308,7 @@ impl ProtocolHandle { ); // if there is only one subscriber, `Notifications` can wait directly on the - // `oneshot::channel()` RX pair without indirection + // `oneshot::channel()`'s RX half without indirection if subscribers.len() == 1 { let (result_tx, rx) = oneshot::channel(); return subscribers[0] @@ -327,8 +321,8 @@ impl ProtocolHandle { .map_err(|_| ()) } - // if there are multiple subscribers, create a task which waits for - // all of them to finish and returns the combined result to `Notifications` + // if there are multiple subscribers, create a task which waits for all of the + // validations to finish and returns the combined result to `Notifications` let mut results: FuturesUnordered<_> = subscribers .iter() .filter_map(|subscriber| { @@ -411,7 +405,7 @@ impl ProtocolHandle { ) -> Result<(), ()> { let mut subscribers = self.subscribers.lock().map_err(|_| ())?; - log::trace!(target: LOG_TARGET, "{}: substream closed for {peer:?}", self.protocol); + log::trace!(target: LOG_TARGET, "{}: notification received from {peer:?}", self.protocol); subscribers.retain(|subscriber| { subscriber @@ -445,12 +439,14 @@ pub fn notification_service( #[cfg(test)] mod tests { use super::*; - use futures::prelude::*; + use crate::protocol::notifications::handler::{ + NotificationsSinkMessage, ASYNC_NOTIFICATIONS_BUFFER_SIZE, + }; #[tokio::test] async fn validate_and_accept_substream() { let (proto, mut notif) = notification_service("/proto/1".into()); - let (mut handle, stream) = proto.split(); + let (handle, _stream) = proto.split(); let peer_id = PeerId::random(); let rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); @@ -472,7 +468,7 @@ mod tests { async fn substream_opened() { let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, _) = NotificationsSink::new(PeerId::random()); - let (mut handle, stream) = proto.split(); + let (handle, _stream) = proto.split(); let peer_id = PeerId::random(); handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); @@ -495,7 +491,7 @@ mod tests { async fn send_sync_notification() { let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); - let (mut handle, stream) = proto.split(); + let (handle, _stream) = proto.split(); let peer_id = PeerId::random(); // validate inbound substream @@ -528,7 +524,7 @@ mod tests { panic!("invalid event received"); } - notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]).unwrap(); assert_eq!( sync_rx.next().await, Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }) @@ -539,7 +535,7 @@ mod tests { async fn send_async_notification() { let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); - let (mut handle, stream) = proto.split(); + let (handle, _stream) = proto.split(); let peer_id = PeerId::random(); // validate inbound substream @@ -581,9 +577,9 @@ mod tests { #[tokio::test] async fn send_sync_notification_to_non_existent_peer() { - let (proto, mut notif) = notification_service("/proto/1".into()); - let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); - let (mut handle, stream) = proto.split(); + let (proto, notif) = notification_service("/proto/1".into()); + let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); + let (_handle, _stream) = proto.split(); let peer = PeerId::random(); if let Err(error::Error::PeerDoesntExist(peer_id)) = @@ -597,9 +593,9 @@ mod tests { #[tokio::test] async fn send_async_notification_to_non_existent_peer() { - let (proto, mut notif) = notification_service("/proto/1".into()); - let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); - let (mut handle, stream) = proto.split(); + let (proto, notif) = notification_service("/proto/1".into()); + let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); + let (_handle, _stream) = proto.split(); let peer = PeerId::random(); if let Err(error::Error::PeerDoesntExist(peer_id)) = @@ -614,8 +610,8 @@ mod tests { #[tokio::test] async fn receive_notification() { let (proto, mut notif) = notification_service("/proto/1".into()); - let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); - let (mut handle, stream) = proto.split(); + let (sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); // validate inbound substream @@ -665,7 +661,7 @@ mod tests { async fn backpressure_works() { let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); - let (mut handle, stream) = proto.split(); + let (handle, _stream) = proto.split(); let peer_id = PeerId::random(); // validate inbound substream @@ -726,8 +722,8 @@ mod tests { #[tokio::test] async fn peer_disconnects_then_sync_notification_is_sent() { let (proto, mut notif) = notification_service("/proto/1".into()); - let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); - let (mut handle, stream) = proto.split(); + let (sink, _, sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); // validate inbound substream @@ -772,8 +768,8 @@ mod tests { #[tokio::test] async fn peer_disconnects_then_async_notification_is_sent() { let (proto, mut notif) = notification_service("/proto/1".into()); - let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); - let (mut handle, stream) = proto.split(); + let (sink, async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); // validate inbound substream @@ -823,8 +819,8 @@ mod tests { #[tokio::test] async fn cloned_service_opening_substream_works() { let (proto, mut notif1) = notification_service("/proto/1".into()); - let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); - let (mut handle, stream) = proto.split(); + let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); + let (handle, _stream) = proto.split(); let mut notif2 = notif1.clone().unwrap(); let peer_id = PeerId::random(); @@ -852,7 +848,7 @@ mod tests { { assert_eq!(peer_id, peer); assert_eq!(handshake, vec![1, 3, 3, 7]); - result_tx.send(ValidationResult::Accept); + result_tx.send(ValidationResult::Accept).unwrap(); } else { panic!("invalid event received"); } @@ -863,8 +859,8 @@ mod tests { #[tokio::test] async fn cloned_service_one_service_rejects_substream() { let (proto, mut notif1) = notification_service("/proto/1".into()); - let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); - let (mut handle, stream) = proto.split(); + let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); + let (handle, _stream) = proto.split(); let mut notif2 = notif1.clone().unwrap(); let mut notif3 = notif2.clone().unwrap(); let peer_id = PeerId::random(); @@ -906,13 +902,13 @@ mod tests { async fn cloned_service_opening_substream_sending_and_receiving_notifications_work() { let (proto, mut notif1) = notification_service("/proto/1".into()); let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); - let (mut handle, stream) = proto.split(); + let (mut handle, _stream) = proto.split(); let mut notif2 = notif1.clone().unwrap(); let mut notif3 = notif1.clone().unwrap(); let peer_id = PeerId::random(); // validate inbound substream - let mut result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); for notif in vec![&mut notif1, &mut notif2, &mut notif3] { // accept the inbound substream for all services @@ -964,7 +960,7 @@ mod tests { for (i, notif) in vec![&mut notif1, &mut notif2, &mut notif3].iter().enumerate() { // send notification from each service and verify peer receives it - notif.send_sync_notification(&peer_id, vec![1, 3, 3, i as u8]); + notif.send_sync_notification(&peer_id, vec![1, 3, 3, i as u8]).unwrap(); assert_eq!( sync_rx.next().await, Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, i as u8] }) diff --git a/client/network/src/service/signature.rs b/client/network/src/service/signature.rs index 024f60e4c466b..5b2ba6be8cf8d 100644 --- a/client/network/src/service/signature.rs +++ b/client/network/src/service/signature.rs @@ -18,6 +18,8 @@ // // If you read this, you are very thorough, congratulations. +//! Signature-related code + use libp2p::{ identity::{Keypair, PublicKey}, PeerId, diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 22c4a745b73c4..e2585ed7065f7 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -18,6 +18,8 @@ // // If you read this, you are very thorough, congratulations. +//! Traits defined by `sc-network`. + use crate::{ config::MultiaddrWithPeerId, error, diff --git a/client/network/sync/src/lib.rs b/client/network/sync/src/lib.rs index 1ad50aa869fea..89957390e96b9 100644 --- a/client/network/sync/src/lib.rs +++ b/client/network/sync/src/lib.rs @@ -4133,7 +4133,7 @@ mod test { let import_queue = Box::new(sc_consensus::import_queue::mock::MockImportQueueHandle::new()); let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), diff --git a/client/network/transactions/src/lib.rs b/client/network/transactions/src/lib.rs index f321f0de371b0..d4ef9d93f7f06 100644 --- a/client/network/transactions/src/lib.rs +++ b/client/network/transactions/src/lib.rs @@ -37,7 +37,6 @@ use prometheus_endpoint::{register, Counter, PrometheusError, Registry, U64}; use sc_network::{ config::{NonDefaultSetConfig, NonReservedPeerMode, ProtocolId, SetConfig}, error, - event::Event, service::traits::{NotificationEvent, NotificationService}, types::ProtocolName, utils::{interval, LruHashSet}, From a3746dce40f4e1ae212cf2392d113aa301798ad3 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Mon, 24 Apr 2023 12:14:04 +0300 Subject: [PATCH 13/43] Start using `NotificationService` for `SyncingEngine` --- client/network/src/protocol.rs | 286 +++++++++++++++--------------- client/network/sync/src/engine.rs | 245 ++++++++++++++----------- 2 files changed, 281 insertions(+), 250 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 8973593212e85..f412568c4d528 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -500,164 +500,164 @@ impl NetworkBehaviour for Protocol { notifications_sink, negotiated_fallback, } => { - // Set number 0 is hardcoded the default set of peers we sync from. - if set_id == HARDCODED_PEERSETS_SYNC { - // `received_handshake` can be either a `Status` message if received from the - // legacy substream ,or a `BlockAnnouncesHandshake` if received from the block - // announces substream. - match as DecodeAll>::decode_all(&mut &received_handshake[..]) { - Ok(GenericMessage::Status(handshake)) => { - let roles = handshake.roles; - let handshake = BlockAnnouncesHandshake:: { - roles: handshake.roles, - best_number: handshake.best_number, - best_hash: handshake.best_hash, - genesis_hash: handshake.genesis_hash, - }; - - let (tx, rx) = oneshot::channel(); - let _ = self.tx.unbounded_send( - crate::SyncEvent::NotificationStreamOpened { - remote: peer_id, - received_handshake: handshake, - sink: notifications_sink, - tx, - }, - ); - self.sync_substream_validations.push(Box::pin(async move { - match rx.await { - Ok(accepted) => - if accepted { - Ok((peer_id, roles)) - } else { - Err(peer_id) - }, - Err(_) => Err(peer_id), - } - })); - - CustomMessageOutcome::None - }, - Ok(msg) => { - debug!( - target: "sync", - "Expected Status message from {}, but got {:?}", - peer_id, - msg, - ); - self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE); - CustomMessageOutcome::None - }, - Err(err) => { - match as DecodeAll>::decode_all( - &mut &received_handshake[..], - ) { - Ok(handshake) => { - let roles = handshake.roles; - - let (tx, rx) = oneshot::channel(); - let _ = self.tx.unbounded_send( - crate::SyncEvent::NotificationStreamOpened { - remote: peer_id, - received_handshake: handshake, - sink: notifications_sink, - tx, - }, - ); - self.sync_substream_validations.push(Box::pin(async move { - match rx.await { - Ok(accepted) => - if accepted { - Ok((peer_id, roles)) - } else { - Err(peer_id) - }, - Err(_) => Err(peer_id), - } - })); - CustomMessageOutcome::None - }, - Err(err2) => { - log::debug!( - target: "sync", - "Couldn't decode handshake sent by {}: {:?}: {} & {}", - peer_id, - received_handshake, - err, - err2, - ); - self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE); - CustomMessageOutcome::None - }, - } - }, - } - } else { - match ( - Roles::decode_all(&mut &received_handshake[..]), - self.peers.get(&peer_id), - ) { - (Ok(roles), _) => CustomMessageOutcome::NotificationStreamOpened { + // // Set number 0 is hardcoded the default set of peers we sync from. + // if set_id == HARDCODED_PEERSETS_SYNC { + // // `received_handshake` can be either a `Status` message if received from the + // // legacy substream ,or a `BlockAnnouncesHandshake` if received from the block + // // announces substream. + // match as DecodeAll>::decode_all(&mut &received_handshake[..]) { + // Ok(GenericMessage::Status(handshake)) => { + // let roles = handshake.roles; + // let handshake = BlockAnnouncesHandshake:: { + // roles: handshake.roles, + // best_number: handshake.best_number, + // best_hash: handshake.best_hash, + // genesis_hash: handshake.genesis_hash, + // }; + + // let (tx, rx) = oneshot::channel(); + // let _ = self.tx.unbounded_send( + // crate::SyncEvent::NotificationStreamOpened { + // remote: peer_id, + // received_handshake: handshake, + // sink: notifications_sink, + // tx, + // }, + // ); + // self.sync_substream_validations.push(Box::pin(async move { + // match rx.await { + // Ok(accepted) => { + // if accepted { + // Ok((peer_id, roles)) + // } else { + // Err(peer_id) + // } + // }, + // Err(_) => Err(peer_id), + // } + // })); + + // CustomMessageOutcome::None + // }, + // Ok(msg) => { + // debug!( + // target: "sync", + // "Expected Status message from {}, but got {:?}", + // peer_id, + // msg, + // ); + // self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE); + // CustomMessageOutcome::None + // }, + // Err(err) => { + // match as DecodeAll>::decode_all( + // &mut &received_handshake[..], + // ) { + // Ok(handshake) => { + // let roles = handshake.roles; + + // let (tx, rx) = oneshot::channel(); + // let _ = self.tx.unbounded_send( + // crate::SyncEvent::NotificationStreamOpened { + // remote: peer_id, + // received_handshake: handshake, + // sink: notifications_sink, + // tx, + // }, + // ); + // self.sync_substream_validations.push(Box::pin(async move { + // match rx.await { + // Ok(accepted) => { + // if accepted { + // Ok((peer_id, roles)) + // } else { + // Err(peer_id) + // } + // }, + // Err(_) => Err(peer_id), + // } + // })); + // CustomMessageOutcome::None + // }, + // Err(err2) => { + // log::debug!( + // target: "sync", + // "Couldn't decode handshake sent by {}: {:?}: {} & {}", + // peer_id, + // received_handshake, + // err, + // err2, + // ); + // self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE); + // CustomMessageOutcome::None + // }, + // } + // }, + // } + // } else { + match (Roles::decode_all(&mut &received_handshake[..]), self.peers.get(&peer_id)) { + (Ok(roles), _) => CustomMessageOutcome::NotificationStreamOpened { + remote: peer_id, + protocol: self.notification_protocols[usize::from(set_id)].clone(), + negotiated_fallback, + roles, + received_handshake, + notifications_sink, + }, + (Err(_), Some(roles)) if received_handshake.is_empty() => { + // As a convenience, we allow opening substreams for "external" + // notification protocols with an empty handshake. This fetches the + // roles from the locally-known roles. + // TODO: remove this after https://github.com/paritytech/substrate/issues/5685 + CustomMessageOutcome::NotificationStreamOpened { remote: peer_id, protocol: self.notification_protocols[usize::from(set_id)].clone(), negotiated_fallback, - roles, + roles: *roles, received_handshake, notifications_sink, - }, - (Err(_), Some(roles)) if received_handshake.is_empty() => { - // As a convenience, we allow opening substreams for "external" - // notification protocols with an empty handshake. This fetches the - // roles from the locally-known roles. - // TODO: remove this after https://github.com/paritytech/substrate/issues/5685 - CustomMessageOutcome::NotificationStreamOpened { - remote: peer_id, - protocol: self.notification_protocols[usize::from(set_id)].clone(), - negotiated_fallback, - roles: *roles, - received_handshake, - notifications_sink, - } - }, - (Err(err), _) => { - debug!(target: "sync", "Failed to parse remote handshake: {}", err); - self.bad_handshake_substreams.insert((peer_id, set_id)); - self.behaviour.disconnect_peer(&peer_id, set_id); - self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE); - self.peers.remove(&peer_id); - CustomMessageOutcome::None - }, - } + } + }, + (Err(err), _) => { + debug!(target: "sync", "Failed to parse remote handshake: {}", err); + self.bad_handshake_substreams.insert((peer_id, set_id)); + self.behaviour.disconnect_peer(&peer_id, set_id); + self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE); + self.peers.remove(&peer_id); + CustomMessageOutcome::None + }, } + // } }, - NotificationsOut::CustomProtocolReplaced { peer_id, notifications_sink, set_id } => + NotificationsOut::CustomProtocolReplaced { peer_id, notifications_sink, set_id } => { if self.bad_handshake_substreams.contains(&(peer_id, set_id)) { CustomMessageOutcome::None - } else if set_id == HARDCODED_PEERSETS_SYNC { - let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationSinkReplaced { - remote: peer_id, - sink: notifications_sink, - }); - CustomMessageOutcome::None + // } else if set_id == HARDCODED_PEERSETS_SYNC { + // let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationSinkReplaced { + // remote: peer_id, + // sink: notifications_sink, + // }); + // CustomMessageOutcome::None } else { CustomMessageOutcome::NotificationStreamReplaced { remote: peer_id, protocol: self.notification_protocols[usize::from(set_id)].clone(), notifications_sink, } - }, + } + }, NotificationsOut::CustomProtocolClosed { peer_id, set_id } => { if self.bad_handshake_substreams.remove(&(peer_id, set_id)) { // The substream that has just been closed had been opened with a bad // handshake. The outer layers have never received an opening event about this // substream, and consequently shouldn't receive a closing event either. CustomMessageOutcome::None - } else if set_id == HARDCODED_PEERSETS_SYNC { - let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationStreamClosed { - remote: peer_id, - }); - self.peers.remove(&peer_id); - CustomMessageOutcome::None + // } else if set_id == HARDCODED_PEERSETS_SYNC { + // let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationStreamClosed { + // remote: peer_id, + // }); + // self.peers.remove(&peer_id); + // CustomMessageOutcome::None } else { CustomMessageOutcome::NotificationStreamClosed { remote: peer_id, @@ -668,12 +668,12 @@ impl NetworkBehaviour for Protocol { NotificationsOut::Notification { peer_id, set_id, message } => { if self.bad_handshake_substreams.contains(&(peer_id, set_id)) { CustomMessageOutcome::None - } else if set_id == HARDCODED_PEERSETS_SYNC { - let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationsReceived { - remote: peer_id, - messages: vec![message.freeze()], - }); - CustomMessageOutcome::None + // } else if set_id == HARDCODED_PEERSETS_SYNC { + // let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationsReceived { + // remote: peer_id, + // messages: vec![message.freeze()], + // }); + // CustomMessageOutcome::None } else { let protocol_name = self.notification_protocols[usize::from(set_id)].clone(); CustomMessageOutcome::NotificationsReceived { diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index 2b8c7a8e986ab..1ec98f49bf550 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -24,7 +24,7 @@ use crate::{ ChainSync, ClientError, SyncingService, }; -use codec::{Decode, Encode}; +use codec::{Decode, DecodeAll, Encode}; use futures::{FutureExt, StreamExt}; use futures_timer::Delay; use libp2p::PeerId; @@ -39,9 +39,10 @@ use sc_network::{ config::{ FullNetworkConfiguration, NonDefaultSetConfig, ProtocolId, SyncMode as SyncOperationMode, }, + service::traits::{NotificationEvent, ValidationResult}, types::ProtocolName, utils::LruHashSet, - NotificationService, NotificationsSink, + NotificationService, }; use sc_network_common::{ role::Roles, @@ -77,6 +78,9 @@ const TICK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(1100) /// Maximum number of known block hashes to keep for a peer. const MAX_KNOWN_BLOCKS: usize = 1024; // ~32kb per peer + LruHashSet overhead +/// Logging target for the file. +const LOG_TARGET: &str = "sync"; + /// If the block announces stream to peer has been inactive for two minutes meaning local node /// has not sent or received block announcements to/from the peer, report the node for inactivity, /// disconnect it and attempt to establish connection to some other peer. @@ -167,8 +171,6 @@ pub struct Peer { pub info: ExtendedPeerInfo, /// Holds a set of blocks known to this peer. pub known_blocks: LruHashSet, - /// Notification sink. - sink: NotificationsSink, /// Instant when the last notification was sent to peer. last_notification_sent: Instant, /// Instant when the last notification was received from peer. @@ -246,7 +248,13 @@ pub struct SyncingEngine { metrics: Option, /// Handle that is used to communicate with `sc_network::Notifications`. - _notification_handle: Box, + notification_handle: Box, + + /// Peers that are on probation, waiting for the inbound substream to be accepted + /// + /// These peers have been validated by `SyncingEngine` and are considered "accepted" + /// and they have a slot reserved for them. TODO: finish this comment + probation: HashMap>, } impl SyncingEngine @@ -342,7 +350,7 @@ where total.saturating_sub(net_config.network_config.default_peers_set_num_full) as usize }; - let (chain_sync, block_announce_config, _notification_handle) = ChainSync::new( + let (chain_sync, block_announce_config, notification_handle) = ChainSync::new( mode, client.clone(), protocol_id, @@ -392,8 +400,9 @@ where default_peers_set_num_full, default_peers_set_num_light, event_streams: Vec::new(), - _notification_handle, + notification_handle, tick_timeout: Delay::new(TICK_TIMEOUT), + probation: HashMap::new(), metrics: if let Some(r) = metrics_registry { match Metrics::register(r, is_major_syncing.clone()) { Ok(metrics) => Some(metrics), @@ -594,7 +603,7 @@ where }; peer.last_notification_sent = Instant::now(); - peer.sink.send_sync_notification(message.encode()); + let _ = self.notification_handle.send_sync_notification(who, message.encode()); } } } @@ -729,63 +738,69 @@ where } } - while let Poll::Ready(Some(event)) = self.rx.poll_next_unpin(cx) { + loop { + let Poll::Ready(Some(event)) = self.notification_handle.next_event().poll_unpin(cx) else { + break; + }; + match event { - sc_network::SyncEvent::NotificationStreamOpened { - remote, - received_handshake, - sink, - tx, - } => match self.on_sync_peer_connected(remote, &received_handshake, sink) { - Ok(()) => { - let _ = tx.send(true); - }, - Err(()) => { + NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => { + let validation_result = self.validate_handshake(&peer, handshake).map_or( + ValidationResult::Reject, + |handshake| { + // TODO: reserve slot for the peer + // TODO: save `Instant::now()` to `probation` and define a constant that + // tells how long the peer is kept on probation before it's + // evicted. TODO: when `ValidateInboundSubstream` is received for a new + // peer and all slots are full, check if a peer can be evicted + // from `probation` TODO: save peer role to `PeerStore` + self.probation.insert(peer, handshake); + ValidationResult::Accept + }, + ); + let _ = result_tx.send(validation_result); + }, + NotificationEvent::NotificationStreamOpened { peer, .. } => { + let Some(handshake) = self.probation.remove(&peer) else { log::debug!( - target: "sync", - "Failed to register peer {remote:?}: {received_handshake:?}", + target: LOG_TARGET, + "{peer} has not been validated by `SyncingEngine` or it took too long to open a substream" ); - let _ = tx.send(false); - }, + continue + }; + + match self.on_sync_peer_connected(peer, &handshake) { + Ok(_) => { + // TODO: remove slot reservation and mark the slot as occupied + }, + Err(_) => { + // TODO: remove slot reservation and mark the slot as free + }, + } + }, + NotificationEvent::NotificationStreamClosed { peer } => { + self.on_sync_peer_disconnected(peer); }, - sc_network::SyncEvent::NotificationStreamClosed { remote } => { - self.evicted.remove(&remote); - if self.on_sync_peer_disconnected(remote).is_err() { + NotificationEvent::NotificationReceived { peer, notification } => { + if !self.peers.contains_key(&peer) { log::trace!( - target: "sync", - "Disconnected peer which had earlier been refused by on_sync_peer_connected {}", - remote + target: LOG_TARGET, + "seceived notification from {peer} who had been earlier refused by `SyncingEngine`", ); + continue } - }, - sc_network::SyncEvent::NotificationsReceived { remote, messages } => { - for message in messages { - if self.peers.contains_key(&remote) { - if let Ok(announce) = BlockAnnounce::decode(&mut message.as_ref()) { - self.push_block_announce_validation(remote, announce); - - // Make sure that the newly added block announce validation future - // was polled once to be registered in the task. - if let Poll::Ready(res) = - self.chain_sync.poll_block_announce_validation(cx) - { - self.process_block_announce_validation_result(res) - } - } else { - log::warn!(target: "sub-libp2p", "Failed to decode block announce"); - } - } else { - log::trace!( - target: "sync", - "Received sync for peer earlier refused by sync layer: {}", - remote - ); - } - } - }, - sc_network::SyncEvent::NotificationSinkReplaced { remote, sink } => { - if let Some(peer) = self.peers.get_mut(&remote) { - peer.sink = sink; + + let Ok(announce) = BlockAnnounce::decode(&mut notification.as_ref()) else { + log::warn!(target: LOG_TARGET, "failed to decode block announce"); + continue + }; + + // push the block announcement for validation and make sure it's polled + // once to register it in the task. + self.push_block_announce_validation(peer, announce); + + if let Poll::Ready(res) = self.chain_sync.poll_block_announce_validation(cx) { + self.process_block_announce_validation_result(res) } }, } @@ -804,94 +819,110 @@ where /// Called by peer when it is disconnecting. /// /// Returns a result if the handshake of this peer was indeed accepted. - pub fn on_sync_peer_disconnected(&mut self, peer: PeerId) -> Result<(), ()> { - if self.peers.remove(&peer).is_some() { - if self.important_peers.contains(&peer) { - log::warn!(target: "sync", "Reserved peer {} disconnected", peer); - } else { - log::debug!(target: "sync", "{} disconnected", peer); - } + pub fn on_sync_peer_disconnected(&mut self, peer: PeerId) { + if !self.peers.remove(&peer).is_some() { + log::debug!(target: LOG_TARGET, "{peer} does not exist in `SyncingEngine`"); + return + } - self.chain_sync.peer_disconnected(&peer); - self.default_peers_set_no_slot_connected_peers.remove(&peer); - self.event_streams - .retain(|stream| stream.unbounded_send(SyncEvent::PeerDisconnected(peer)).is_ok()); - Ok(()) + if self.important_peers.contains(&peer) { + log::warn!(target: "sync", "Reserved peer {} disconnected", peer); } else { - Err(()) + log::debug!(target: "sync", "{} disconnected", peer); } + + self.chain_sync.peer_disconnected(&peer); + // TODO: remove this bookkeeping when `ProtocolController` is ready + self.default_peers_set_no_slot_connected_peers.remove(&peer); + self.event_streams + .retain(|stream| stream.unbounded_send(SyncEvent::PeerDisconnected(peer)).is_ok()); } - /// Called on the first connection between two peers on the default set, after their exchange - /// of handshake. - /// - /// Returns `Ok` if the handshake is accepted and the peer added to the list of peers we sync - /// from. - pub fn on_sync_peer_connected( + fn validate_handshake( &mut self, - who: PeerId, - status: &BlockAnnouncesHandshake, - sink: NotificationsSink, - ) -> Result<(), ()> { - log::trace!(target: "sync", "New peer {} {:?}", who, status); - - if self.peers.contains_key(&who) { - log::error!(target: "sync", "Called on_sync_peer_connected with already connected peer {}", who); + peer: &PeerId, + handshake: Vec, + ) -> Result, ()> { + log::trace!(target: LOG_TARGET, "New peer {peer} {handshake:?}"); + + let handshake = as DecodeAll>::decode_all(&mut &handshake[..]) + .map_err(|error| { + log::debug!(target: LOG_TARGET, "failed to decode handshake for {peer}: {error:?}"); + })?; + + if self.peers.contains_key(&peer) { + log::error!( + target: LOG_TARGET, + "Called on_sync_peer_connected with already connected peer {peer}", + ); debug_assert!(false); return Err(()) } - if status.genesis_hash != self.genesis_hash { - self.network_service.report_peer(who, rep::GENESIS_MISMATCH); + if handshake.genesis_hash != self.genesis_hash { + // TODO: report peer but verify that `PeerStore` doesn't try to disconnect it + // self.network_service.report_peer(peer, rep::GENESIS_MISMATCH); - if self.important_peers.contains(&who) { + if self.important_peers.contains(&peer) { log::error!( - target: "sync", - "Reserved peer id `{}` is on a different chain (our genesis: {} theirs: {})", - who, + target: LOG_TARGET, + "Reserved peer id `{peer}` is on a different chain (our genesis: {} theirs: {})", self.genesis_hash, - status.genesis_hash, + handshake.genesis_hash, ); - } else if self.boot_node_ids.contains(&who) { + } else if self.boot_node_ids.contains(&peer) { log::error!( - target: "sync", - "Bootnode with peer id `{}` is on a different chain (our genesis: {} theirs: {})", - who, + target: LOG_TARGET, + "Bootnode with peer id `{peer}` is on a different chain (our genesis: {} theirs: {})", self.genesis_hash, - status.genesis_hash, + handshake.genesis_hash, ); } else { log::debug!( - target: "sync", + target: LOG_TARGET, "Peer is on different chain (our genesis: {} theirs: {})", - self.genesis_hash, status.genesis_hash + self.genesis_hash, + handshake.genesis_hash ); } return Err(()) } - let no_slot_peer = self.default_peers_set_no_slot_peers.contains(&who); + let no_slot_peer = self.default_peers_set_no_slot_peers.contains(&peer); let this_peer_reserved_slot: usize = if no_slot_peer { 1 } else { 0 }; - if status.roles.is_full() && + if handshake.roles.is_full() && self.chain_sync.num_peers() >= self.default_peers_set_num_full + self.default_peers_set_no_slot_connected_peers.len() + this_peer_reserved_slot { - log::debug!(target: "sync", "Too many full nodes, rejecting {}", who); + log::debug!(target: LOG_TARGET, "Too many full nodes, rejecting {peer}"); return Err(()) } - if status.roles.is_light() && + if handshake.roles.is_light() && (self.peers.len() - self.chain_sync.num_peers()) >= self.default_peers_set_num_light { // Make sure that not all slots are occupied by light clients. - log::debug!(target: "sync", "Too many light nodes, rejecting {}", who); + log::debug!(target: LOG_TARGET, "Too many light nodes, rejecting {peer}"); return Err(()) } + Ok(handshake) + } + + /// Called on the first connection between two peers on the default set, after their exchange + /// of handshake. + /// + /// Returns `Ok` if the handshake is accepted and the peer added to the list of peers we sync + /// from. + pub fn on_sync_peer_connected( + &mut self, + who: PeerId, + status: &BlockAnnouncesHandshake, + ) -> Result<(), ()> { let peer = Peer { info: ExtendedPeerInfo { roles: status.roles, @@ -901,7 +932,6 @@ where known_blocks: LruHashSet::new( NonZeroUsize::new(MAX_KNOWN_BLOCKS).expect("Constant is nonzero"), ), - sink, last_notification_sent: Instant::now(), last_notification_received: Instant::now(), }; @@ -918,10 +948,11 @@ where None }; - log::debug!(target: "sync", "Connected {}", who); + log::debug!(target: LOG_TARGET, "connected {}", who); self.peers.insert(who, peer); - if no_slot_peer { + // TODO: remove these bookkeepings once `ProtocolController` is ready + if self.default_peers_set_no_slot_peers.contains(&who) { self.default_peers_set_no_slot_connected_peers.insert(who); } From 15d093bfa0d9d8423d9416189b44a68669d5e2fb Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 25 Apr 2023 17:49:37 +0300 Subject: [PATCH 14/43] Validate inbound substream before emitting `NotificationStreamOpened` --- .../grandpa/src/communication/tests.rs | 3 + client/network-gossip/src/bridge.rs | 2 + client/network/Cargo.toml | 2 +- client/network/src/protocol.rs | 202 ++----- .../src/protocol/notifications/behaviour.rs | 508 +++++++++++++----- .../src/protocol/notifications/handler.rs | 38 +- .../src/protocol/notifications/service.rs | 81 ++- .../src/protocol/notifications/tests.rs | 1 + client/network/src/service/traits.rs | 6 + client/network/sync/src/engine.rs | 70 +-- client/network/test/src/service.rs | 367 ++++++------- client/network/transactions/src/lib.rs | 6 +- 12 files changed, 760 insertions(+), 526 deletions(-) diff --git a/client/consensus/grandpa/src/communication/tests.rs b/client/consensus/grandpa/src/communication/tests.rs index f95559d80a5e7..3929f21ee8a5a 100644 --- a/client/consensus/grandpa/src/communication/tests.rs +++ b/client/consensus/grandpa/src/communication/tests.rs @@ -473,6 +473,7 @@ fn good_commit_leads_to_relay() { peer: sender_id, role: ObservedRole::Full, negotiated_fallback: None, + handshake: vec![1, 3, 3, 7], }, ); let _ = tester.notification_tx.unbounded_send( @@ -489,6 +490,7 @@ fn good_commit_leads_to_relay() { peer: receiver_id, role: ObservedRole::Full, negotiated_fallback: None, + handshake: vec![1, 3, 3, 7], }, ); @@ -622,6 +624,7 @@ fn bad_commit_leads_to_report() { peer: sender_id, role: ObservedRole::Full, negotiated_fallback: None, + handshake: vec![1, 3, 3, 7], }, ); let _ = tester.notification_tx.unbounded_send( diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 35ca9f222197a..8382ec4297c19 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -625,6 +625,7 @@ mod tests { peer: remote_peer, role: ObservedRole::Authority, negotiated_fallback: None, + handshake: vec![1, 3, 3, 7], }) .await .unwrap(); @@ -787,6 +788,7 @@ mod tests { peer: remote_peer, role: ObservedRole::Authority, negotiated_fallback: None, + handshake: vec![1, 3, 3, 7], }) .unwrap(); diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index f2f1e5b2ea5fd..e833f89024085 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -38,7 +38,7 @@ serde_json = "1.0.85" smallvec = "1.8.0" thiserror = "1.0" tokio = { version = "1.22.0", features = ["macros", "sync"] } -tokio-stream = "0.1.12" +tokio-stream = "0.1.7" unsigned-varint = { version = "0.7.1", features = ["futures", "asynchronous_codec"] } zeroize = "1.4.3" prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.10.0-dev", path = "../../utils/prometheus" } diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index f412568c4d528..fded364ef7146 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -24,7 +24,7 @@ use crate::{ use bytes::Bytes; use codec::{DecodeAll, Encode}; -use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; +use futures::{stream::FuturesUnordered, StreamExt}; use libp2p::{ core::Endpoint, swarm::{ @@ -35,7 +35,7 @@ use libp2p::{ }; use log::{debug, error, warn}; -use sc_network_common::{role::Roles, sync::message::BlockAnnouncesHandshake}; +use sc_network_common::role::Roles; use sc_utils::mpsc::TracingUnboundedSender; use sp_runtime::traits::Block as BlockT; @@ -47,7 +47,6 @@ use std::{ task::Poll, }; -use message::{generic::Message as GenericMessage, Message}; use notifications::{Notifications, NotificationsOut}; pub use notifications::{ @@ -95,7 +94,6 @@ pub struct Protocol { /// Connected peers. peers: HashMap, sync_substream_validations: FuturesUnordered, - tx: TracingUnboundedSender>, _marker: std::marker::PhantomData, } @@ -106,7 +104,7 @@ impl Protocol { network_config: &config::NetworkConfiguration, notification_protocols: Vec, mut block_announces_protocol: config::NonDefaultSetConfig, - tx: TracingUnboundedSender>, + _tx: TracingUnboundedSender>, ) -> error::Result<(Self, sc_peerset::PeersetHandle, Vec<(PeerId, Multiaddr)>)> { let mut known_addresses = Vec::new(); @@ -213,7 +211,6 @@ impl Protocol { bad_handshake_substreams: Default::default(), peers: HashMap::new(), sync_substream_validations: FuturesUnordered::new(), - tx, // TODO: remove when `BlockAnnouncesHandshake` is moved away from `Protocol` _marker: Default::default(), }; @@ -249,7 +246,7 @@ impl Protocol { /// Returns the number of peers we're connected to. pub fn num_connected_peers(&self) -> usize { - self.peers.len() + self.behaviour.num_connected_peers(HARDCODED_PEERSETS_SYNC) } /// Adjusts the reputation of a node. @@ -500,144 +497,55 @@ impl NetworkBehaviour for Protocol { notifications_sink, negotiated_fallback, } => { - // // Set number 0 is hardcoded the default set of peers we sync from. - // if set_id == HARDCODED_PEERSETS_SYNC { - // // `received_handshake` can be either a `Status` message if received from the - // // legacy substream ,or a `BlockAnnouncesHandshake` if received from the block - // // announces substream. - // match as DecodeAll>::decode_all(&mut &received_handshake[..]) { - // Ok(GenericMessage::Status(handshake)) => { - // let roles = handshake.roles; - // let handshake = BlockAnnouncesHandshake:: { - // roles: handshake.roles, - // best_number: handshake.best_number, - // best_hash: handshake.best_hash, - // genesis_hash: handshake.genesis_hash, - // }; - - // let (tx, rx) = oneshot::channel(); - // let _ = self.tx.unbounded_send( - // crate::SyncEvent::NotificationStreamOpened { - // remote: peer_id, - // received_handshake: handshake, - // sink: notifications_sink, - // tx, - // }, - // ); - // self.sync_substream_validations.push(Box::pin(async move { - // match rx.await { - // Ok(accepted) => { - // if accepted { - // Ok((peer_id, roles)) - // } else { - // Err(peer_id) - // } - // }, - // Err(_) => Err(peer_id), - // } - // })); - - // CustomMessageOutcome::None - // }, - // Ok(msg) => { - // debug!( - // target: "sync", - // "Expected Status message from {}, but got {:?}", - // peer_id, - // msg, - // ); - // self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE); - // CustomMessageOutcome::None - // }, - // Err(err) => { - // match as DecodeAll>::decode_all( - // &mut &received_handshake[..], - // ) { - // Ok(handshake) => { - // let roles = handshake.roles; - - // let (tx, rx) = oneshot::channel(); - // let _ = self.tx.unbounded_send( - // crate::SyncEvent::NotificationStreamOpened { - // remote: peer_id, - // received_handshake: handshake, - // sink: notifications_sink, - // tx, - // }, - // ); - // self.sync_substream_validations.push(Box::pin(async move { - // match rx.await { - // Ok(accepted) => { - // if accepted { - // Ok((peer_id, roles)) - // } else { - // Err(peer_id) - // } - // }, - // Err(_) => Err(peer_id), - // } - // })); - // CustomMessageOutcome::None - // }, - // Err(err2) => { - // log::debug!( - // target: "sync", - // "Couldn't decode handshake sent by {}: {:?}: {} & {}", - // peer_id, - // received_handshake, - // err, - // err2, - // ); - // self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE); - // CustomMessageOutcome::None - // }, - // } - // }, - // } - // } else { - match (Roles::decode_all(&mut &received_handshake[..]), self.peers.get(&peer_id)) { - (Ok(roles), _) => CustomMessageOutcome::NotificationStreamOpened { - remote: peer_id, - protocol: self.notification_protocols[usize::from(set_id)].clone(), - negotiated_fallback, - roles, - received_handshake, - notifications_sink, - }, - (Err(_), Some(roles)) if received_handshake.is_empty() => { - // As a convenience, we allow opening substreams for "external" - // notification protocols with an empty handshake. This fetches the - // roles from the locally-known roles. - // TODO: remove this after https://github.com/paritytech/substrate/issues/5685 - CustomMessageOutcome::NotificationStreamOpened { + if set_id != HARDCODED_PEERSETS_SYNC { + match ( + Roles::decode_all(&mut &received_handshake[..]), + self.peers.get(&peer_id), + ) { + (Ok(roles), _) => CustomMessageOutcome::NotificationStreamOpened { remote: peer_id, protocol: self.notification_protocols[usize::from(set_id)].clone(), negotiated_fallback, - roles: *roles, + roles, received_handshake, notifications_sink, - } - }, - (Err(err), _) => { - debug!(target: "sync", "Failed to parse remote handshake: {}", err); - self.bad_handshake_substreams.insert((peer_id, set_id)); - self.behaviour.disconnect_peer(&peer_id, set_id); - self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE); - self.peers.remove(&peer_id); - CustomMessageOutcome::None - }, + }, + (Err(_), Some(roles)) if received_handshake.is_empty() => { + // As a convenience, we allow opening substreams for "external" + // notification protocols with an empty handshake. This fetches the + // roles from the locally-known roles. + // TODO: remove this after https://github.com/paritytech/substrate/issues/5685 + CustomMessageOutcome::NotificationStreamOpened { + remote: peer_id, + protocol: self.notification_protocols[usize::from(set_id)].clone(), + negotiated_fallback, + roles: *roles, + received_handshake, + notifications_sink, + } + }, + (Err(err), _) => { + debug!(target: "sync", "Failed to parse remote handshake: {}", err); + self.bad_handshake_substreams.insert((peer_id, set_id)); + self.behaviour.disconnect_peer(&peer_id, set_id); + self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE); + self.peers.remove(&peer_id); + CustomMessageOutcome::None + }, + } + } else { + CustomMessageOutcome::None } - // } }, NotificationsOut::CustomProtocolReplaced { peer_id, notifications_sink, set_id } => { if self.bad_handshake_substreams.contains(&(peer_id, set_id)) { CustomMessageOutcome::None - // } else if set_id == HARDCODED_PEERSETS_SYNC { - // let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationSinkReplaced { - // remote: peer_id, - // sink: notifications_sink, - // }); - // CustomMessageOutcome::None + } else if set_id == HARDCODED_PEERSETS_SYNC { + // let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationSinkReplaced { + // remote: peer_id, + // sink: notifications_sink, + // }); + CustomMessageOutcome::None } else { CustomMessageOutcome::NotificationStreamReplaced { remote: peer_id, @@ -652,12 +560,12 @@ impl NetworkBehaviour for Protocol { // handshake. The outer layers have never received an opening event about this // substream, and consequently shouldn't receive a closing event either. CustomMessageOutcome::None - // } else if set_id == HARDCODED_PEERSETS_SYNC { - // let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationStreamClosed { - // remote: peer_id, - // }); - // self.peers.remove(&peer_id); - // CustomMessageOutcome::None + } else if set_id == HARDCODED_PEERSETS_SYNC { + // let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationStreamClosed { + // remote: peer_id, + // }); + // self.peers.remove(&peer_id); + CustomMessageOutcome::None } else { CustomMessageOutcome::NotificationStreamClosed { remote: peer_id, @@ -668,12 +576,12 @@ impl NetworkBehaviour for Protocol { NotificationsOut::Notification { peer_id, set_id, message } => { if self.bad_handshake_substreams.contains(&(peer_id, set_id)) { CustomMessageOutcome::None - // } else if set_id == HARDCODED_PEERSETS_SYNC { - // let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationsReceived { - // remote: peer_id, - // messages: vec![message.freeze()], - // }); - // CustomMessageOutcome::None + } else if set_id == HARDCODED_PEERSETS_SYNC { + // let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationsReceived { + // remote: peer_id, + // messages: vec![message.freeze()], + // }); + CustomMessageOutcome::None } else { let protocol_name = self.notification_protocols[usize::from(set_id)].clone(); CustomMessageOutcome::NotificationsReceived { diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 99dd01819e79f..6e7db6a5624a5 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -22,12 +22,13 @@ use crate::{ handler::{self, NotificationsSink, NotifsHandlerIn, NotifsHandlerOut, NotifsHandlerProto}, service::{NotificationCommand, ProtocolHandle}, }, + service::traits::ValidationResult, types::ProtocolName, }; use bytes::BytesMut; use fnv::FnvHashMap; -use futures::prelude::*; +use futures::{future::BoxFuture, prelude::*, stream::FuturesUnordered}; use libp2p::{ core::{ConnectedPoint, Endpoint, Multiaddr}, swarm::{ @@ -41,6 +42,7 @@ use log::{error, trace, warn}; use parking_lot::RwLock; use rand::distributions::{Distribution as _, Uniform}; use smallvec::SmallVec; +use tokio::sync::oneshot::error::RecvError; use tokio_stream::StreamMap; use sc_network_common::role::ObservedRole; @@ -56,6 +58,13 @@ use std::{ time::{Duration, Instant}, }; +/// Type representing a pending substream validation. +type PendingInboundValidation = + BoxFuture<'static, (Result, sc_peerset::IncomingIndex)>; + +/// Logging target for the file. +const LOG_TARGET: &str = "sub-libp2p"; + /// Network behaviour that handles opening substreams for custom protocols with other peers. /// /// # How it works @@ -132,7 +141,7 @@ pub struct Notifications { /// /// By design, we never remove elements from this list. Elements are removed only when the /// `Delay` triggers. As such, this stream may produce obsolete elements. - delays: stream::FuturesUnordered< + delays: FuturesUnordered< Pin + Send>>, >, @@ -149,6 +158,15 @@ pub struct Notifications { /// Events to produce from `poll()`. events: VecDeque>, + + /// Pending inbound substream validations. + // + // NOTE: it's possible to read a stale response from `pending_inbound_validations` + // as the substream may get closed by the remote peer before the protocol has had + // a chance to validate it. [`Notifications`] must compare the `sc_peerset::IncomingIndex` + // returned by the completed future against the `sc_peerset::IncomingIndex` stored in + // `PeerState::Incoming` to check whether the completed future is stale or not. + pending_inbound_validations: FuturesUnordered, } /// Configuration for a notifications protocol. @@ -243,6 +261,9 @@ enum PeerState { /// If `Some`, any dial attempts to this peer are delayed until the given `Instant`. backoff_until: Option, + /// Whether the substream has been accepted by `Peerset`. + accepted: bool, + /// List of connections with this peer, and their state. connections: SmallVec<[(ConnectionId, ConnectionState); crate::MAX_CONNECTIONS_PER_PEER]>, }, @@ -311,6 +332,8 @@ struct IncomingPeer { alive: bool, /// Id that the we sent to the peerset. incoming_id: sc_peerset::IncomingIndex, + /// Received handshake. + handshake: Vec, } /// Event that can be emitted by the `Notifications`. @@ -410,6 +433,7 @@ impl Notifications { incoming: SmallVec::new(), next_incoming_index: sc_peerset::IncomingIndex(0), events: VecDeque::new(), + pending_inbound_validations: FuturesUnordered::new(), } } @@ -427,6 +451,11 @@ impl Notifications { } } + /// Get the number of connected peers for a given set. + pub fn num_connected_peers(&self, set_id: sc_peerset::SetId) -> usize { + self.protocol_handles[usize::from(set_id)].num_peers() + } + /// Returns the number of discovered nodes that we keep in memory. pub fn num_discovered_peers(&self) -> usize { self.peerset.num_discovered_peers() @@ -453,7 +482,7 @@ impl Notifications { let mut entry = if let Entry::Occupied(entry) = self.peers.entry((*peer_id, set_id)) { entry } else { - return + return; }; match mem::replace(entry.get_mut(), PeerState::Poisoned) { @@ -483,6 +512,9 @@ impl Notifications { let event = NotificationsOut::CustomProtocolClosed { peer_id: *peer_id, set_id }; self.events.push_back(ToSwarm::GenerateEvent(event)); + // TODO(aaro): error handling? + let _ = self.protocol_handles[usize::from(set_id)] + .report_substream_closed(*peer_id); } for (connec_id, connec_state) in @@ -521,7 +553,7 @@ impl Notifications { // Incoming => Disabled. // Ongoing opening requests from the remote are rejected. - PeerState::Incoming { mut connections, backoff_until } => { + PeerState::Incoming { mut connections, backoff_until, .. } => { let inc = if let Some(inc) = self .incoming .iter_mut() @@ -533,7 +565,7 @@ impl Notifications { target: "sub-libp2p", "State mismatch in libp2p: no entry in incoming for incoming peer" ); - return + return; }; inc.alive = false; @@ -589,7 +621,7 @@ impl Notifications { trace!(target: "sub-libp2p", "Libp2p <= Dial {}", entry.key().0); self.events.push_back(ToSwarm::Dial { opts: entry.key().0.into() }); entry.insert(PeerState::Requested); - return + return; }, }; @@ -789,7 +821,7 @@ impl Notifications { Entry::Vacant(entry) => { trace!(target: "sub-libp2p", "PSM => Drop({}, {:?}): Already disabled.", entry.key().0, set_id); - return + return; }, }; @@ -825,6 +857,9 @@ impl Notifications { let event = NotificationsOut::CustomProtocolClosed { peer_id: entry.key().0, set_id }; self.events.push_back(ToSwarm::GenerateEvent(event)); + // TODO(aaro): error handling? + let _ = self.protocol_handles[usize::from(set_id)] + .report_substream_closed(entry.key().0); } for (connec_id, connec_state) in @@ -887,45 +922,65 @@ impl Notifications { } } - /// Function that is called when the peerset wants us to accept a connection - /// request from a peer. - fn peerset_report_accept(&mut self, index: sc_peerset::IncomingIndex) { + fn protocol_report_accept(&mut self, index: sc_peerset::IncomingIndex) { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) } else { - error!(target: "sub-libp2p", "PSM => Accept({:?}): Invalid index", index); - return + error!(target: LOG_TARGET, "Protocol => Accept({:?}): Invalid index", index); + return; }; if !incoming.alive { - trace!(target: "sub-libp2p", "PSM => Accept({:?}, {}, {:?}): Obsolete incoming", - index, incoming.peer_id, incoming.set_id); + trace!( + target: LOG_TARGET, + "Protocol => Accept({:?}, {}, {:?}): Obsolete incoming", + index, + incoming.peer_id, + incoming.set_id + ); match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { Some(PeerState::DisabledPendingEnable { .. }) | Some(PeerState::Enabled { .. }) => { }, _ => { - trace!(target: "sub-libp2p", "PSM <= Dropped({}, {:?})", - incoming.peer_id, incoming.set_id); + trace!( + target: LOG_TARGET, + "Protocol <= Dropped({}, {:?})", + incoming.peer_id, + incoming.set_id + ); self.peerset.dropped(incoming.set_id, incoming.peer_id, DropReason::Unknown); }, } - return + return; } let state = match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { Some(s) => s, None => { debug_assert!(false); - return + return; }, }; match mem::replace(state, PeerState::Poisoned) { // Incoming => Enabled - PeerState::Incoming { mut connections, .. } => { - trace!(target: "sub-libp2p", "PSM => Accept({:?}, {}, {:?}): Enabling connections.", - index, incoming.peer_id, incoming.set_id); + PeerState::Incoming { mut connections, accepted, .. } => { + trace!( + target: LOG_TARGET, + "Protocol => Accept({:?}, {}, {:?}): Accepting connections.", + index, + incoming.peer_id, + incoming.set_id + ); + + if !accepted { + log::error!( + target: LOG_TARGET, + "protocol accepted stream but it had not been accepted by the peerset" + ); + debug_assert!(false); + } debug_assert!(connections .iter() @@ -934,8 +989,13 @@ impl Notifications { .iter_mut() .filter(|(_, s)| matches!(s, ConnectionState::OpenDesiredByRemote)) { - trace!(target: "sub-libp2p", "Handler({:?}, {:?}) <= Open({:?})", - incoming.peer_id, *connec_id, incoming.set_id); + trace!( + target: LOG_TARGET, + "Handler({:?}, {:?}) <= Open({:?})", + incoming.peer_id, + *connec_id, + incoming.set_id + ); self.events.push_back(ToSwarm::NotifyHandler { peer_id: incoming.peer_id, handler: NotifyHandler::One(*connec_id), @@ -949,9 +1009,99 @@ impl Notifications { // Any state other than `Incoming` is invalid. peer => { - error!(target: "sub-libp2p", - "State mismatch in libp2p: Expected alive incoming. Got {:?}.", - peer); + error!( + target: LOG_TARGET, + "State mismatch in libp2p: Expected alive incoming. Got {:?}.", peer + ); + debug_assert!(false); + }, + } + } + + /// Protocol rejected inbound substream. + fn protocol_report_reject(&mut self, index: sc_peerset::IncomingIndex) { + self.peerset_report_reject(index); + } + + /// Function that is called when the peerset wants us to accept a connection + /// request from a peer. + fn peerset_report_accept(&mut self, index: sc_peerset::IncomingIndex) { + let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) + { + self.incoming.remove(pos) + } else { + error!(target: LOG_TARGET, "PSM => Accept({:?}): Invalid index", index); + return; + }; + + if !incoming.alive { + trace!( + target: LOG_TARGET, + "PSM => Accept({:?}, {}, {:?}): Obsolete incoming", + index, + incoming.peer_id, + incoming.set_id + ); + match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { + Some(PeerState::DisabledPendingEnable { .. }) | Some(PeerState::Enabled { .. }) => { + }, + _ => { + trace!( + target: LOG_TARGET, + "PSM <= Dropped({}, {:?})", + incoming.peer_id, + incoming.set_id + ); + self.peerset.dropped(incoming.set_id, incoming.peer_id, DropReason::Unknown); + }, + } + return; + } + + let state = match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { + Some(s) => s, + None => { + debug_assert!(false); + return; + }, + }; + + match mem::replace(state, PeerState::Poisoned) { + // Incoming => Enabled + PeerState::Incoming { ref mut accepted, backoff_until, connections } => { + trace!( + target: LOG_TARGET, + "PSM => Accept({:?}, {}, {:?}): Send substream for validation to protocol.", + index, + incoming.peer_id, + incoming.set_id + ); + + match self.protocol_handles[usize::from(incoming.set_id)] + .report_incoming_substream(incoming.peer_id, incoming.handshake.clone()) + { + Ok(rx) => { + *accepted = true; + self.pending_inbound_validations + .push(Box::pin(async move { (rx.await, index) })); + self.incoming.push(incoming); + }, + Err(err) => { + error!(target: LOG_TARGET, "protocol has exited: {err:?}"); + debug_assert!(false); + return; + }, + } + + *state = PeerState::Incoming { backoff_until, accepted: true, connections }; + }, + + // Any state other than `Incoming` is invalid. + peer => { + error!( + target: LOG_TARGET, + "State mismatch in libp2p: Expected alive incoming. Got {:?}.", peer + ); debug_assert!(false); }, } @@ -964,26 +1114,26 @@ impl Notifications { self.incoming.remove(pos) } else { error!(target: "sub-libp2p", "PSM => Reject({:?}): Invalid index", index); - return + return; }; if !incoming.alive { trace!(target: "sub-libp2p", "PSM => Reject({:?}, {}, {:?}): Obsolete incoming, \ ignoring", index, incoming.peer_id, incoming.set_id); - return + return; } let state = match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { Some(s) => s, None => { debug_assert!(false); - return + return; }, }; match mem::replace(state, PeerState::Poisoned) { // Incoming => Disabled - PeerState::Incoming { mut connections, backoff_until } => { + PeerState::Incoming { mut connections, backoff_until, .. } => { trace!(target: "sub-libp2p", "PSM => Reject({:?}, {}, {:?}): Rejecting connections.", index, incoming.peer_id, incoming.set_id); @@ -1078,8 +1228,8 @@ impl NetworkBehaviour for Notifications { for set_id in (0..self.notif_protocols.len()).map(sc_peerset::SetId::from) { match self.peers.entry((peer_id, set_id)).or_insert(PeerState::Poisoned) { // Requested | PendingRequest => Enabled - st @ &mut PeerState::Requested | - st @ &mut PeerState::PendingRequest { .. } => { + st @ &mut PeerState::Requested + | st @ &mut PeerState::PendingRequest { .. } => { trace!(target: "sub-libp2p", "Libp2p => Connected({}, {:?}, {:?}): Connection was requested by PSM.", peer_id, set_id, endpoint @@ -1116,10 +1266,10 @@ impl NetworkBehaviour for Notifications { // In all other states, add this new connection to the list of closed // inactive connections. - PeerState::Incoming { connections, .. } | - PeerState::Disabled { connections, .. } | - PeerState::DisabledPendingEnable { connections, .. } | - PeerState::Enabled { connections, .. } => { + PeerState::Incoming { connections, .. } + | PeerState::Disabled { connections, .. } + | PeerState::DisabledPendingEnable { connections, .. } + | PeerState::Enabled { connections, .. } => { trace!(target: "sub-libp2p", "Libp2p => Connected({}, {:?}, {:?}, {:?}): Secondary connection. Leaving closed.", peer_id, set_id, endpoint, connection_id); @@ -1137,7 +1287,7 @@ impl NetworkBehaviour for Notifications { } else { error!(target: "sub-libp2p", "inject_connection_closed: State mismatch in the custom protos handler"); debug_assert!(false); - return + return; }; match mem::replace(entry.get_mut(), PeerState::Poisoned) { @@ -1223,7 +1373,7 @@ impl NetworkBehaviour for Notifications { }, // Incoming => Incoming | Disabled | Backoff | Ø - PeerState::Incoming { mut connections, backoff_until } => { + PeerState::Incoming { mut connections, backoff_until, .. } => { trace!( target: "sub-libp2p", "Libp2p => Disconnected({}, {:?}, {:?}): OpenDesiredByRemote.", @@ -1297,8 +1447,11 @@ impl NetworkBehaviour for Notifications { *entry.get_mut() = PeerState::Disabled { connections, backoff_until }; } else { - *entry.get_mut() = - PeerState::Incoming { connections, backoff_until }; + *entry.get_mut() = PeerState::Incoming { + connections, + backoff_until, + accepted: false, + }; } }, @@ -1351,6 +1504,9 @@ impl NetworkBehaviour for Notifications { set_id, }; self.events.push_back(ToSwarm::GenerateEvent(event)); + // TODO(aaro): error handling? + let _ = self.protocol_handles[usize::from(set_id)] + .report_substream_closed(peer_id); } } } else { @@ -1392,9 +1548,9 @@ impl NetworkBehaviour for Notifications { } }, - PeerState::Requested | - PeerState::PendingRequest { .. } | - PeerState::Backoff { .. } => { + PeerState::Requested + | PeerState::PendingRequest { .. } + | PeerState::Backoff { .. } => { // This is a serious bug either in this state machine or in libp2p. error!(target: "sub-libp2p", "`inject_connection_closed` called for unknown peer {}", @@ -1428,8 +1584,8 @@ impl NetworkBehaviour for Notifications { // "Basic" situation: we failed to reach a peer that the peerset // requested. - st @ PeerState::Requested | - st @ PeerState::PendingRequest { .. } => { + st @ PeerState::Requested + | st @ PeerState::PendingRequest { .. } => { trace!(target: "sub-libp2p", "PSM <= Dropped({}, {:?})", peer_id, set_id); self.peerset.dropped(set_id, peer_id, DropReason::Unknown); @@ -1437,7 +1593,9 @@ impl NetworkBehaviour for Notifications { let ban_duration = match st { PeerState::PendingRequest { timer_deadline, .. } if timer_deadline > now => - cmp::max(timer_deadline - now, Duration::from_secs(5)), + { + cmp::max(timer_deadline - now, Duration::from_secs(5)) + }, _ => Duration::from_secs(5), }; @@ -1461,10 +1619,10 @@ impl NetworkBehaviour for Notifications { // We can still get dial failures even if we are already connected // to the peer, as an extra diagnostic for an earlier attempt. - st @ PeerState::Disabled { .. } | - st @ PeerState::Enabled { .. } | - st @ PeerState::DisabledPendingEnable { .. } | - st @ PeerState::Incoming { .. } => { + st @ PeerState::Disabled { .. } + | st @ PeerState::Enabled { .. } + | st @ PeerState::DisabledPendingEnable { .. } + | st @ PeerState::Incoming { .. } => { *entry.into_mut() = st; }, @@ -1496,7 +1654,7 @@ impl NetworkBehaviour for Notifications { event: THandlerOutEvent, ) { match event { - NotifsHandlerOut::OpenDesiredByRemote { protocol_index } => { + NotifsHandlerOut::OpenDesiredByRemote { protocol_index, handshake } => { let set_id = sc_peerset::SetId::from(protocol_index); trace!(target: "sub-libp2p", @@ -1512,12 +1670,12 @@ impl NetworkBehaviour for Notifications { "OpenDesiredByRemote: State mismatch in the custom protos handler" ); debug_assert!(false); - return + return; }; match mem::replace(entry.get_mut(), PeerState::Poisoned) { // Incoming => Incoming - PeerState::Incoming { mut connections, backoff_until } => { + PeerState::Incoming { mut connections, backoff_until, accepted } => { debug_assert!(connections .iter() .any(|(_, s)| matches!(s, ConnectionState::OpenDesiredByRemote))); @@ -1545,7 +1703,8 @@ impl NetworkBehaviour for Notifications { debug_assert!(false); } - *entry.into_mut() = PeerState::Incoming { connections, backoff_until }; + *entry.into_mut() = + PeerState::Incoming { connections, backoff_until, accepted }; }, PeerState::Enabled { mut connections } => { @@ -1574,8 +1733,8 @@ impl NetworkBehaviour for Notifications { // more to do. debug_assert!(matches!( connec_state, - ConnectionState::OpenDesiredByRemote | - ConnectionState::Closing | ConnectionState::Opening + ConnectionState::OpenDesiredByRemote + | ConnectionState::Closing | ConnectionState::Opening )); } } else { @@ -1603,16 +1762,19 @@ impl NetworkBehaviour for Notifications { trace!(target: "sub-libp2p", "PSM <= Incoming({}, {:?}).", peer_id, incoming_id); self.peerset.incoming(set_id, peer_id, incoming_id); - // TODO: report opened substream to the protocol. self.incoming.push(IncomingPeer { peer_id, set_id, alive: true, incoming_id, + handshake, }); - *entry.into_mut() = - PeerState::Incoming { connections, backoff_until }; + *entry.into_mut() = PeerState::Incoming { + connections, + backoff_until, + accepted: false, + }; } else { // Connections in `OpeningThenClosing` and `Closing` state can be // in a Closed phase, and as such can emit `OpenDesiredByRemote` @@ -1696,7 +1858,7 @@ impl NetworkBehaviour for Notifications { } else { error!(target: "sub-libp2p", "CloseDesired: State mismatch in the custom protos handler"); debug_assert!(false); - return + return; }; match mem::replace(entry.get_mut(), PeerState::Poisoned) { @@ -1715,12 +1877,12 @@ impl NetworkBehaviour for Notifications { error!(target: "sub-libp2p", "CloseDesired: State mismatch in the custom protos handler"); debug_assert!(false); - return + return; }; if matches!(connections[pos].1, ConnectionState::Closing) { *entry.into_mut() = PeerState::Enabled { connections }; - return + return; } debug_assert!(matches!(connections[pos].1, ConnectionState::Open(_))); @@ -1766,13 +1928,16 @@ impl NetworkBehaviour for Notifications { trace!(target: "sub-libp2p", "External API <= Closed({}, {:?})", peer_id, set_id); let event = NotificationsOut::CustomProtocolClosed { peer_id, set_id }; self.events.push_back(ToSwarm::GenerateEvent(event)); + // TODO(aaro): error handling? + let _ = self.protocol_handles[usize::from(set_id)] + .report_substream_closed(peer_id); } }, // All connections in `Disabled` and `DisabledPendingEnable` have been sent a // `Close` message already, and as such ignore any `CloseDesired` message. - state @ PeerState::Disabled { .. } | - state @ PeerState::DisabledPendingEnable { .. } => { + state @ PeerState::Disabled { .. } + | state @ PeerState::DisabledPendingEnable { .. } => { *entry.into_mut() = state; }, state => { @@ -1792,10 +1957,10 @@ impl NetworkBehaviour for Notifications { match self.peers.get_mut(&(peer_id, set_id)) { // Move the connection from `Closing` to `Closed`. - Some(PeerState::Incoming { connections, .. }) | - Some(PeerState::DisabledPendingEnable { connections, .. }) | - Some(PeerState::Disabled { connections, .. }) | - Some(PeerState::Enabled { connections, .. }) => { + Some(PeerState::Incoming { connections, .. }) + | Some(PeerState::DisabledPendingEnable { connections, .. }) + | Some(PeerState::Disabled { connections, .. }) + | Some(PeerState::Enabled { connections, .. }) => { if let Some((_, connec_state)) = connections.iter_mut().find(|(c, s)| { *c == connection_id && matches!(s, ConnectionState::Closing) }) { @@ -1846,15 +2011,16 @@ impl NetworkBehaviour for Notifications { peer_id, set_id, negotiated_fallback: negotiated_fallback.clone(), - received_handshake, + received_handshake: received_handshake.clone(), notifications_sink: notifications_sink.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); let _ = self.protocol_handles[protocol_index] .report_substream_opened( peer_id, - ObservedRole::Full, /* TODO: this needs to be queried by - * the protocol from `Peerset` */ + received_handshake, + ObservedRole::Full, /* TODO(aaro): this needs to be + * queried from `Peerstore` */ negotiated_fallback, notifications_sink.clone(), ); @@ -1862,8 +2028,8 @@ impl NetworkBehaviour for Notifications { *connec_state = ConnectionState::Open(notifications_sink); } else if let Some((_, connec_state)) = connections.iter_mut().find(|(c, s)| { - *c == connection_id && - matches!(s, ConnectionState::OpeningThenClosing) + *c == connection_id + && matches!(s, ConnectionState::OpeningThenClosing) }) { *connec_state = ConnectionState::Closing; } else { @@ -1873,9 +2039,9 @@ impl NetworkBehaviour for Notifications { } }, - Some(PeerState::Incoming { connections, .. }) | - Some(PeerState::DisabledPendingEnable { connections, .. }) | - Some(PeerState::Disabled { connections, .. }) => { + Some(PeerState::Incoming { connections, .. }) + | Some(PeerState::DisabledPendingEnable { connections, .. }) + | Some(PeerState::Disabled { connections, .. }) => { if let Some((_, connec_state)) = connections.iter_mut().find(|(c, s)| { *c == connection_id && matches!(s, ConnectionState::OpeningThenClosing) }) { @@ -1908,7 +2074,7 @@ impl NetworkBehaviour for Notifications { } else { error!(target: "sub-libp2p", "OpenResultErr: State mismatch in the custom protos handler"); debug_assert!(false); - return + return; }; match mem::replace(entry.get_mut(), PeerState::Poisoned) { @@ -1924,8 +2090,8 @@ impl NetworkBehaviour for Notifications { *connec_state = ConnectionState::Closed; } else if let Some((_, connec_state)) = connections.iter_mut().find(|(c, s)| { - *c == connection_id && - matches!(s, ConnectionState::OpeningThenClosing) + *c == connection_id + && matches!(s, ConnectionState::OpeningThenClosing) }) { *connec_state = ConnectionState::Closing; } else { @@ -1949,17 +2115,17 @@ impl NetworkBehaviour for Notifications { *entry.into_mut() = PeerState::Enabled { connections }; } }, - mut state @ PeerState::Incoming { .. } | - mut state @ PeerState::DisabledPendingEnable { .. } | - mut state @ PeerState::Disabled { .. } => { + mut state @ PeerState::Incoming { .. } + | mut state @ PeerState::DisabledPendingEnable { .. } + | mut state @ PeerState::Disabled { .. } => { match &mut state { - PeerState::Incoming { connections, .. } | - PeerState::Disabled { connections, .. } | - PeerState::DisabledPendingEnable { connections, .. } => { + PeerState::Incoming { connections, .. } + | PeerState::Disabled { connections, .. } + | PeerState::DisabledPendingEnable { connections, .. } => { if let Some((_, connec_state)) = connections.iter_mut().find(|(c, s)| { - *c == connection_id && - matches!(s, ConnectionState::OpeningThenClosing) + *c == connection_id + && matches!(s, ConnectionState::OpeningThenClosing) }) { *connec_state = ConnectionState::Closing; } else { @@ -2008,7 +2174,7 @@ impl NetworkBehaviour for Notifications { message: message.clone(), }; self.events.push_back(NetworkBehaviourAction::GenerateEvent(event)); - // TODO: error handling + // TODO(aaro): error handling let _ = self.protocol_handles[protocol_index] .report_notification_received(peer_id, message.to_vec()); } else { @@ -2031,7 +2197,7 @@ impl NetworkBehaviour for Notifications { _params: &mut impl PollParameters, ) -> Poll>> { if let Some(event) = self.events.pop_front() { - return Poll::Ready(event) + return Poll::Ready(event); } // Poll for instructions from the peerset. @@ -2052,7 +2218,7 @@ impl NetworkBehaviour for Notifications { }, Poll::Ready(None) => { error!(target: "sub-libp2p", "Peerset receiver stream has returned None"); - break + break; }, Poll::Pending => break, } @@ -2065,19 +2231,36 @@ impl NetworkBehaviour for Notifications { NotificationCommand::SetHandshake(handshake) => { self.set_notif_protocol_handshake(set_id.into(), handshake); }, - NotificationCommand::OpenSubstream(_peer) | - NotificationCommand::CloseSubstream(_peer) => { + NotificationCommand::OpenSubstream(_peer) + | NotificationCommand::CloseSubstream(_peer) => { todo!("substream control not implemented"); }, }, Poll::Ready(None) => { error!(target: "sub-libp2p", "Protocol command streams have been shut down"); - break + break; }, Poll::Pending => break, } } + while let Poll::Ready(Some((result, index))) = + self.pending_inbound_validations.poll_next_unpin(cx) + { + match result { + Ok(ValidationResult::Accept) => { + self.protocol_report_accept(index); + }, + Ok(ValidationResult::Reject) => { + self.protocol_report_reject(index); + }, + Err(_) => { + error!(target: "sub-libp2p", "Protocol ha shut down"); + break; + }, + } + } + while let Poll::Ready(Some((delay_id, peer_id, set_id))) = Pin::new(&mut self.delays).poll_next(cx) { @@ -2137,7 +2320,7 @@ impl NetworkBehaviour for Notifications { } if let Some(event) = self.events.pop_front() { - return Poll::Ready(event) + return Poll::Ready(event); } Poll::Pending @@ -2159,8 +2342,9 @@ mod tests { (ConnectionState::Closing, ConnectionState::Closing) => true, (ConnectionState::Opening, ConnectionState::Opening) => true, (ConnectionState::OpeningThenClosing, ConnectionState::OpeningThenClosing) => true, - (ConnectionState::OpenDesiredByRemote, ConnectionState::OpenDesiredByRemote) => - true, + (ConnectionState::OpenDesiredByRemote, ConnectionState::OpenDesiredByRemote) => { + true + }, (ConnectionState::Open(_), ConnectionState::Open(_)) => true, _ => false, } @@ -2349,13 +2533,17 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); - if let Some(&PeerState::Incoming { ref connections, backoff_until: None }) = + if let Some(&PeerState::Incoming { ref connections, backoff_until: None, accepted }) = notif.peers.get(&(peer, 0.into())) { assert_eq!(connections.len(), 1); + assert_eq!(accepted, false); assert_eq!(connections[0], (conn, ConnectionState::OpenDesiredByRemote)); } else { panic!("invalid state"); @@ -2389,7 +2577,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); notif.disconnect_peer(&peer, 0.into()); @@ -2494,7 +2685,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); // attempt to connect to the peer and verify that the peer state is `Enabled` @@ -2573,7 +2767,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); notif.peerset_report_connect(peer, set_id); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); @@ -2677,7 +2874,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -2724,7 +2924,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); if let Some(PeerState::Incoming { connections, .. }) = notif.peers.get(&(peer, set_id)) { assert_eq!(connections.len(), 1); @@ -2812,7 +3015,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -2861,7 +3067,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -2928,7 +3137,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -2991,7 +3203,10 @@ mod tests { notif.on_connection_handler_event( peer, conn2, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); if let Some(PeerState::Enabled { connections, .. }) = notif.peers.get(&(peer, set_id)) { @@ -3340,7 +3555,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); // manually add backoff for the entry @@ -3401,7 +3619,10 @@ mod tests { notif.on_connection_handler_event( peer, conn1, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!( notif.peers.get_mut(&(peer, 0.into())), @@ -3459,7 +3680,10 @@ mod tests { notif.on_connection_handler_event( peer, conn1, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!( notif.peers.get_mut(&(peer, 0.into())), @@ -3645,7 +3869,10 @@ mod tests { notif.on_connection_handler_event( peer, conn1, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -3653,7 +3880,10 @@ mod tests { notif.on_connection_handler_event( peer, conn2, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); if let Some(PeerState::Incoming { ref connections, .. }) = notif.peers.get(&(peer, set_id)) @@ -3728,7 +3958,7 @@ mod tests { .await; if notif.peers.get(&(peer, set_id)).is_none() { - break + break; } } }) @@ -3772,7 +4002,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -3841,7 +4074,7 @@ mod tests { assert!(std::matches!(connections[0], (_, ConnectionState::Closing))); if timer_deadline != &prev_instant { - break + break; } } else { panic!("invalid state"); @@ -3884,7 +4117,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4044,7 +4280,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4080,7 +4319,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4122,7 +4364,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4222,7 +4467,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4260,7 +4508,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); assert!(std::matches!( @@ -4300,7 +4551,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4429,7 +4683,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); notif.incoming[0].alive = false; @@ -4473,7 +4730,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); diff --git a/client/network/src/protocol/notifications/handler.rs b/client/network/src/protocol/notifications/handler.rs index cee75398c7ec7..5e3a5755dc1c2 100644 --- a/client/network/src/protocol/notifications/handler.rs +++ b/client/network/src/protocol/notifications/handler.rs @@ -299,6 +299,8 @@ pub enum NotifsHandlerOut { OpenDesiredByRemote { /// Index of the protocol in the list of protocols passed at initialization. protocol_index: usize, + /// Received handshake. + handshake: Vec, }, /// The remote would like the substreams to be closed. Send a [`NotifsHandlerIn::Close`] in @@ -495,7 +497,10 @@ impl ConnectionHandler for NotifsHandler { match protocol_info.state { State::Closed { pending_opening } => { self.events_queue.push_back(ConnectionHandlerEvent::Custom( - NotifsHandlerOut::OpenDesiredByRemote { protocol_index }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index, + handshake: in_substream_open.handshake, + }, )); protocol_info.state = State::OpenDesiredByRemote { @@ -1610,18 +1615,25 @@ pub mod tests { drop(io2); futures::future::poll_fn(|cx| { - assert!(std::matches!( - handler.poll(cx), - Poll::Ready(ConnectionHandlerEvent::Custom( - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, - )) - )); - assert!(std::matches!( - handler.poll(cx), - Poll::Ready(ConnectionHandlerEvent::Custom(NotifsHandlerOut::CloseDesired { - protocol_index: 0 - },)) - )); + if let Poll::Ready(ConnectionHandlerEvent::Custom( + NotifsHandlerOut::OpenDesiredByRemote { protocol_index, handshake }, + )) = handler.poll(cx) + { + assert_eq!(protocol_index, 0); + assert_eq!(handshake, b"hello, world"); + } else { + panic!("invalid event received"); + } + + if let Poll::Ready(ConnectionHandlerEvent::Custom(NotifsHandlerOut::CloseDesired { + protocol_index, + })) = handler.poll(cx) + { + assert_eq!(protocol_index, 0); + } else { + panic!("invalid event received"); + } + Poll::Ready(()) }) .await; diff --git a/client/network/src/protocol/notifications/service.rs b/client/network/src/protocol/notifications/service.rs index 8f7e4cc298550..48b101fcd1473 100644 --- a/client/network/src/protocol/notifications/service.rs +++ b/client/network/src/protocol/notifications/service.rs @@ -70,6 +70,9 @@ enum InnerNotificationEvent { /// Peer ID. peer: PeerId, + /// Received handshake. + handshake: Vec, + /// Role of the peer. role: ObservedRole, @@ -202,6 +205,7 @@ impl NotificationService for NotificationHandle { Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }), InnerNotificationEvent::NotificationStreamOpened { peer, + handshake, role, negotiated_fallback, sink, @@ -209,6 +213,7 @@ impl NotificationService for NotificationHandle { self.peers.insert(peer, sink); Some(NotificationEvent::NotificationStreamOpened { peer, + handshake, role, negotiated_fallback, }) @@ -236,6 +241,10 @@ impl NotificationService for NotificationHandle { subscribers: self.subscribers.clone(), })) } + + fn protocol(&self) -> &ProtocolName { + &self.protocol + } } /// Channel pair which allows `Notifications` to interact with a protocol. @@ -282,11 +291,14 @@ pub struct ProtocolHandle { /// Subscribers of the notification protocol. subscribers: Subscribers, + + /// Number of connected peers. + num_peers: usize, } impl ProtocolHandle { fn new(protocol: ProtocolName, subscribers: Subscribers) -> Self { - Self { protocol, subscribers } + Self { protocol, subscribers, num_peers: 0usize } } /// Report to the protocol that a substream has been opened and it must be validated by the @@ -358,8 +370,9 @@ impl ProtocolHandle { /// Report to the protocol that a substream has been opened and that it can now use the handle /// to send notifications to the remote peer. pub fn report_substream_opened( - &self, + &mut self, peer: PeerId, + handshake: Vec, role: ObservedRole, negotiated_fallback: Option, sink: NotificationsSink, @@ -372,12 +385,14 @@ impl ProtocolHandle { subscriber .unbounded_send(InnerNotificationEvent::NotificationStreamOpened { peer, + handshake: handshake.clone(), role: role.clone(), negotiated_fallback: negotiated_fallback.clone(), sink: sink.clone(), }) .is_ok() }); + self.num_peers += 1; Ok(()) } @@ -393,6 +408,7 @@ impl ProtocolHandle { .unbounded_send(InnerNotificationEvent::NotificationStreamClosed { peer }) .is_ok() }); + self.num_peers -= 1; Ok(()) } @@ -418,6 +434,11 @@ impl ProtocolHandle { Ok(()) } + + /// Get the number of connected peers. + pub fn num_peers(&self) -> usize { + self.num_peers + } } /// Create new (protocol, notification) handle pair. @@ -468,20 +489,24 @@ mod tests { async fn substream_opened() { let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, _) = NotificationsSink::new(PeerId::random()); - let (handle, _stream) = proto.split(); + let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); - handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, role, negotiated_fallback, + handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); } @@ -491,7 +516,7 @@ mod tests { async fn send_sync_notification() { let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); - let (handle, _stream) = proto.split(); + let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); // validate inbound substream @@ -509,17 +534,21 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, role, negotiated_fallback, + handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); } @@ -535,7 +564,7 @@ mod tests { async fn send_async_notification() { let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); - let (handle, _stream) = proto.split(); + let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); // validate inbound substream @@ -553,17 +582,21 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, role, negotiated_fallback, + handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); } @@ -629,17 +662,21 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, role, negotiated_fallback, + handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); } @@ -661,7 +698,7 @@ mod tests { async fn backpressure_works() { let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); - let (handle, _stream) = proto.split(); + let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); // validate inbound substream @@ -679,17 +716,21 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, role, negotiated_fallback, + handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); } @@ -741,17 +782,21 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, role, negotiated_fallback, + handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); } @@ -787,17 +832,21 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, role, negotiated_fallback, + handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); } @@ -928,18 +977,22 @@ mod tests { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that then notification stream has been opened - handle.report_substream_opened(peer_id, ObservedRole::Full, None, sink).unwrap(); + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); for notif in vec![&mut notif1, &mut notif2, &mut notif3] { if let Some(NotificationEvent::NotificationStreamOpened { peer, role, negotiated_fallback, + handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); } diff --git a/client/network/src/protocol/notifications/tests.rs b/client/network/src/protocol/notifications/tests.rs index 085fcb489977a..6602220146321 100644 --- a/client/network/src/protocol/notifications/tests.rs +++ b/client/network/src/protocol/notifications/tests.rs @@ -232,6 +232,7 @@ impl NetworkBehaviour for CustomProtoWithAddr { } #[test] +#[ignore] fn reconnect_after_disconnect() { // We connect two nodes together, then force a disconnect (through the API of the `Service`), // check that the disconnect worked, and finally check whether they successfully reconnect. diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index e2585ed7065f7..2a1b1a7e058ce 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -668,6 +668,9 @@ pub enum NotificationEvent { /// Peer ID. peer: PeerId, + /// Received handshake. + handshake: Vec, + /// Role of the peer. role: ObservedRole, @@ -764,4 +767,7 @@ pub trait NotificationService: Debug + Send { /// Make a copy of the object so it can be shared between protocol components /// who wish to have access to the same underlying notification protocol. fn clone(&mut self) -> Result, ()>; + + /// Get protocol name of the `NotificationService`. + fn protocol(&self) -> &ProtocolName; } diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index 1ec98f49bf550..68fb9a1d9efc9 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -197,9 +197,6 @@ pub struct SyncingEngine { /// Channel for receiving service commands service_rx: TracingUnboundedReceiver>, - /// Channel for receiving inbound connections from `Protocol`. - rx: sc_utils::mpsc::TracingUnboundedReceiver>, - /// Assigned roles. roles: Roles, @@ -249,12 +246,6 @@ pub struct SyncingEngine { /// Handle that is used to communicate with `sc_network::Notifications`. notification_handle: Box, - - /// Peers that are on probation, waiting for the inbound substream to be accepted - /// - /// These peers have been validated by `SyncingEngine` and are considered "accepted" - /// and they have a slot reserved for them. TODO: finish this comment - probation: HashMap>, } impl SyncingEngine @@ -282,7 +273,7 @@ where block_request_protocol_name: ProtocolName, state_request_protocol_name: ProtocolName, warp_sync_protocol_name: Option, - rx: sc_utils::mpsc::TracingUnboundedReceiver>, + _rx: sc_utils::mpsc::TracingUnboundedReceiver>, ) -> Result<(Self, SyncingService, NonDefaultSetConfig), ClientError> { let mode = match net_config.network_config.sync_mode { SyncOperationMode::Full => SyncMode::Full, @@ -391,7 +382,6 @@ where num_connected: num_connected.clone(), is_major_syncing: is_major_syncing.clone(), service_rx, - rx, genesis_hash, important_peers, default_peers_set_no_slot_connected_peers: HashSet::new(), @@ -402,7 +392,6 @@ where event_streams: Vec::new(), notification_handle, tick_timeout: Delay::new(TICK_TIMEOUT), - probation: HashMap::new(), metrics: if let Some(r) = metrics_registry { match Metrics::register(r, is_major_syncing.clone()) { Ok(metrics) => Some(metrics), @@ -745,36 +734,33 @@ where match event { NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => { - let validation_result = self.validate_handshake(&peer, handshake).map_or( - ValidationResult::Reject, - |handshake| { - // TODO: reserve slot for the peer - // TODO: save `Instant::now()` to `probation` and define a constant that - // tells how long the peer is kept on probation before it's - // evicted. TODO: when `ValidateInboundSubstream` is received for a new - // peer and all slots are full, check if a peer can be evicted - // from `probation` TODO: save peer role to `PeerStore` - self.probation.insert(peer, handshake); - ValidationResult::Accept - }, - ); + // TODO(aaro): put peer on probation and wait to receive + // `NotificationStreamOpened` from them + let validation_result = self + .validate_handshake(&peer, handshake) + .map_or(ValidationResult::Reject, |_| ValidationResult::Accept); + let _ = result_tx.send(validation_result); }, - NotificationEvent::NotificationStreamOpened { peer, .. } => { - let Some(handshake) = self.probation.remove(&peer) else { - log::debug!( - target: LOG_TARGET, - "{peer} has not been validated by `SyncingEngine` or it took too long to open a substream" - ); - continue - }; + NotificationEvent::NotificationStreamOpened { peer, handshake, .. } => { + log::debug!( + target: LOG_TARGET, + "substream opened for {peer}, handshake {handshake:?}" + ); - match self.on_sync_peer_connected(peer, &handshake) { - Ok(_) => { - // TODO: remove slot reservation and mark the slot as occupied - }, - Err(_) => { - // TODO: remove slot reservation and mark the slot as free + match self.validate_handshake(&peer, handshake) { + Ok(handshake) => + if self.on_sync_peer_connected(peer, &handshake).is_err() { + log::debug!(target: LOG_TARGET, "failed to register peer"); + self.network_service.disconnect_peer( + peer, + self.block_announce_protocol_name.clone(), + ); + }, + Err(err) => { + log::debug!(target: LOG_TARGET, "failed to decode handshake: {err:?}"); + self.network_service + .disconnect_peer(peer, self.block_announce_protocol_name.clone()); }, } }, @@ -832,7 +818,7 @@ where } self.chain_sync.peer_disconnected(&peer); - // TODO: remove this bookkeeping when `ProtocolController` is ready + // TODO(aaro): remove this bookkeeping when `ProtocolController` is ready self.default_peers_set_no_slot_connected_peers.remove(&peer); self.event_streams .retain(|stream| stream.unbounded_send(SyncEvent::PeerDisconnected(peer)).is_ok()); @@ -860,7 +846,7 @@ where } if handshake.genesis_hash != self.genesis_hash { - // TODO: report peer but verify that `PeerStore` doesn't try to disconnect it + // TODO(aaro): report peer but verify that `PeerStore` doesn't try to disconnect it // self.network_service.report_peer(peer, rep::GENESIS_MISMATCH); if self.important_peers.contains(&peer) { @@ -951,7 +937,7 @@ where log::debug!(target: LOG_TARGET, "connected {}", who); self.peers.insert(who, peer); - // TODO: remove these bookkeepings once `ProtocolController` is ready + // TODO(aaro): remove these bookkeepings once `ProtocolController` is ready if self.default_peers_set_no_slot_peers.contains(&who) { self.default_peers_set_no_slot_connected_peers.insert(who); } diff --git a/client/network/test/src/service.rs b/client/network/test/src/service.rs index 203a8c7105a61..ed4e0cb7aaa31 100644 --- a/client/network/test/src/service.rs +++ b/client/network/test/src/service.rs @@ -23,8 +23,9 @@ use sc_consensus::{ImportQueue, Link}; use sc_network::{ config::{self, FullNetworkConfiguration, MultiaddrWithPeerId, ProtocolId, TransportConfig}, event::Event, + service::traits::{NotificationEvent, ValidationResult}, NetworkEventStream, NetworkNotification, NetworkPeers, NetworkService, NetworkStateInfo, - NetworkWorker, + NetworkWorker, NotificationService, }; use sc_network_common::role::Roles; use sc_network_light::light_client_requests::handler::LightClientRequestHandler; @@ -115,7 +116,7 @@ impl TestNetworkBuilder { self } - pub fn build(mut self) -> TestNetwork { + pub fn build(mut self) -> (TestNetwork, Box) { let client = self.client.as_mut().map_or( Arc::new(TestClientBuilder::with_default_backend().build_with_longest_chain().0), |v| v.clone(), @@ -257,7 +258,7 @@ impl TestNetworkBuilder { }); tokio::spawn(engine.run()); - TestNetwork::new(worker) + (TestNetwork::new(worker), handle) } } @@ -265,18 +266,18 @@ impl TestNetworkBuilder { /// The nodes are connected together and have the `PROTOCOL_NAME` protocol registered. fn build_nodes_one_proto() -> ( Arc, - impl Stream, + Box, Arc, - impl Stream, + Box, ) { let listen_addr = config::build_multiaddr![Memory(rand::random::())]; - let (node1, events_stream1) = TestNetworkBuilder::new() + let (network1, handle1) = TestNetworkBuilder::new() .with_listen_addresses(vec![listen_addr.clone()]) - .build() - .start_network(); + .build(); + let (node1, _) = network1.start_network(); - let (node2, events_stream2) = TestNetworkBuilder::new() + let (network2, handle2) = TestNetworkBuilder::new() .with_set_config(config::SetConfig { reserved_nodes: vec![MultiaddrWithPeerId { multiaddr: listen_addr, @@ -284,10 +285,11 @@ fn build_nodes_one_proto() -> ( }], ..Default::default() }) - .build() - .start_network(); + .build(); + + let (node2, _) = network2.start_network(); - (node1, events_stream1, node2, events_stream2) + (node1, handle1, node2, handle2) } #[tokio::test] @@ -295,22 +297,14 @@ async fn notifications_state_consistent() { // Runs two nodes and ensures that events are propagated out of the API in a consistent // correct order, which means no notification received on a closed substream. - let (node1, mut events_stream1, node2, mut events_stream2) = build_nodes_one_proto(); + let (node1, mut handle1, node2, mut handle2) = build_nodes_one_proto(); // Write some initial notifications that shouldn't get through. for _ in 0..(rand::random::() % 5) { - node1.write_notification( - node2.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle1.send_sync_notification(&node2.local_peer_id(), b"hello world".to_vec()); } for _ in 0..(rand::random::() % 5) { - node2.write_notification( - node1.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle2.send_sync_notification(&node1.local_peer_id(), b"hello world".to_vec()); } // True if we have an active substream from node1 to node2. @@ -332,18 +326,10 @@ async fn notifications_state_consistent() { // Start by sending a notification from node1 to node2 and vice-versa. Part of the // test consists in ensuring that notifications get ignored if the stream isn't open. if rand::random::() % 5 >= 3 { - node1.write_notification( - node2.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle1.send_sync_notification(&node2.local_peer_id(), b"hello world".to_vec()); } if rand::random::() % 5 >= 3 { - node2.write_notification( - node1.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle2.send_sync_notification(&node1.local_peer_id(), b"hello world".to_vec()); } // Also randomly disconnect the two nodes from time to time. @@ -356,8 +342,8 @@ async fn notifications_state_consistent() { // Grab next event from either `events_stream1` or `events_stream2`. let next_event = { - let next1 = events_stream1.next(); - let next2 = events_stream2.next(); + let next1 = handle1.next_event(); + let next2 = handle2.next_event(); // We also await on a small timer, otherwise it is possible for the test to wait // forever while nothing at all happens on the network. let continue_test = futures_timer::Delay::new(Duration::from_millis(20)); @@ -372,58 +358,55 @@ async fn notifications_state_consistent() { }; match next_event { - future::Either::Left(Event::NotificationStreamOpened { remote, protocol, .. }) => - if protocol == PROTOCOL_NAME.into() { - something_happened = true; - assert!(!node1_to_node2_open); - node1_to_node2_open = true; - assert_eq!(remote, node2.local_peer_id()); - }, - future::Either::Right(Event::NotificationStreamOpened { remote, protocol, .. }) => - if protocol == PROTOCOL_NAME.into() { - something_happened = true; - assert!(!node2_to_node1_open); - node2_to_node1_open = true; - assert_eq!(remote, node1.local_peer_id()); - }, - future::Either::Left(Event::NotificationStreamClosed { remote, protocol, .. }) => - if protocol == PROTOCOL_NAME.into() { - assert!(node1_to_node2_open); - node1_to_node2_open = false; - assert_eq!(remote, node2.local_peer_id()); - }, - future::Either::Right(Event::NotificationStreamClosed { remote, protocol, .. }) => - if protocol == PROTOCOL_NAME.into() { - assert!(node2_to_node1_open); - node2_to_node1_open = false; - assert_eq!(remote, node1.local_peer_id()); - }, - future::Either::Left(Event::NotificationsReceived { remote, .. }) => { + future::Either::Left(NotificationEvent::ValidateInboundSubstream { + result_tx, .. + }) => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + future::Either::Right(NotificationEvent::ValidateInboundSubstream { + result_tx, + .. + }) => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + future::Either::Left(NotificationEvent::NotificationStreamOpened { peer, .. }) => { + something_happened = true; + assert!(!node1_to_node2_open); + node1_to_node2_open = true; + assert_eq!(peer, node2.local_peer_id()); + }, + future::Either::Right(NotificationEvent::NotificationStreamOpened { peer, .. }) => { + something_happened = true; + assert!(!node2_to_node1_open); + node2_to_node1_open = true; + assert_eq!(peer, node1.local_peer_id()); + }, + future::Either::Left(NotificationEvent::NotificationStreamClosed { peer, .. }) => { assert!(node1_to_node2_open); - assert_eq!(remote, node2.local_peer_id()); + node1_to_node2_open = false; + assert_eq!(peer, node2.local_peer_id()); + }, + future::Either::Right(NotificationEvent::NotificationStreamClosed { peer, .. }) => { + assert!(node2_to_node1_open); + node2_to_node1_open = false; + assert_eq!(peer, node1.local_peer_id()); + }, + future::Either::Left(NotificationEvent::NotificationReceived { peer, .. }) => { + assert!(node1_to_node2_open); + assert_eq!(peer, node2.local_peer_id()); if rand::random::() % 5 >= 4 { - node1.write_notification( - node2.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle1 + .send_sync_notification(&node2.local_peer_id(), b"hello world".to_vec()); } }, - future::Either::Right(Event::NotificationsReceived { remote, .. }) => { + future::Either::Right(NotificationEvent::NotificationReceived { peer, .. }) => { assert!(node2_to_node1_open); - assert_eq!(remote, node1.local_peer_id()); + assert_eq!(peer, node1.local_peer_id()); if rand::random::() % 5 >= 4 { - node2.write_notification( - node1.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle2 + .send_sync_notification(&node1.local_peer_id(), b"hello world".to_vec()); } }, - - // Add new events here. - future::Either::Left(Event::Dht(_)) => {}, - future::Either::Right(Event::Dht(_)) => {}, }; } } @@ -433,20 +416,28 @@ async fn lots_of_incoming_peers_works() { sp_tracing::try_init_simple(); let listen_addr = config::build_multiaddr![Memory(rand::random::())]; - let (main_node, _) = TestNetworkBuilder::new() + let (main_node, mut handle1) = TestNetworkBuilder::new() .with_listen_addresses(vec![listen_addr.clone()]) .with_set_config(config::SetConfig { in_peers: u32::MAX, ..Default::default() }) - .build() - .start_network(); + .build(); + let (main_node, _) = main_node.start_network(); let main_node_peer_id = main_node.local_peer_id(); + tokio::spawn(async move { + while let Some(event) = handle1.next_event().await { + if let NotificationEvent::ValidateInboundSubstream { result_tx, .. } = event { + result_tx.send(ValidationResult::Accept).unwrap(); + } + } + }); + // We spawn background tasks and push them in this `Vec`. They will all be waited upon before // this test ends. let mut background_tasks_to_wait = Vec::new(); for _ in 0..32 { - let (_dialing_node, event_stream) = TestNetworkBuilder::new() + let (dialing_node, mut handle) = TestNetworkBuilder::new() .with_set_config(config::SetConfig { reserved_nodes: vec![MultiaddrWithPeerId { multiaddr: listen_addr.clone(), @@ -454,8 +445,8 @@ async fn lots_of_incoming_peers_works() { }], ..Default::default() }) - .build() - .start_network(); + .build(); + let (_, _) = dialing_node.start_network(); background_tasks_to_wait.push(tokio::spawn(async move { // Create a dummy timer that will "never" fire, and that will be overwritten when we @@ -463,34 +454,23 @@ async fn lots_of_incoming_peers_works() { // make the code below way more complicated. let mut timer = futures_timer::Delay::new(Duration::from_secs(3600 * 24 * 7)).fuse(); - let mut event_stream = event_stream.fuse(); - let mut sync_protocol_name = None; loop { futures::select! { _ = timer => { // Test succeeds when timer fires. return; } - ev = event_stream.next() => { - match ev.unwrap() { - Event::NotificationStreamOpened { protocol, remote, .. } => { - if let None = sync_protocol_name { - sync_protocol_name = Some(protocol.clone()); - } - - assert_eq!(remote, main_node_peer_id); - // Test succeeds after 5 seconds. This timer is here in order to - // detect a potential problem after opening. - timer = futures_timer::Delay::new(Duration::from_secs(5)).fuse(); - } - Event::NotificationStreamClosed { protocol, .. } => { - if Some(protocol) != sync_protocol_name { - // Test failed. - panic!(); - } - } - _ => {} + ev = handle.next_event().fuse() => match ev.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); } + NotificationEvent::NotificationStreamOpened { peer, .. } => { + assert_eq!(peer, main_node_peer_id); + // Test succeeds after 5 seconds. This timer is here in order to + // detect a potential problem after opening. + timer = futures_timer::Delay::new(Duration::from_secs(5)).fuse(); + } + _ => {} } } } @@ -500,94 +480,104 @@ async fn lots_of_incoming_peers_works() { future::join_all(background_tasks_to_wait).await; } -#[tokio::test] -async fn notifications_back_pressure() { - // Node 1 floods node 2 with notifications. Random sleeps are done on node 2 to simulate the - // node being busy. We make sure that all notifications are received. - - const TOTAL_NOTIFS: usize = 10_000; - - let (node1, mut events_stream1, node2, mut events_stream2) = build_nodes_one_proto(); - let node2_id = node2.local_peer_id(); - - let receiver = tokio::spawn(async move { - let mut received_notifications = 0; - let mut sync_protocol_name = None; - - while received_notifications < TOTAL_NOTIFS { - match events_stream2.next().await.unwrap() { - Event::NotificationStreamOpened { protocol, .. } => { - if let None = sync_protocol_name { - sync_protocol_name = Some(protocol); - } - }, - Event::NotificationStreamClosed { protocol, .. } => { - if Some(&protocol) != sync_protocol_name.as_ref() { - panic!() - } - }, - Event::NotificationsReceived { messages, .. } => - for message in messages { - assert_eq!(message.0, PROTOCOL_NAME.into()); - assert_eq!(message.1, format!("hello #{}", received_notifications)); - received_notifications += 1; - }, - _ => {}, - }; - - if rand::random::() < 2 { - tokio::time::sleep(Duration::from_millis(rand::random::() % 750)).await; - } - } - }); - - // Wait for the `NotificationStreamOpened`. - loop { - match events_stream1.next().await.unwrap() { - Event::NotificationStreamOpened { .. } => break, - _ => {}, - }; - } - - // Sending! - for num in 0..TOTAL_NOTIFS { - let notif = node1.notification_sender(node2_id, PROTOCOL_NAME.into()).unwrap(); - notif - .ready() - .await - .unwrap() - .send(format!("hello #{}", num).into_bytes()) - .unwrap(); - } - - receiver.await.unwrap(); -} +// #[tokio::test] +// async fn notifications_back_pressure() { +// // Node 1 floods node 2 with notifications. Random sleeps are done on node 2 to simulate the +// // node being busy. We make sure that all notifications are received. + +// const TOTAL_NOTIFS: usize = 10_000; + +// let (node1, mut events_stream1, node2, mut events_stream2) = build_nodes_one_proto(); +// let node2_id = node2.local_peer_id(); + +// let receiver = tokio::spawn(async move { +// let mut received_notifications = 0; +// let mut sync_protocol_name = None; + +// while received_notifications < TOTAL_NOTIFS { +// match events_stream2.next().await.unwrap() { +// Event::NotificationStreamOpened { protocol, .. } => { +// if let None = sync_protocol_name { +// sync_protocol_name = Some(protocol); +// } +// }, +// Event::NotificationStreamClosed { protocol, .. } => { +// if Some(&protocol) != sync_protocol_name.as_ref() { +// panic!() +// } +// }, +// Event::NotificationsReceived { messages, .. } => { +// for message in messages { +// assert_eq!(message.0, PROTOCOL_NAME.into()); +// assert_eq!(message.1, format!("hello #{}", received_notifications)); +// received_notifications += 1; +// } +// }, +// _ => {}, +// }; + +// if rand::random::() < 2 { +// tokio::time::sleep(Duration::from_millis(rand::random::() % 750)).await; +// } +// } +// }); + +// // Wait for the `NotificationStreamOpened`. +// loop { +// match events_stream1.next().await.unwrap() { +// Event::NotificationStreamOpened { .. } => break, +// _ => {}, +// }; +// } + +// // Sending! +// for num in 0..TOTAL_NOTIFS { +// let notif = node1.notification_sender(node2_id, PROTOCOL_NAME.into()).unwrap(); +// notif +// .ready() +// .await +// .unwrap() +// .send(format!("hello #{}", num).into_bytes()) +// .unwrap(); +// } + +// receiver.await.unwrap(); +// } #[tokio::test] async fn fallback_name_working() { + sp_tracing::try_init_simple(); // Node 1 supports the protocols "new" and "old". Node 2 only supports "old". Checks whether // they can connect. const NEW_PROTOCOL_NAME: &str = "/new-shiny-protocol-that-isnt-PROTOCOL_NAME"; let listen_addr = config::build_multiaddr![Memory(rand::random::())]; - let (config, handle) = config::NonDefaultSetConfig::new( + let (config, mut handle1) = config::NonDefaultSetConfig::new( NEW_PROTOCOL_NAME.into(), vec![PROTOCOL_NAME.into()], 1024 * 1024, None, Default::default(), ); + // <<<<<<< HEAD let (node1, mut events_stream1) = TestNetworkBuilder::new() .with_notification_protocol(config) + // ||||||| parent of 6114a5d425 (Validate inbound substream before emitting + // `NotificationStreamOpened`) let (node1, mut events_stream1) = TestNetworkBuilder::new() + // ======= + // let (network1, _) = TestNetworkBuilder::new() + // >>>>>>> 6114a5d425 (Validate inbound substream before emitting + // `NotificationStreamOpened`) .with_config(config::NetworkConfiguration { listen_addresses: vec![listen_addr.clone()], transport: TransportConfig::MemoryOnly, ..config::NetworkConfiguration::new_local() }) - .build() - .start_network(); + .build(); - let (_, mut events_stream2) = TestNetworkBuilder::new() + let (node1, _) = network1.start_network(); + + let (network2, mut handle2) = TestNetworkBuilder::new() .with_set_config(config::SetConfig { reserved_nodes: vec![MultiaddrWithPeerId { multiaddr: listen_addr, @@ -595,34 +585,37 @@ async fn fallback_name_working() { }], ..Default::default() }) - .build() - .start_network(); + .build(); + let _ = network2.start_network(); let receiver = tokio::spawn(async move { // Wait for the `NotificationStreamOpened`. loop { - match events_stream2.next().await.unwrap() { - Event::NotificationStreamOpened { protocol, negotiated_fallback, .. } => { - assert_eq!(protocol, PROTOCOL_NAME.into()); + match handle2.next_event().await.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + NotificationEvent::NotificationStreamOpened { negotiated_fallback, .. } => { assert_eq!(negotiated_fallback, None); break }, _ => {}, - }; + } } }); // Wait for the `NotificationStreamOpened`. loop { - match events_stream1.next().await.unwrap() { - Event::NotificationStreamOpened { protocol, negotiated_fallback, .. } - if protocol == NEW_PROTOCOL_NAME.into() => - { + match handle1.next_event().await.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + NotificationEvent::NotificationStreamOpened { negotiated_fallback, .. } => { assert_eq!(negotiated_fallback, Some(PROTOCOL_NAME.into())); break }, _ => {}, - }; + } } receiver.await.unwrap(); @@ -645,6 +638,7 @@ async fn ensure_listen_addresses_consistent_with_transport_memory() { ) }) .build() + .0 .start_network(); } @@ -664,6 +658,7 @@ async fn ensure_listen_addresses_consistent_with_transport_not_memory() { ) }) .build() + .0 .start_network(); } @@ -689,6 +684,7 @@ async fn ensure_boot_node_addresses_consistent_with_transport_memory() { ) }) .build() + .0 .start_network(); } @@ -713,6 +709,7 @@ async fn ensure_boot_node_addresses_consistent_with_transport_not_memory() { ) }) .build() + .0 .start_network(); } @@ -741,6 +738,7 @@ async fn ensure_reserved_node_addresses_consistent_with_transport_memory() { ) }) .build() + .0 .start_network(); } @@ -768,6 +766,7 @@ async fn ensure_reserved_node_addresses_consistent_with_transport_not_memory() { ) }) .build() + .0 .start_network(); } @@ -790,6 +789,7 @@ async fn ensure_public_addresses_consistent_with_transport_memory() { ) }) .build() + .0 .start_network(); } @@ -811,5 +811,6 @@ async fn ensure_public_addresses_consistent_with_transport_not_memory() { ) }) .build() + .0 .start_network(); } diff --git a/client/network/transactions/src/lib.rs b/client/network/transactions/src/lib.rs index d4ef9d93f7f06..f2d46c4f18495 100644 --- a/client/network/transactions/src/lib.rs +++ b/client/network/transactions/src/lib.rs @@ -37,7 +37,7 @@ use prometheus_endpoint::{register, Counter, PrometheusError, Registry, U64}; use sc_network::{ config::{NonDefaultSetConfig, NonReservedPeerMode, ProtocolId, SetConfig}, error, - service::traits::{NotificationEvent, NotificationService}, + service::traits::{NotificationEvent, NotificationService, ValidationResult}, types::ProtocolName, utils::{interval, LruHashSet}, NetworkEventStream, NetworkNotification, NetworkPeers, @@ -323,6 +323,9 @@ where fn handle_notification_event(&mut self, event: NotificationEvent) { match event { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + let _ = result_tx.send(ValidationResult::Accept); + }, NotificationEvent::NotificationStreamOpened { peer, role, .. } => { let _was_in = self.peers.insert( peer, @@ -348,7 +351,6 @@ where warn!(target: "sub-libp2p", "Failed to decode transactions list"); } }, - event => log::debug!(target: "sync", "ignoring {event:?}"), } } From 513ca87b32889dbc89ded6c68370e5042474d289 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 9 May 2023 11:46:57 +0300 Subject: [PATCH 15/43] Convert statement store to use `NotificationService` --- bin/node-template/node/src/service.rs | 4 +- bin/node/cli/src/service.rs | 4 +- .../consensus/beefy/src/communication/mod.rs | 4 +- client/consensus/beefy/src/lib.rs | 6 +- client/consensus/beefy/src/tests.rs | 10 +- client/consensus/beefy/src/worker.rs | 7 +- .../grandpa/src/communication/mod.rs | 4 +- .../grandpa/src/communication/tests.rs | 8 +- client/consensus/grandpa/src/lib.rs | 6 +- client/consensus/grandpa/src/observer.rs | 4 +- client/consensus/grandpa/src/tests.rs | 48 +++--- client/network-gossip/src/bridge.rs | 12 +- client/network/src/config.rs | 4 +- .../src/protocol/notifications/service.rs | 5 +- client/network/statement/src/lib.rs | 79 ++++----- client/network/sync/src/engine.rs | 10 +- client/network/sync/src/lib.rs | 4 +- client/network/test/src/lib.rs | 8 +- client/network/test/src/service.rs | 157 +++++++++--------- client/network/transactions/src/lib.rs | 14 +- 20 files changed, 192 insertions(+), 206 deletions(-) diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 748b77ad612eb..bed2e35e37d5e 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -156,7 +156,7 @@ pub fn new_full(config: Configuration) -> Result { &client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"), &config.chain_spec, ); - let (grandpa_protocol_config, grandpa_notification_handle) = + let (grandpa_protocol_config, grandpa_notification_service) = sc_consensus_grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone()); net_config.add_notification_protocol(grandpa_protocol_config); @@ -296,7 +296,7 @@ pub fn new_full(config: Configuration) -> Result { link: grandpa_link, network, sync: Arc::new(sync_service), - notification_handle: grandpa_notification_handle, + notification_service: grandpa_notification_service, voting_rule: sc_consensus_grandpa::VotingRulesBuilder::default().build(), prometheus_registry, shared_voter_state: SharedVoterState::empty(), diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index f64219b689680..def0c50e95d75 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -349,7 +349,7 @@ pub fn new_full_base( &client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"), &config.chain_spec, ); - let (grandpa_protocol_config, grandpa_notification_handle) = + let (grandpa_protocol_config, grandpa_notification_service) = grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone()); net_config.add_notification_protocol(grandpa_protocol_config); @@ -550,7 +550,7 @@ pub fn new_full_base( link: grandpa_link, network: network.clone(), sync: Arc::new(sync_service.clone()), - notification_handle: grandpa_notification_handle, + notification_service: grandpa_notification_service, telemetry: telemetry.as_ref().map(|x| x.handle()), voting_rule: grandpa::VotingRulesBuilder::default().build(), prometheus_registry: prometheus_registry.clone(), diff --git a/client/consensus/beefy/src/communication/mod.rs b/client/consensus/beefy/src/communication/mod.rs index 4f2ae69df94f1..178d2dff09573 100644 --- a/client/consensus/beefy/src/communication/mod.rs +++ b/client/consensus/beefy/src/communication/mod.rs @@ -68,7 +68,7 @@ pub(crate) mod beefy_protocol_name { pub fn beefy_peers_set_config( gossip_protocol_name: sc_network::ProtocolName, ) -> (sc_network::config::NonDefaultSetConfig, Box) { - let (mut cfg, notification_handle) = sc_network::config::NonDefaultSetConfig::new( + let (mut cfg, notification_service) = sc_network::config::NonDefaultSetConfig::new( gossip_protocol_name, Vec::new(), 1024 * 1024, @@ -76,7 +76,7 @@ pub fn beefy_peers_set_config( Default::default(), ); cfg.allow_non_reserved(25, 25); - (cfg, notification_handle) + (cfg, notification_service) } // cost scalars for reporting peers. diff --git a/client/consensus/beefy/src/lib.rs b/client/consensus/beefy/src/lib.rs index 965fb7fc9d933..6601c8d02bd90 100644 --- a/client/consensus/beefy/src/lib.rs +++ b/client/consensus/beefy/src/lib.rs @@ -181,7 +181,7 @@ pub struct BeefyNetworkParams { /// Syncing service implementing a sync oracle and an event stream for peers. pub sync: Arc, /// Handle for receiving notification events. - pub notification_handle: Box, + pub notification_service: Box, /// Chain specific BEEFY gossip protocol name. See /// [`communication::beefy_protocol_name::gossip_protocol_name`]. pub gossip_protocol_name: ProtocolName, @@ -247,7 +247,7 @@ pub async fn start_beefy_gadget( let BeefyNetworkParams { network, sync, - notification_handle, + notification_service, gossip_protocol_name, justifications_protocol_name, .. @@ -262,7 +262,7 @@ pub async fn start_beefy_gadget( let mut gossip_engine = GossipEngine::new( network.clone(), sync.clone(), - notification_handle, + notification_service, gossip_protocol_name, gossip_validator.clone(), None, diff --git a/client/consensus/beefy/src/tests.rs b/client/consensus/beefy/src/tests.rs index 3f83117a0052e..3c5344e1cffcb 100644 --- a/client/consensus/beefy/src/tests.rs +++ b/client/consensus/beefy/src/tests.rs @@ -369,7 +369,7 @@ async fn voter_init_setup( let mut gossip_engine = sc_network_gossip::GossipEngine::new( net.peer(0).network_service().clone(), net.peer(0).sync_service().clone(), - net.peer(0).take_notification_handle(&beefy_gossip_proto_name()).unwrap(), + net.peer(0).take_notification_service(&beefy_gossip_proto_name()).unwrap(), "/beefy/whatever", gossip_validator, None, @@ -390,11 +390,11 @@ where { let tasks = FuturesUnordered::new(); - let mut notification_handles = peers + let mut notification_services = peers .iter() .map(|(peer_id, _, _)| { let peer = &mut net.peers[*peer_id]; - (*peer_id, peer.take_notification_handle(&beefy_gossip_proto_name()).unwrap()) + (*peer_id, peer.take_notification_service(&beefy_gossip_proto_name()).unwrap()) }) .collect::>(); @@ -415,7 +415,7 @@ where let network_params = crate::BeefyNetworkParams { network: peer.network_service().clone(), sync: peer.sync_service().clone(), - notification_handle: notification_handles.remove(&peer_id).unwrap(), + notification_service: notification_services.remove(&peer_id).unwrap(), gossip_protocol_name: beefy_gossip_proto_name(), justifications_protocol_name: on_demand_justif_handler.protocol_name(), _phantom: PhantomData, @@ -1289,7 +1289,7 @@ async fn gossipped_finality_proofs() { let mut charlie_gossip_engine = sc_network_gossip::GossipEngine::new( charlie.network_service().clone(), charlie.sync_service().clone(), - charlie.take_notification_handle(&beefy_gossip_proto_name()).unwrap(), + charlie.take_notification_service(&beefy_gossip_proto_name()).unwrap(), beefy_gossip_proto_name(), charlie_gossip_validator.clone(), None, diff --git a/client/consensus/beefy/src/worker.rs b/client/consensus/beefy/src/worker.rs index b87d67d371a0b..c5cb08f27f409 100644 --- a/client/consensus/beefy/src/worker.rs +++ b/client/consensus/beefy/src/worker.rs @@ -1089,15 +1089,16 @@ pub(crate) mod tests { let api = Arc::new(TestApi::with_validator_set(&genesis_validator_set)); let network = peer.network_service().clone(); let sync = peer.sync_service().clone(); - let notification_handle = - peer.take_notification_handle(&crate::tests::beefy_gossip_proto_name()).unwrap(); + let notification_service = peer + .take_notification_service(&crate::tests::beefy_gossip_proto_name()) + .unwrap(); let known_peers = Arc::new(Mutex::new(KnownPeers::new())); let (gossip_validator, gossip_report_stream) = GossipValidator::new(known_peers.clone()); let gossip_validator = Arc::new(gossip_validator); let gossip_engine = GossipEngine::new( network.clone(), sync.clone(), - notification_handle, + notification_service, "/beefy/1", gossip_validator.clone(), None, diff --git a/client/consensus/grandpa/src/communication/mod.rs b/client/consensus/grandpa/src/communication/mod.rs index 23cc1ecd7d470..da2aaebbdc1a3 100644 --- a/client/consensus/grandpa/src/communication/mod.rs +++ b/client/consensus/grandpa/src/communication/mod.rs @@ -247,7 +247,7 @@ impl, S: Syncing> NetworkBridge { pub(crate) fn new( service: N, sync: S, - notification_handle: Box, + notification_service: Box, config: crate::Config, set_state: crate::environment::SharedVoterSetState, prometheus_registry: Option<&Registry>, @@ -261,7 +261,7 @@ impl, S: Syncing> NetworkBridge { let gossip_engine = Arc::new(Mutex::new(GossipEngine::new( service.clone(), sync.clone(), - notification_handle, + notification_service, protocol, validator.clone(), prometheus_registry, diff --git a/client/consensus/grandpa/src/communication/tests.rs b/client/consensus/grandpa/src/communication/tests.rs index 3929f21ee8a5a..16f03bc0b6129 100644 --- a/client/consensus/grandpa/src/communication/tests.rs +++ b/client/consensus/grandpa/src/communication/tests.rs @@ -272,6 +272,10 @@ impl NotificationService for TestNotificationService { fn clone(&mut self) -> Result, ()> { unimplemented!(); } + + fn protocol(&self) -> &ProtocolName { + unimplemented!(); + } } pub(crate) struct Tester { @@ -345,7 +349,7 @@ pub(crate) fn make_test_network() -> (impl Future, TestNetwork) let (tx, rx) = tracing_unbounded("test", 100_000); let (notification_tx, notification_rx) = tracing_unbounded("test-notification", 100_000); - let notification_handle = TestNotificationService { rx: notification_rx }; + let notification_service = TestNotificationService { rx: notification_rx }; let net = TestNetwork { sender: tx }; let sync = TestSync {}; @@ -363,7 +367,7 @@ pub(crate) fn make_test_network() -> (impl Future, TestNetwork) let bridge = super::NetworkBridge::new( net.clone(), sync, - Box::new(notification_handle), + Box::new(notification_service), config(), voter_set_state(), None, diff --git a/client/consensus/grandpa/src/lib.rs b/client/consensus/grandpa/src/lib.rs index 66423d59f5d53..13f2620698712 100644 --- a/client/consensus/grandpa/src/lib.rs +++ b/client/consensus/grandpa/src/lib.rs @@ -681,7 +681,7 @@ pub struct GrandpaParams { /// Event stream for syncing-related events. pub sync: S, /// Handle for interacting with `Notifications`. - pub notification_handle: Box, + pub notification_service: Box, /// A voting rule used to potentially restrict target votes. pub voting_rule: VR, /// The prometheus metrics registry. @@ -735,7 +735,7 @@ where link, network, sync, - notification_handle, + notification_service, voting_rule, prometheus_registry, shared_voter_state, @@ -761,7 +761,7 @@ where let network = NetworkBridge::new( network, sync, - notification_handle, + notification_service, config.clone(), persistent_data.set_state.clone(), prometheus_registry.as_ref(), diff --git a/client/consensus/grandpa/src/observer.rs b/client/consensus/grandpa/src/observer.rs index d8503b653a3e2..608ff5e46a0e8 100644 --- a/client/consensus/grandpa/src/observer.rs +++ b/client/consensus/grandpa/src/observer.rs @@ -169,7 +169,7 @@ pub fn run_grandpa_observer( link: LinkHalf, network: N, sync: S, - notification_handle: Box, + notification_service: Box, ) -> sp_blockchain::Result + Send> where BE: Backend + Unpin + 'static, @@ -191,7 +191,7 @@ where let network = NetworkBridge::new( network, sync, - notification_handle, + notification_service, config.clone(), persistent_data.set_state.clone(), None, diff --git a/client/consensus/grandpa/src/tests.rs b/client/consensus/grandpa/src/tests.rs index e111948c909f3..ca6ad4693821e 100644 --- a/client/consensus/grandpa/src/tests.rs +++ b/client/consensus/grandpa/src/tests.rs @@ -313,8 +313,8 @@ fn initialize_grandpa( (net.peers[peer_id].network_service().clone(), link) }; let sync = net.peers[peer_id].sync_service().clone(); - let notification_handle = net.peers[peer_id] - .take_notification_handle(&grandpa_protocol_name::NAME.into()) + let notification_service = net.peers[peer_id] + .take_notification_service(&grandpa_protocol_name::NAME.into()) .unwrap(); let grandpa_params = GrandpaParams { @@ -331,7 +331,7 @@ fn initialize_grandpa( link, network: net_service, sync, - notification_handle, + notification_service, voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -466,8 +466,8 @@ async fn finalize_3_voters_1_full_observer() { let net_service = net.peers[peer_id].network_service().clone(); let sync = net.peers[peer_id].sync_service().clone(); let link = net.peers[peer_id].data.lock().take().expect("link initialized at startup; qed"); - let notification_handle = net.peers[peer_id] - .take_notification_handle(&grandpa_protocol_name::NAME.into()) + let notification_service = net.peers[peer_id] + .take_notification_service(&grandpa_protocol_name::NAME.into()) .unwrap(); let grandpa_params = GrandpaParams { @@ -484,7 +484,7 @@ async fn finalize_3_voters_1_full_observer() { link, network: net_service, sync, - notification_handle, + notification_service, voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -552,7 +552,7 @@ async fn transition_3_voters_twice_1_full_observer() { for (peer_id, local_key) in all_peers.clone().into_iter().enumerate() { let keystore = create_keystore(local_key); - let (net_service, link, sync, notification_handle) = { + let (net_service, link, sync, notification_service) = { let mut net = net.lock(); let link = net.peers[peer_id].data.lock().take().expect("link initialized at startup; qed"); @@ -561,7 +561,7 @@ async fn transition_3_voters_twice_1_full_observer() { link, net.peers[peer_id].sync_service().clone(), net.peers[peer_id] - .take_notification_handle(&grandpa_protocol_name::NAME.into()) + .take_notification_service(&grandpa_protocol_name::NAME.into()) .unwrap(), ) }; @@ -580,7 +580,7 @@ async fn transition_3_voters_twice_1_full_observer() { link, network: net_service, sync, - notification_handle, + notification_service, voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1016,7 +1016,7 @@ async fn voter_persists_its_votes() { net.peers[1].network_service().clone(), net.peers[1].sync_service().clone(), net.peers[1] - .take_notification_handle(&grandpa_protocol_name::NAME.into()) + .take_notification_service(&grandpa_protocol_name::NAME.into()) .unwrap(), config.clone(), set_state, @@ -1036,8 +1036,8 @@ async fn voter_persists_its_votes() { (net.peers[0].network_service().clone(), link) }; let sync = net.peers[0].sync_service().clone(); - let notification_handle = net.peers[0] - .take_notification_handle(&grandpa_protocol_name::NAME.into()) + let notification_service = net.peers[0] + .take_notification_service(&grandpa_protocol_name::NAME.into()) .unwrap(); let grandpa_params = GrandpaParams { @@ -1054,7 +1054,7 @@ async fn voter_persists_its_votes() { link, network: net_service, sync, - notification_handle, + notification_service, voting_rule: VotingRulesBuilder::default().build(), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1076,8 +1076,8 @@ async fn voter_persists_its_votes() { net.add_authority_peer(); let net_service = net.peers[2].network_service().clone(); let sync = net.peers[2].sync_service().clone(); - let notification_handle = net.peers[2] - .take_notification_handle(&grandpa_protocol_name::NAME.into()) + let notification_service = net.peers[2] + .take_notification_service(&grandpa_protocol_name::NAME.into()) .unwrap(); // but we'll reuse the client from the first peer (alice_voter1) // since we want to share the same database, so that we can @@ -1101,7 +1101,7 @@ async fn voter_persists_its_votes() { link, network: net_service, sync, - notification_handle, + notification_service, voting_rule: VotingRulesBuilder::default().build(), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1251,7 +1251,7 @@ async fn finalize_3_voters_1_light_observer() { let mut net = GrandpaTestNet::new(TestApi::new(voters), 3, 1); let voters = initialize_grandpa(&mut net, authorities); let notification_service = net.peers[3] - .take_notification_handle(&grandpa_protocol_name::NAME.into()) + .take_notification_service(&grandpa_protocol_name::NAME.into()) .unwrap(); let observer = observer::run_grandpa_observer( Config { @@ -1316,9 +1316,9 @@ async fn voter_catches_up_to_latest_round_when_behind() { link, network: net.peer(peer_id).network_service().clone(), sync: net.peer(peer_id).sync_service().clone(), - notification_handle: net + notification_service: net .peer(peer_id) - .take_notification_handle(&grandpa_protocol_name::NAME.into()) + .take_notification_service(&grandpa_protocol_name::NAME.into()) .unwrap(), voting_rule: (), prometheus_registry: None, @@ -1494,7 +1494,7 @@ async fn grandpa_environment_respects_voting_rules() { let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); let mut notification_service = - peer.take_notification_handle(&grandpa_protocol_name::NAME.into()).unwrap(); + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); // add 21 blocks @@ -1622,7 +1622,7 @@ async fn grandpa_environment_passes_actual_best_block_to_voting_rules() { let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); let notification_service = - peer.take_notification_handle(&grandpa_protocol_name::NAME.into()).unwrap(); + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let client = peer.client().as_client().clone(); let select_chain = MockSelectChain::default(); @@ -1686,7 +1686,7 @@ async fn grandpa_environment_checks_if_best_block_is_descendent_of_finality_targ let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); let notification_service = - peer.take_notification_handle(&grandpa_protocol_name::NAME.into()).unwrap(); + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let client = peer.client().as_client().clone(); let select_chain = MockSelectChain::default(); @@ -1800,7 +1800,7 @@ async fn grandpa_environment_never_overwrites_round_voter_state() { let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); let notification_service = - peer.take_notification_handle(&grandpa_protocol_name::NAME.into()).unwrap(); + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let keystore = create_keystore(peers[0]); @@ -2014,7 +2014,7 @@ async fn grandpa_environment_doesnt_send_equivocation_reports_for_itself() { let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); let notification_service = - peer.take_notification_handle(&grandpa_protocol_name::NAME.into()).unwrap(); + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let keystore = create_keystore(alice); test_environment( diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 8382ec4297c19..16407664c48dd 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -56,7 +56,7 @@ pub struct GossipEngine { /// Incoming events from the syncing service. sync_event_stream: Pin + Send>>, /// Handle for polling notification-related events. - notification_handle: Box, + notification_service: Box, /// Outgoing events to the consumer. message_sinks: HashMap>>, /// Buffered messages (see [`ForwardingState`]). @@ -86,7 +86,7 @@ impl GossipEngine { pub fn new( network: N, sync: S, - notification_handle: Box, + notification_service: Box, protocol: impl Into, validator: Arc>, metrics_registry: Option<&Registry>, @@ -103,7 +103,7 @@ impl GossipEngine { state_machine: ConsensusGossip::new(validator, protocol.clone(), metrics_registry), network: Box::new(network), sync: Box::new(sync), - notification_handle, + notification_service, periodic_maintenance_interval: futures_timer::Delay::new(PERIODIC_MAINTENANCE_INTERVAL), protocol, @@ -189,7 +189,7 @@ impl Future for GossipEngine { 'outer: loop { match &mut this.forwarding_state { ForwardingState::Idle => { - let next_notification = this.notification_handle.next_event().poll_unpin(cx); + let next_notification = this.notification_service.next_event().poll_unpin(cx); let sync_event_stream = this.sync_event_stream.poll_next_unpin(cx); if next_notification.is_pending() && sync_event_stream.is_pending() { @@ -554,6 +554,10 @@ mod tests { fn clone(&mut self) -> Result, ()> { unimplemented!(); } + + fn protocol(&self) -> &ProtocolName { + unimplemented!(); + } } struct AllowAll; diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 57960822ff75f..20dd6799e259f 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -531,7 +531,7 @@ impl NonDefaultSetConfig { handshake: Option, set_config: SetConfig, ) -> (Self, Box) { - let (protocol_handle_pair, notification_handle) = + let (protocol_handle_pair, notification_service) = notification_service(protocol_name.clone()); ( Self { @@ -542,7 +542,7 @@ impl NonDefaultSetConfig { set_config, protocol_handle_pair: Some(protocol_handle_pair), }, - notification_handle, + notification_service, ) } diff --git a/client/network/src/protocol/notifications/service.rs b/client/network/src/protocol/notifications/service.rs index 48b101fcd1473..00c506fcb4b4a 100644 --- a/client/network/src/protocol/notifications/service.rs +++ b/client/network/src/protocol/notifications/service.rs @@ -227,10 +227,10 @@ impl NotificationService for NotificationHandle { } } + // Clone [`NotificationService`] fn clone(&mut self) -> Result, ()> { let mut subscribers = self.subscribers.lock().map_err(|_| ())?; - let (event_tx, event_rx) = - tracing_unbounded("mpsc-notification-to-protocol-clonable", 100_000); + let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); subscribers.push(event_tx); Ok(Box::new(NotificationHandle { @@ -242,6 +242,7 @@ impl NotificationService for NotificationHandle { })) } + /// Get protocol name. fn protocol(&self) -> &ProtocolName { &self.protocol } diff --git a/client/network/statement/src/lib.rs b/client/network/statement/src/lib.rs index ac9b5f0efc1dc..350c4d4994994 100644 --- a/client/network/statement/src/lib.rs +++ b/client/network/statement/src/lib.rs @@ -35,10 +35,10 @@ use prometheus_endpoint::{register, Counter, PrometheusError, Registry, U64}; use sc_network::{ config::{NonDefaultSetConfig, NonReservedPeerMode, SetConfig}, error, - event::Event, + service::traits::{NotificationEvent, NotificationService, ValidationResult}, types::ProtocolName, utils::{interval, LruHashSet}, - NetworkEventStream, NetworkNotification, NetworkPeers, NotificationService, + NetworkEventStream, NetworkNotification, NetworkPeers, }; use sc_network_common::{ role::ObservedRole, @@ -150,7 +150,6 @@ impl StatementHandlerPrototype { metrics_registry: Option<&Registry>, executor: impl Fn(Pin + Send>>) + Send, ) -> error::Result> { - let net_event_stream = network.event_stream("statement-handler-net"); let sync_event_stream = sync.event_stream("statement-handler-sync"); let (queue_sender, mut queue_receiver) = async_channel::bounded(100_000); @@ -187,7 +186,6 @@ impl StatementHandlerPrototype { pending_statements_peers: HashMap::new(), network, sync, - net_event_stream: net_event_stream.fuse(), sync_event_stream: sync_event_stream.fuse(), peers: HashMap::new(), statement_store, @@ -223,8 +221,6 @@ pub struct StatementHandler< network: N, /// Syncing service. sync: S, - /// Stream of networking events. - net_event_stream: stream::Fuse + Send>>>, /// Receiver for syncing-related events. sync_event_stream: stream::Fuse + Send>>>, /// Notification service. @@ -267,14 +263,6 @@ where log::warn!(target: LOG_TARGET, "Inconsistent state, no peers for pending statement!"); } }, - network_event = self.net_event_stream.next() => { - if let Some(network_event) = network_event { - self.handle_network_event(network_event).await; - } else { - // Networking has seemingly closed. Closing as well. - return; - } - }, sync_event = self.sync_event_stream.next() => { if let Some(sync_event) = sync_event { self.handle_sync_event(sync_event); @@ -283,6 +271,14 @@ where return; } } + event = self.notification_service.next_event().fuse() => { + if let Some(event) = event { + self.handle_notification_event(event) + } else { + // `Notifications` has seemingly closed. Closing as well. + return + } + } } } } @@ -309,14 +305,14 @@ where } } - async fn handle_network_event(&mut self, event: Event) { + fn handle_notification_event(&mut self, event: NotificationEvent) { match event { - Event::Dht(_) => {}, - Event::NotificationStreamOpened { remote, protocol, role, .. } - if protocol == self.protocol_name => - { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + let _ = result_tx.send(ValidationResult::Accept); + }, + NotificationEvent::NotificationStreamOpened { peer, role, .. } => { let _was_in = self.peers.insert( - remote, + peer, Peer { known_statements: LruHashSet::new( NonZeroUsize::new(MAX_KNOWN_STATEMENTS).expect("Constant is nonzero"), @@ -326,39 +322,26 @@ where ); debug_assert!(_was_in.is_none()); }, - Event::NotificationStreamClosed { remote, protocol } - if protocol == self.protocol_name => - { - let _peer = self.peers.remove(&remote); + NotificationEvent::NotificationStreamClosed { peer } => { + let _peer = self.peers.remove(&peer); debug_assert!(_peer.is_some()); }, + NotificationEvent::NotificationReceived { peer, notification } => { + // Accept statements only when node is not major syncing + if self.sync.is_major_syncing() { + log::trace!( + target: LOG_TARGET, + "{peer}: Ignoring statements while major syncing or offline" + ); + return + } - Event::NotificationsReceived { remote, messages } => { - for (protocol, message) in messages { - if protocol != self.protocol_name { - continue - } - // Accept statements only when node is not major syncing - if self.sync.is_major_syncing() { - log::trace!( - target: LOG_TARGET, - "{remote}: Ignoring statements while major syncing or offline" - ); - continue - } - if let Ok(statements) = ::decode(&mut message.as_ref()) { - self.on_statements(remote, statements); - } else { - log::debug!( - target: LOG_TARGET, - "Failed to decode statement list from {remote}" - ); - } + if let Ok(statements) = ::decode(&mut notification.as_ref()) { + self.on_statements(peer, statements); + } else { + log::debug!(target: LOG_TARGET, "Failed to decode statement list from {peer}"); } }, - - // Not our concern. - Event::NotificationStreamOpened { .. } | Event::NotificationStreamClosed { .. } => {}, } } diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index 68fb9a1d9efc9..0fed553c7ba5f 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -245,7 +245,7 @@ pub struct SyncingEngine { metrics: Option, /// Handle that is used to communicate with `sc_network::Notifications`. - notification_handle: Box, + notification_service: Box, } impl SyncingEngine @@ -341,7 +341,7 @@ where total.saturating_sub(net_config.network_config.default_peers_set_num_full) as usize }; - let (chain_sync, block_announce_config, notification_handle) = ChainSync::new( + let (chain_sync, block_announce_config, notification_service) = ChainSync::new( mode, client.clone(), protocol_id, @@ -390,7 +390,7 @@ where default_peers_set_num_full, default_peers_set_num_light, event_streams: Vec::new(), - notification_handle, + notification_service, tick_timeout: Delay::new(TICK_TIMEOUT), metrics: if let Some(r) = metrics_registry { match Metrics::register(r, is_major_syncing.clone()) { @@ -592,7 +592,7 @@ where }; peer.last_notification_sent = Instant::now(); - let _ = self.notification_handle.send_sync_notification(who, message.encode()); + let _ = self.notification_service.send_sync_notification(who, message.encode()); } } } @@ -728,7 +728,7 @@ where } loop { - let Poll::Ready(Some(event)) = self.notification_handle.next_event().poll_unpin(cx) else { + let Poll::Ready(Some(event)) = self.notification_service.next_event().poll_unpin(cx) else { break; }; diff --git a/client/network/sync/src/lib.rs b/client/network/sync/src/lib.rs index 89957390e96b9..b71d247a308aa 100644 --- a/client/network/sync/src/lib.rs +++ b/client/network/sync/src/lib.rs @@ -1420,7 +1420,7 @@ where state_request_protocol_name: ProtocolName, warp_sync_protocol_name: Option, ) -> Result<(Self, NonDefaultSetConfig, Box), ClientError> { - let (block_announce_config, notification_handle) = Self::get_block_announce_proto_config( + let (block_announce_config, notification_service) = Self::get_block_announce_proto_config( protocol_id, fork_id, roles, @@ -1476,7 +1476,7 @@ where }; sync.reset_sync_start_point()?; - Ok((sync, block_announce_config, notification_handle)) + Ok((sync, block_announce_config, notification_service)) } /// Returns the median seen block number. diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index a023f70dbdaac..5bc57d53d8bff 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -240,7 +240,7 @@ pub struct Peer { imported_blocks_stream: Pin> + Send>>, finality_notification_stream: Pin> + Send>>, listen_addr: Multiaddr, - notification_handles: HashMap>, + notification_services: HashMap>, } impl Peer @@ -512,11 +512,11 @@ where } /// Take notification handle for enabled protocol. - pub fn take_notification_handle( + pub fn take_notification_service( &mut self, protocol: &ProtocolName, ) -> Option> { - self.notification_handles.remove(protocol) + self.notification_services.remove(protocol) } /// Get a reference to the network worker. @@ -985,7 +985,7 @@ where backend: Some(backend), imported_blocks_stream, finality_notification_stream, - notification_handles: HashMap::from_iter(notif_handles.into_iter()), + notification_services: HashMap::from_iter(notif_handles.into_iter()), block_import, verifier, network, diff --git a/client/network/test/src/service.rs b/client/network/test/src/service.rs index ed4e0cb7aaa31..8277626af7d48 100644 --- a/client/network/test/src/service.rs +++ b/client/network/test/src/service.rs @@ -24,8 +24,8 @@ use sc_network::{ config::{self, FullNetworkConfiguration, MultiaddrWithPeerId, ProtocolId, TransportConfig}, event::Event, service::traits::{NotificationEvent, ValidationResult}, - NetworkEventStream, NetworkNotification, NetworkPeers, NetworkService, NetworkStateInfo, - NetworkWorker, NotificationService, + NetworkEventStream, NetworkPeers, NetworkService, NetworkStateInfo, NetworkWorker, + NotificationService, }; use sc_network_common::role::Roles; use sc_network_light::light_client_requests::handler::LightClientRequestHandler; @@ -116,7 +116,7 @@ impl TestNetworkBuilder { self } - pub fn build(mut self) -> (TestNetwork, Box) { + pub fn build(mut self) -> (TestNetwork, Option>) { let client = self.client.as_mut().map_or( Arc::new(TestClientBuilder::with_default_backend().build_with_longest_chain().0), |v| v.clone(), @@ -199,10 +199,11 @@ impl TestNetworkBuilder { .unwrap(); let mut link = self.link.unwrap_or(Box::new(chain_sync_service.clone())); - if !self.notification_protocols.is_empty() { + let handle = if !self.notification_protocols.is_empty() { for config in self.notification_protocols { full_net_config.add_notification_protocol(config); } + None } else { let (config, handle) = config::NonDefaultSetConfig::new( PROTOCOL_NAME.into(), @@ -212,7 +213,8 @@ impl TestNetworkBuilder { self.set_config.unwrap_or_default(), ); full_net_config.add_notification_protocol(config); - } + Some(handle) + }; for config in [ block_request_protocol_config, @@ -266,9 +268,9 @@ impl TestNetworkBuilder { /// The nodes are connected together and have the `PROTOCOL_NAME` protocol registered. fn build_nodes_one_proto() -> ( Arc, - Box, + Option>, Arc, - Box, + Option>, ) { let listen_addr = config::build_multiaddr![Memory(rand::random::())]; @@ -297,7 +299,8 @@ async fn notifications_state_consistent() { // Runs two nodes and ensures that events are propagated out of the API in a consistent // correct order, which means no notification received on a closed substream. - let (node1, mut handle1, node2, mut handle2) = build_nodes_one_proto(); + let (node1, handle1, node2, handle2) = build_nodes_one_proto(); + let (mut handle1, mut handle2) = (handle1.unwrap(), handle2.unwrap()); // Write some initial notifications that shouldn't get through. for _ in 0..(rand::random::() % 5) { @@ -416,10 +419,11 @@ async fn lots_of_incoming_peers_works() { sp_tracing::try_init_simple(); let listen_addr = config::build_multiaddr![Memory(rand::random::())]; - let (main_node, mut handle1) = TestNetworkBuilder::new() + let (main_node, handle1) = TestNetworkBuilder::new() .with_listen_addresses(vec![listen_addr.clone()]) .with_set_config(config::SetConfig { in_peers: u32::MAX, ..Default::default() }) .build(); + let mut handle1 = handle1.unwrap(); let (main_node, _) = main_node.start_network(); let main_node_peer_id = main_node.local_peer_id(); @@ -437,7 +441,7 @@ async fn lots_of_incoming_peers_works() { let mut background_tasks_to_wait = Vec::new(); for _ in 0..32 { - let (dialing_node, mut handle) = TestNetworkBuilder::new() + let (dialing_node, handle) = TestNetworkBuilder::new() .with_set_config(config::SetConfig { reserved_nodes: vec![MultiaddrWithPeerId { multiaddr: listen_addr.clone(), @@ -446,6 +450,7 @@ async fn lots_of_incoming_peers_works() { ..Default::default() }) .build(); + let mut handle = handle.unwrap(); let (_, _) = dialing_node.start_network(); background_tasks_to_wait.push(tokio::spawn(async move { @@ -480,69 +485,63 @@ async fn lots_of_incoming_peers_works() { future::join_all(background_tasks_to_wait).await; } -// #[tokio::test] -// async fn notifications_back_pressure() { -// // Node 1 floods node 2 with notifications. Random sleeps are done on node 2 to simulate the -// // node being busy. We make sure that all notifications are received. - -// const TOTAL_NOTIFS: usize = 10_000; - -// let (node1, mut events_stream1, node2, mut events_stream2) = build_nodes_one_proto(); -// let node2_id = node2.local_peer_id(); - -// let receiver = tokio::spawn(async move { -// let mut received_notifications = 0; -// let mut sync_protocol_name = None; - -// while received_notifications < TOTAL_NOTIFS { -// match events_stream2.next().await.unwrap() { -// Event::NotificationStreamOpened { protocol, .. } => { -// if let None = sync_protocol_name { -// sync_protocol_name = Some(protocol); -// } -// }, -// Event::NotificationStreamClosed { protocol, .. } => { -// if Some(&protocol) != sync_protocol_name.as_ref() { -// panic!() -// } -// }, -// Event::NotificationsReceived { messages, .. } => { -// for message in messages { -// assert_eq!(message.0, PROTOCOL_NAME.into()); -// assert_eq!(message.1, format!("hello #{}", received_notifications)); -// received_notifications += 1; -// } -// }, -// _ => {}, -// }; - -// if rand::random::() < 2 { -// tokio::time::sleep(Duration::from_millis(rand::random::() % 750)).await; -// } -// } -// }); - -// // Wait for the `NotificationStreamOpened`. -// loop { -// match events_stream1.next().await.unwrap() { -// Event::NotificationStreamOpened { .. } => break, -// _ => {}, -// }; -// } - -// // Sending! -// for num in 0..TOTAL_NOTIFS { -// let notif = node1.notification_sender(node2_id, PROTOCOL_NAME.into()).unwrap(); -// notif -// .ready() -// .await -// .unwrap() -// .send(format!("hello #{}", num).into_bytes()) -// .unwrap(); -// } - -// receiver.await.unwrap(); -// } +#[tokio::test] +async fn notifications_back_pressure() { + // Node 1 floods node 2 with notifications. Random sleeps are done on node 2 to simulate the + // node being busy. We make sure that all notifications are received. + + const TOTAL_NOTIFS: usize = 10_000; + + let (node1, handle1, node2, handle2) = build_nodes_one_proto(); + let (mut handle1, mut handle2) = (handle1.unwrap(), handle2.unwrap()); + let node2_id = node2.local_peer_id(); + + let receiver = tokio::spawn(async move { + let mut received_notifications = 0; + // let mut sync_protocol_name = None; + + while received_notifications < TOTAL_NOTIFS { + match handle2.next_event().await.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + NotificationEvent::NotificationReceived { notification, .. } => { + assert_eq!( + notification, + format!("hello #{}", received_notifications).into_bytes() + ); + received_notifications += 1; + }, + _ => {}, + } + + if rand::random::() < 2 { + tokio::time::sleep(Duration::from_millis(rand::random::() % 750)).await; + } + } + }); + + // Wait for the `NotificationStreamOpened`. + loop { + match handle1.next_event().await.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + NotificationEvent::NotificationStreamOpened { .. } => break, + _ => {}, + }; + } + + // Sending! + for num in 0..TOTAL_NOTIFS { + handle1 + .send_async_notification(&node2_id, format!("hello #{}", num).into_bytes()) + .await + .unwrap(); + } + + receiver.await.unwrap(); +} #[tokio::test] async fn fallback_name_working() { @@ -559,15 +558,8 @@ async fn fallback_name_working() { None, Default::default(), ); - // <<<<<<< HEAD - let (node1, mut events_stream1) = TestNetworkBuilder::new() + let (network1, _) = TestNetworkBuilder::new() .with_notification_protocol(config) - // ||||||| parent of 6114a5d425 (Validate inbound substream before emitting - // `NotificationStreamOpened`) let (node1, mut events_stream1) = TestNetworkBuilder::new() - // ======= - // let (network1, _) = TestNetworkBuilder::new() - // >>>>>>> 6114a5d425 (Validate inbound substream before emitting - // `NotificationStreamOpened`) .with_config(config::NetworkConfiguration { listen_addresses: vec![listen_addr.clone()], transport: TransportConfig::MemoryOnly, @@ -577,7 +569,7 @@ async fn fallback_name_working() { let (node1, _) = network1.start_network(); - let (network2, mut handle2) = TestNetworkBuilder::new() + let (network2, handle2) = TestNetworkBuilder::new() .with_set_config(config::SetConfig { reserved_nodes: vec![MultiaddrWithPeerId { multiaddr: listen_addr, @@ -586,6 +578,7 @@ async fn fallback_name_working() { ..Default::default() }) .build(); + let mut handle2 = handle2.unwrap(); let _ = network2.start_network(); let receiver = tokio::spawn(async move { diff --git a/client/network/transactions/src/lib.rs b/client/network/transactions/src/lib.rs index f2d46c4f18495..2f4b5133f95f6 100644 --- a/client/network/transactions/src/lib.rs +++ b/client/network/transactions/src/lib.rs @@ -124,7 +124,7 @@ pub struct TransactionsHandlerPrototype { protocol_name: ProtocolName, /// Handle that is used to communicate with `sc_network::Notifications`. - notification_handle: Box, + notification_service: Box, } impl TransactionsHandlerPrototype { @@ -141,7 +141,7 @@ impl TransactionsHandlerPrototype { format!("/{}/transactions/1", array_bytes::bytes2hex("", genesis_hash)) } .into(); - let (config, notification_handle) = NonDefaultSetConfig::new( + let (config, notification_service) = NonDefaultSetConfig::new( protocol_name.clone(), vec![format!("/{}/transactions/1", protocol_id.as_ref()).into()], MAX_TRANSACTIONS_SIZE, @@ -154,7 +154,7 @@ impl TransactionsHandlerPrototype { }, ); - (Self { protocol_name, notification_handle }, config) + (Self { protocol_name, notification_service }, config) } /// Turns the prototype into the actual handler. Returns a controller that allows controlling @@ -179,7 +179,7 @@ impl TransactionsHandlerPrototype { let handler = TransactionsHandler { protocol_name: self.protocol_name, - notification_handle: self.notification_handle, + notification_service: self.notification_service, propagate_timeout: (Box::pin(interval(PROPAGATE_TIMEOUT)) as Pin + Send>>) .fuse(), @@ -262,7 +262,7 @@ pub struct TransactionsHandler< /// Prometheus metrics. metrics: Option, /// Handle that is used to communicate with `sc_network::Notifications`. - notification_handle: Box, + notification_service: Box, } /// Peer information @@ -309,7 +309,7 @@ where ToHandler::PropagateTransactions => self.propagate_transactions(), } }, - event = self.notification_handle.next_event().fuse() => { + event = self.notification_service.next_event().fuse() => { if let Some(event) = event { self.handle_notification_event(event) } else { @@ -468,7 +468,7 @@ where } trace!(target: "sync", "Sending {} transactions to {}", to_send.len(), who); // TODO: don't ignore error - let _ = self.notification_handle.send_sync_notification(who, to_send.encode()); + let _ = self.notification_service.send_sync_notification(who, to_send.encode()); } } From 2e9da9c80774fcd8428f2dce68d13cdde6133528 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 16 May 2023 11:27:40 +0300 Subject: [PATCH 16/43] Split `NotificationService` files logically --- .../src/protocol/notifications/behaviour.rs | 149 ++- .../src/protocol/notifications/service.rs | 1037 ----------------- .../protocol/notifications/service/metrics.rs | 17 + .../src/protocol/notifications/service/mod.rs | 463 ++++++++ .../protocol/notifications/service/tests.rs | 579 +++++++++ 5 files changed, 1132 insertions(+), 1113 deletions(-) delete mode 100644 client/network/src/protocol/notifications/service.rs create mode 100644 client/network/src/protocol/notifications/service/metrics.rs create mode 100644 client/network/src/protocol/notifications/service/mod.rs create mode 100644 client/network/src/protocol/notifications/service/tests.rs diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 6e7db6a5624a5..39c0440230abd 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -19,7 +19,7 @@ use crate::{ config::ProtocolHandlePair, protocol::notifications::{ - handler::{self, NotificationsSink, NotifsHandlerIn, NotifsHandlerOut, NotifsHandlerProto}, + handler::{self, NotificationsSink, NotifsHandler, NotifsHandlerIn, NotifsHandlerOut}, service::{NotificationCommand, ProtocolHandle}, }, service::traits::ValidationResult, @@ -482,7 +482,7 @@ impl Notifications { let mut entry = if let Entry::Occupied(entry) = self.peers.entry((*peer_id, set_id)) { entry } else { - return; + return }; match mem::replace(entry.get_mut(), PeerState::Poisoned) { @@ -565,7 +565,7 @@ impl Notifications { target: "sub-libp2p", "State mismatch in libp2p: no entry in incoming for incoming peer" ); - return; + return }; inc.alive = false; @@ -621,7 +621,7 @@ impl Notifications { trace!(target: "sub-libp2p", "Libp2p <= Dial {}", entry.key().0); self.events.push_back(ToSwarm::Dial { opts: entry.key().0.into() }); entry.insert(PeerState::Requested); - return; + return }, }; @@ -821,7 +821,7 @@ impl Notifications { Entry::Vacant(entry) => { trace!(target: "sub-libp2p", "PSM => Drop({}, {:?}): Already disabled.", entry.key().0, set_id); - return; + return }, }; @@ -928,7 +928,7 @@ impl Notifications { self.incoming.remove(pos) } else { error!(target: LOG_TARGET, "Protocol => Accept({:?}): Invalid index", index); - return; + return }; if !incoming.alive { @@ -952,14 +952,14 @@ impl Notifications { self.peerset.dropped(incoming.set_id, incoming.peer_id, DropReason::Unknown); }, } - return; + return } let state = match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { Some(s) => s, None => { debug_assert!(false); - return; + return }, }; @@ -1031,7 +1031,7 @@ impl Notifications { self.incoming.remove(pos) } else { error!(target: LOG_TARGET, "PSM => Accept({:?}): Invalid index", index); - return; + return }; if !incoming.alive { @@ -1055,14 +1055,14 @@ impl Notifications { self.peerset.dropped(incoming.set_id, incoming.peer_id, DropReason::Unknown); }, } - return; + return } let state = match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { Some(s) => s, None => { debug_assert!(false); - return; + return }, }; @@ -1089,7 +1089,7 @@ impl Notifications { Err(err) => { error!(target: LOG_TARGET, "protocol has exited: {err:?}"); debug_assert!(false); - return; + return }, } @@ -1114,20 +1114,20 @@ impl Notifications { self.incoming.remove(pos) } else { error!(target: "sub-libp2p", "PSM => Reject({:?}): Invalid index", index); - return; + return }; if !incoming.alive { trace!(target: "sub-libp2p", "PSM => Reject({:?}, {}, {:?}): Obsolete incoming, \ ignoring", index, incoming.peer_id, incoming.set_id); - return; + return } let state = match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { Some(s) => s, None => { debug_assert!(false); - return; + return }, }; @@ -1228,8 +1228,8 @@ impl NetworkBehaviour for Notifications { for set_id in (0..self.notif_protocols.len()).map(sc_peerset::SetId::from) { match self.peers.entry((peer_id, set_id)).or_insert(PeerState::Poisoned) { // Requested | PendingRequest => Enabled - st @ &mut PeerState::Requested - | st @ &mut PeerState::PendingRequest { .. } => { + st @ &mut PeerState::Requested | + st @ &mut PeerState::PendingRequest { .. } => { trace!(target: "sub-libp2p", "Libp2p => Connected({}, {:?}, {:?}): Connection was requested by PSM.", peer_id, set_id, endpoint @@ -1266,10 +1266,10 @@ impl NetworkBehaviour for Notifications { // In all other states, add this new connection to the list of closed // inactive connections. - PeerState::Incoming { connections, .. } - | PeerState::Disabled { connections, .. } - | PeerState::DisabledPendingEnable { connections, .. } - | PeerState::Enabled { connections, .. } => { + PeerState::Incoming { connections, .. } | + PeerState::Disabled { connections, .. } | + PeerState::DisabledPendingEnable { connections, .. } | + PeerState::Enabled { connections, .. } => { trace!(target: "sub-libp2p", "Libp2p => Connected({}, {:?}, {:?}, {:?}): Secondary connection. Leaving closed.", peer_id, set_id, endpoint, connection_id); @@ -1287,7 +1287,7 @@ impl NetworkBehaviour for Notifications { } else { error!(target: "sub-libp2p", "inject_connection_closed: State mismatch in the custom protos handler"); debug_assert!(false); - return; + return }; match mem::replace(entry.get_mut(), PeerState::Poisoned) { @@ -1548,9 +1548,9 @@ impl NetworkBehaviour for Notifications { } }, - PeerState::Requested - | PeerState::PendingRequest { .. } - | PeerState::Backoff { .. } => { + PeerState::Requested | + PeerState::PendingRequest { .. } | + PeerState::Backoff { .. } => { // This is a serious bug either in this state machine or in libp2p. error!(target: "sub-libp2p", "`inject_connection_closed` called for unknown peer {}", @@ -1584,8 +1584,8 @@ impl NetworkBehaviour for Notifications { // "Basic" situation: we failed to reach a peer that the peerset // requested. - st @ PeerState::Requested - | st @ PeerState::PendingRequest { .. } => { + st @ PeerState::Requested | + st @ PeerState::PendingRequest { .. } => { trace!(target: "sub-libp2p", "PSM <= Dropped({}, {:?})", peer_id, set_id); self.peerset.dropped(set_id, peer_id, DropReason::Unknown); @@ -1593,9 +1593,7 @@ impl NetworkBehaviour for Notifications { let ban_duration = match st { PeerState::PendingRequest { timer_deadline, .. } if timer_deadline > now => - { - cmp::max(timer_deadline - now, Duration::from_secs(5)) - }, + cmp::max(timer_deadline - now, Duration::from_secs(5)), _ => Duration::from_secs(5), }; @@ -1619,10 +1617,10 @@ impl NetworkBehaviour for Notifications { // We can still get dial failures even if we are already connected // to the peer, as an extra diagnostic for an earlier attempt. - st @ PeerState::Disabled { .. } - | st @ PeerState::Enabled { .. } - | st @ PeerState::DisabledPendingEnable { .. } - | st @ PeerState::Incoming { .. } => { + st @ PeerState::Disabled { .. } | + st @ PeerState::Enabled { .. } | + st @ PeerState::DisabledPendingEnable { .. } | + st @ PeerState::Incoming { .. } => { *entry.into_mut() = st; }, @@ -1670,7 +1668,7 @@ impl NetworkBehaviour for Notifications { "OpenDesiredByRemote: State mismatch in the custom protos handler" ); debug_assert!(false); - return; + return }; match mem::replace(entry.get_mut(), PeerState::Poisoned) { @@ -1733,8 +1731,8 @@ impl NetworkBehaviour for Notifications { // more to do. debug_assert!(matches!( connec_state, - ConnectionState::OpenDesiredByRemote - | ConnectionState::Closing | ConnectionState::Opening + ConnectionState::OpenDesiredByRemote | + ConnectionState::Closing | ConnectionState::Opening )); } } else { @@ -1858,7 +1856,7 @@ impl NetworkBehaviour for Notifications { } else { error!(target: "sub-libp2p", "CloseDesired: State mismatch in the custom protos handler"); debug_assert!(false); - return; + return }; match mem::replace(entry.get_mut(), PeerState::Poisoned) { @@ -1877,12 +1875,12 @@ impl NetworkBehaviour for Notifications { error!(target: "sub-libp2p", "CloseDesired: State mismatch in the custom protos handler"); debug_assert!(false); - return; + return }; if matches!(connections[pos].1, ConnectionState::Closing) { *entry.into_mut() = PeerState::Enabled { connections }; - return; + return } debug_assert!(matches!(connections[pos].1, ConnectionState::Open(_))); @@ -1936,8 +1934,8 @@ impl NetworkBehaviour for Notifications { // All connections in `Disabled` and `DisabledPendingEnable` have been sent a // `Close` message already, and as such ignore any `CloseDesired` message. - state @ PeerState::Disabled { .. } - | state @ PeerState::DisabledPendingEnable { .. } => { + state @ PeerState::Disabled { .. } | + state @ PeerState::DisabledPendingEnable { .. } => { *entry.into_mut() = state; }, state => { @@ -1957,10 +1955,10 @@ impl NetworkBehaviour for Notifications { match self.peers.get_mut(&(peer_id, set_id)) { // Move the connection from `Closing` to `Closed`. - Some(PeerState::Incoming { connections, .. }) - | Some(PeerState::DisabledPendingEnable { connections, .. }) - | Some(PeerState::Disabled { connections, .. }) - | Some(PeerState::Enabled { connections, .. }) => { + Some(PeerState::Incoming { connections, .. }) | + Some(PeerState::DisabledPendingEnable { connections, .. }) | + Some(PeerState::Disabled { connections, .. }) | + Some(PeerState::Enabled { connections, .. }) => { if let Some((_, connec_state)) = connections.iter_mut().find(|(c, s)| { *c == connection_id && matches!(s, ConnectionState::Closing) }) { @@ -2028,8 +2026,8 @@ impl NetworkBehaviour for Notifications { *connec_state = ConnectionState::Open(notifications_sink); } else if let Some((_, connec_state)) = connections.iter_mut().find(|(c, s)| { - *c == connection_id - && matches!(s, ConnectionState::OpeningThenClosing) + *c == connection_id && + matches!(s, ConnectionState::OpeningThenClosing) }) { *connec_state = ConnectionState::Closing; } else { @@ -2039,9 +2037,9 @@ impl NetworkBehaviour for Notifications { } }, - Some(PeerState::Incoming { connections, .. }) - | Some(PeerState::DisabledPendingEnable { connections, .. }) - | Some(PeerState::Disabled { connections, .. }) => { + Some(PeerState::Incoming { connections, .. }) | + Some(PeerState::DisabledPendingEnable { connections, .. }) | + Some(PeerState::Disabled { connections, .. }) => { if let Some((_, connec_state)) = connections.iter_mut().find(|(c, s)| { *c == connection_id && matches!(s, ConnectionState::OpeningThenClosing) }) { @@ -2074,7 +2072,7 @@ impl NetworkBehaviour for Notifications { } else { error!(target: "sub-libp2p", "OpenResultErr: State mismatch in the custom protos handler"); debug_assert!(false); - return; + return }; match mem::replace(entry.get_mut(), PeerState::Poisoned) { @@ -2090,8 +2088,8 @@ impl NetworkBehaviour for Notifications { *connec_state = ConnectionState::Closed; } else if let Some((_, connec_state)) = connections.iter_mut().find(|(c, s)| { - *c == connection_id - && matches!(s, ConnectionState::OpeningThenClosing) + *c == connection_id && + matches!(s, ConnectionState::OpeningThenClosing) }) { *connec_state = ConnectionState::Closing; } else { @@ -2115,17 +2113,17 @@ impl NetworkBehaviour for Notifications { *entry.into_mut() = PeerState::Enabled { connections }; } }, - mut state @ PeerState::Incoming { .. } - | mut state @ PeerState::DisabledPendingEnable { .. } - | mut state @ PeerState::Disabled { .. } => { + mut state @ PeerState::Incoming { .. } | + mut state @ PeerState::DisabledPendingEnable { .. } | + mut state @ PeerState::Disabled { .. } => { match &mut state { - PeerState::Incoming { connections, .. } - | PeerState::Disabled { connections, .. } - | PeerState::DisabledPendingEnable { connections, .. } => { + PeerState::Incoming { connections, .. } | + PeerState::Disabled { connections, .. } | + PeerState::DisabledPendingEnable { connections, .. } => { if let Some((_, connec_state)) = connections.iter_mut().find(|(c, s)| { - *c == connection_id - && matches!(s, ConnectionState::OpeningThenClosing) + *c == connection_id && + matches!(s, ConnectionState::OpeningThenClosing) }) { *connec_state = ConnectionState::Closing; } else { @@ -2173,7 +2171,7 @@ impl NetworkBehaviour for Notifications { set_id, message: message.clone(), }; - self.events.push_back(NetworkBehaviourAction::GenerateEvent(event)); + self.events.push_back(ToSwarm::GenerateEvent(event)); // TODO(aaro): error handling let _ = self.protocol_handles[protocol_index] .report_notification_received(peer_id, message.to_vec()); @@ -2197,7 +2195,7 @@ impl NetworkBehaviour for Notifications { _params: &mut impl PollParameters, ) -> Poll>> { if let Some(event) = self.events.pop_front() { - return Poll::Ready(event); + return Poll::Ready(event) } // Poll for instructions from the peerset. @@ -2218,7 +2216,7 @@ impl NetworkBehaviour for Notifications { }, Poll::Ready(None) => { error!(target: "sub-libp2p", "Peerset receiver stream has returned None"); - break; + break }, Poll::Pending => break, } @@ -2231,14 +2229,14 @@ impl NetworkBehaviour for Notifications { NotificationCommand::SetHandshake(handshake) => { self.set_notif_protocol_handshake(set_id.into(), handshake); }, - NotificationCommand::OpenSubstream(_peer) - | NotificationCommand::CloseSubstream(_peer) => { + NotificationCommand::OpenSubstream(_peer) | + NotificationCommand::CloseSubstream(_peer) => { todo!("substream control not implemented"); }, }, Poll::Ready(None) => { error!(target: "sub-libp2p", "Protocol command streams have been shut down"); - break; + break }, Poll::Pending => break, } @@ -2256,7 +2254,7 @@ impl NetworkBehaviour for Notifications { }, Err(_) => { error!(target: "sub-libp2p", "Protocol ha shut down"); - break; + break }, } } @@ -2320,7 +2318,7 @@ impl NetworkBehaviour for Notifications { } if let Some(event) = self.events.pop_front() { - return Poll::Ready(event); + return Poll::Ready(event) } Poll::Pending @@ -2342,9 +2340,8 @@ mod tests { (ConnectionState::Closing, ConnectionState::Closing) => true, (ConnectionState::Opening, ConnectionState::Opening) => true, (ConnectionState::OpeningThenClosing, ConnectionState::OpeningThenClosing) => true, - (ConnectionState::OpenDesiredByRemote, ConnectionState::OpenDesiredByRemote) => { - true - }, + (ConnectionState::OpenDesiredByRemote, ConnectionState::OpenDesiredByRemote) => + true, (ConnectionState::Open(_), ConnectionState::Open(_)) => true, _ => false, } @@ -3958,7 +3955,7 @@ mod tests { .await; if notif.peers.get(&(peer, set_id)).is_none() { - break; + break } } }) @@ -4074,7 +4071,7 @@ mod tests { assert!(std::matches!(connections[0], (_, ConnectionState::Closing))); if timer_deadline != &prev_instant { - break; + break } } else { panic!("invalid state"); @@ -4811,7 +4808,7 @@ mod tests { #[cfg(debug_assertions)] fn open_result_ok_non_existent_peer() { let (mut notif, _peerset, _notif_service) = development_notifs(); - let conn = ConnectionId::new(); + let conn = ConnectionId::new_unchecked(0usize); let connected = ConnectedPoint::Listener { local_addr: Multiaddr::empty(), send_back_addr: Multiaddr::empty(), diff --git a/client/network/src/protocol/notifications/service.rs b/client/network/src/protocol/notifications/service.rs deleted file mode 100644 index 00c506fcb4b4a..0000000000000 --- a/client/network/src/protocol/notifications/service.rs +++ /dev/null @@ -1,1037 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Notification service implementation. - -use crate::{ - error, - protocol::notifications::handler::NotificationsSink, - service::traits::{NotificationEvent, NotificationService, ValidationResult}, - types::ProtocolName, -}; - -use futures::{ - stream::{FuturesUnordered, Stream}, - StreamExt, -}; -use libp2p::PeerId; -use tokio::sync::{mpsc, oneshot}; -use tokio_stream::wrappers::ReceiverStream; - -use sc_network_common::role::ObservedRole; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; - -use std::{ - collections::HashMap, - fmt::Debug, - sync::{Arc, Mutex}, -}; - -/// Logging target for the file. -const LOG_TARGET: &str = "notification-service"; - -/// Type representing subscribers of a notification protocol. -type Subscribers = Arc>>>; - -/// Inner notification event to deal with `NotificationsSinks` without exposing that -/// implementation detail to [`NotificationService`] consumers. -#[derive(Debug)] -enum InnerNotificationEvent { - /// Validate inbound substream. - ValidateInboundSubstream { - /// Peer ID. - peer: PeerId, - - /// Received handshake. - handshake: Vec, - - /// `oneshot::Sender` for sending validation result back to `Notifications` - result_tx: oneshot::Sender, - }, - - /// Remote identified by `PeerId` opened a substream and sent `Handshake`. - /// Validate `Handshake` and report status (accept/reject) to `Notifications`. - NotificationStreamOpened { - /// Peer ID. - peer: PeerId, - - /// Received handshake. - handshake: Vec, - - /// Role of the peer. - role: ObservedRole, - - /// Negotiated fallback. - negotiated_fallback: Option, - - /// Notification sink. - sink: NotificationsSink, - }, - - /// Substream was closed. - NotificationStreamClosed { - /// Peer Id. - peer: PeerId, - }, - - /// Notification was received from the substream. - NotificationReceived { - /// Peer ID. - peer: PeerId, - - /// Received notification. - notification: Vec, - }, -} - -/// Notification commands. -/// -/// Sent by the installed protocols to `Notifications` to open/close/modify substreams. -pub enum NotificationCommand { - /// Instruct `Notifications` to open a substream to peer. - OpenSubstream(PeerId), - - /// Instruct `Notifications` to close the substream to peer. - CloseSubstream(PeerId), - - /// Set handshake for the notifications protocol. - SetHandshake(Vec), -} - -/// Handle that is passed on to the notifications protocol. -#[derive(Debug)] -pub struct NotificationHandle { - /// Protocol name. - protocol: ProtocolName, - - /// TX channel for sending commands to `Notifications`. - tx: mpsc::Sender, - - /// RX channel for receiving events from `Notifications`. - rx: TracingUnboundedReceiver, - - /// All subscribers of `NotificationEvent`s. - subscribers: Subscribers, - - /// Connected peers. - peers: HashMap, -} - -impl NotificationHandle { - /// Create new [`NotificationHandle`]. - fn new( - protocol: ProtocolName, - tx: mpsc::Sender, - rx: TracingUnboundedReceiver, - subscribers: Arc>>>, - ) -> Self { - Self { protocol, tx, rx, subscribers, peers: HashMap::new() } - } -} - -#[async_trait::async_trait] -impl NotificationService for NotificationHandle { - /// Instruct `Notifications` to open a new substream for `peer`. - async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { - todo!("support for opening substreams not implemented yet"); - } - - /// Instruct `Notifications` to close substream for `peer`. - async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { - todo!("support for closing substreams not implemented yet"); - } - - /// Send synchronous `notification` to `peer`. - fn send_sync_notification( - &self, - peer: &PeerId, - notification: Vec, - ) -> Result<(), error::Error> { - log::trace!(target: LOG_TARGET, "{}: send sync notification to {peer:?}", self.protocol); - - self.peers - .get(&peer) - // TODO: check what the current implementation does in case the peer doesn't exist - .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? - .send_sync_notification(notification); - Ok(()) - } - - /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. - async fn send_async_notification( - &self, - peer: &PeerId, - notification: Vec, - ) -> Result<(), error::Error> { - log::trace!(target: LOG_TARGET, "{}: send async notification to {peer:?}", self.protocol); - - self.peers - .get(&peer) - // TODO: check what the current implementation does in case the peer doesn't exist - .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? - .reserve_notification() - .await - .map_err(|_| error::Error::ConnectionClosed)? - .send(notification) - .map_err(|_| error::Error::ChannelClosed) - } - - /// Set handshake for the notification protocol replacing the old handshake. - async fn set_hanshake(&mut self, handshake: Vec) -> Result<(), ()> { - log::trace!(target: LOG_TARGET, "{}: set handshake to {handshake:?}", self.protocol); - - self.tx.send(NotificationCommand::SetHandshake(handshake)).await.map_err(|_| ()) - } - - /// Get next event from the `Notifications` event stream. - async fn next_event(&mut self) -> Option { - match self.rx.next().await? { - InnerNotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => - Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }), - InnerNotificationEvent::NotificationStreamOpened { - peer, - handshake, - role, - negotiated_fallback, - sink, - } => { - self.peers.insert(peer, sink); - Some(NotificationEvent::NotificationStreamOpened { - peer, - handshake, - role, - negotiated_fallback, - }) - }, - InnerNotificationEvent::NotificationStreamClosed { peer } => { - self.peers.remove(&peer); - Some(NotificationEvent::NotificationStreamClosed { peer }) - }, - InnerNotificationEvent::NotificationReceived { peer, notification } => - Some(NotificationEvent::NotificationReceived { peer, notification }), - } - } - - // Clone [`NotificationService`] - fn clone(&mut self) -> Result, ()> { - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; - let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); - subscribers.push(event_tx); - - Ok(Box::new(NotificationHandle { - protocol: self.protocol.clone(), - tx: self.tx.clone(), - rx: event_rx, - peers: self.peers.clone(), - subscribers: self.subscribers.clone(), - })) - } - - /// Get protocol name. - fn protocol(&self) -> &ProtocolName { - &self.protocol - } -} - -/// Channel pair which allows `Notifications` to interact with a protocol. -#[derive(Debug)] -pub struct ProtocolHandlePair { - /// Protocol name. - protocol: ProtocolName, - - /// Subscribers of the notification protocol events. - subscribers: Subscribers, - - // Receiver for notification commands received from the protocol implementation. - rx: mpsc::Receiver, -} - -impl ProtocolHandlePair { - /// Create new [`ProtocolHandlePair`]. - fn new( - protocol: ProtocolName, - subscribers: Subscribers, - rx: mpsc::Receiver, - ) -> Self { - Self { protocol, subscribers, rx } - } - - /// Consume `self` and split [`ProtocolHandlePair`] into a handle which allows it to send events - /// to the protocol into a stream of commands received from the protocol. - pub fn split( - self, - ) -> (ProtocolHandle, Box + Send + Unpin>) { - ( - ProtocolHandle::new(self.protocol, self.subscribers), - Box::new(ReceiverStream::new(self.rx)), - ) - } -} - -/// Handle that is passed on to `Notifications` and allows it to directly communicate -/// with the protocol. -#[derive(Debug)] -pub struct ProtocolHandle { - /// Protocol name. - protocol: ProtocolName, - - /// Subscribers of the notification protocol. - subscribers: Subscribers, - - /// Number of connected peers. - num_peers: usize, -} - -impl ProtocolHandle { - fn new(protocol: ProtocolName, subscribers: Subscribers) -> Self { - Self { protocol, subscribers, num_peers: 0usize } - } - - /// Report to the protocol that a substream has been opened and it must be validated by the - /// protocol. - /// - /// Return `oneshot::Receiver` which allows `Notifications` to poll for the validation result - /// from protocol. - pub fn report_incoming_substream( - &self, - peer: PeerId, - handshake: Vec, - ) -> Result, ()> { - let subscribers = self.subscribers.lock().map_err(|_| ())?; - - log::trace!( - target: LOG_TARGET, - "{}: report incoming substream for {peer}, handshake {handshake:?}", - self.protocol - ); - - // if there is only one subscriber, `Notifications` can wait directly on the - // `oneshot::channel()`'s RX half without indirection - if subscribers.len() == 1 { - let (result_tx, rx) = oneshot::channel(); - return subscribers[0] - .unbounded_send(InnerNotificationEvent::ValidateInboundSubstream { - peer, - handshake, - result_tx, - }) - .map(|_| rx) - .map_err(|_| ()) - } - - // if there are multiple subscribers, create a task which waits for all of the - // validations to finish and returns the combined result to `Notifications` - let mut results: FuturesUnordered<_> = subscribers - .iter() - .filter_map(|subscriber| { - let (result_tx, rx) = oneshot::channel(); - - subscriber - .unbounded_send(InnerNotificationEvent::ValidateInboundSubstream { - peer, - handshake: handshake.clone(), - result_tx, - }) - .is_ok() - .then_some(rx) - }) - .collect(); - - let (tx, rx) = oneshot::channel(); - tokio::spawn(async move { - while let Some(event) = results.next().await { - match event { - Err(_) | Ok(ValidationResult::Reject) => - return tx.send(ValidationResult::Reject), - Ok(ValidationResult::Accept) => {}, - } - } - - return tx.send(ValidationResult::Accept) - }); - - Ok(rx) - } - - /// Report to the protocol that a substream has been opened and that it can now use the handle - /// to send notifications to the remote peer. - pub fn report_substream_opened( - &mut self, - peer: PeerId, - handshake: Vec, - role: ObservedRole, - negotiated_fallback: Option, - sink: NotificationsSink, - ) -> Result<(), ()> { - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; - - log::trace!(target: LOG_TARGET, "{}: substream opened for {peer:?}", self.protocol); - - subscribers.retain(|subscriber| { - subscriber - .unbounded_send(InnerNotificationEvent::NotificationStreamOpened { - peer, - handshake: handshake.clone(), - role: role.clone(), - negotiated_fallback: negotiated_fallback.clone(), - sink: sink.clone(), - }) - .is_ok() - }); - self.num_peers += 1; - - Ok(()) - } - - /// Substream was closed. - pub fn report_substream_closed(&mut self, peer: PeerId) -> Result<(), ()> { - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; - - log::trace!(target: LOG_TARGET, "{}: substream closed for {peer:?}", self.protocol); - - subscribers.retain(|subscriber| { - subscriber - .unbounded_send(InnerNotificationEvent::NotificationStreamClosed { peer }) - .is_ok() - }); - self.num_peers -= 1; - - Ok(()) - } - - /// Notification was received from the substream. - pub fn report_notification_received( - &mut self, - peer: PeerId, - notification: Vec, - ) -> Result<(), ()> { - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; - - log::trace!(target: LOG_TARGET, "{}: notification received from {peer:?}", self.protocol); - - subscribers.retain(|subscriber| { - subscriber - .unbounded_send(InnerNotificationEvent::NotificationReceived { - peer, - notification: notification.clone(), - }) - .is_ok() - }); - - Ok(()) - } - - /// Get the number of connected peers. - pub fn num_peers(&self) -> usize { - self.num_peers - } -} - -/// Create new (protocol, notification) handle pair. -/// -/// Handle pair allows `Notifications` and the protocol to communicate with each other directly. -pub fn notification_service( - protocol: ProtocolName, -) -> (ProtocolHandlePair, Box) { - let (cmd_tx, cmd_rx) = mpsc::channel(64); // TODO: zzz - let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); - let subscribers = Arc::new(Mutex::new(vec![event_tx])); - - ( - ProtocolHandlePair::new(protocol.clone(), subscribers.clone(), cmd_rx), - Box::new(NotificationHandle::new(protocol.clone(), cmd_tx, event_rx, subscribers)), - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::protocol::notifications::handler::{ - NotificationsSinkMessage, ASYNC_NOTIFICATIONS_BUFFER_SIZE, - }; - - #[tokio::test] - async fn validate_and_accept_substream() { - let (proto, mut notif) = notification_service("/proto/1".into()); - let (handle, _stream) = proto.split(); - - let peer_id = PeerId::random(); - let rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); - - if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = - notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(handshake, vec![1, 3, 3, 7]); - let _ = result_tx.send(ValidationResult::Accept).unwrap(); - } else { - panic!("invalid event received"); - } - - assert_eq!(rx.await.unwrap(), ValidationResult::Accept); - } - - #[tokio::test] - async fn substream_opened() { - let (proto, mut notif) = notification_service("/proto/1".into()); - let (sink, _, _) = NotificationsSink::new(PeerId::random()); - let (mut handle, _stream) = proto.split(); - - let peer_id = PeerId::random(); - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); - - if let Some(NotificationEvent::NotificationStreamOpened { - peer, - role, - negotiated_fallback, - handshake, - }) = notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); - assert_eq!(handshake, vec![1, 3, 3, 7]); - } else { - panic!("invalid event received"); - } - } - - #[tokio::test] - async fn send_sync_notification() { - let (proto, mut notif) = notification_service("/proto/1".into()); - let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); - let (mut handle, _stream) = proto.split(); - let peer_id = PeerId::random(); - - // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); - - if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = - notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(handshake, vec![1, 3, 3, 7]); - let _ = result_tx.send(ValidationResult::Accept).unwrap(); - } else { - panic!("invalid event received"); - } - assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); - - // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); - - if let Some(NotificationEvent::NotificationStreamOpened { - peer, - role, - negotiated_fallback, - handshake, - }) = notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); - assert_eq!(handshake, vec![1, 3, 3, 7]); - } else { - panic!("invalid event received"); - } - - notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]).unwrap(); - assert_eq!( - sync_rx.next().await, - Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }) - ); - } - - #[tokio::test] - async fn send_async_notification() { - let (proto, mut notif) = notification_service("/proto/1".into()); - let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); - let (mut handle, _stream) = proto.split(); - let peer_id = PeerId::random(); - - // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); - - if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = - notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(handshake, vec![1, 3, 3, 7]); - let _ = result_tx.send(ValidationResult::Accept).unwrap(); - } else { - panic!("invalid event received"); - } - assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); - - // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); - - if let Some(NotificationEvent::NotificationStreamOpened { - peer, - role, - negotiated_fallback, - handshake, - }) = notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); - assert_eq!(handshake, vec![1, 3, 3, 7]); - } else { - panic!("invalid event received"); - } - - notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); - assert_eq!( - async_rx.next().await, - Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }) - ); - } - - #[tokio::test] - async fn send_sync_notification_to_non_existent_peer() { - let (proto, notif) = notification_service("/proto/1".into()); - let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); - let (_handle, _stream) = proto.split(); - let peer = PeerId::random(); - - if let Err(error::Error::PeerDoesntExist(peer_id)) = - notif.send_sync_notification(&peer, vec![1, 3, 3, 7]) - { - assert_eq!(peer, peer_id); - } else { - panic!("invalid error received from `send_sync_notification()`"); - } - } - - #[tokio::test] - async fn send_async_notification_to_non_existent_peer() { - let (proto, notif) = notification_service("/proto/1".into()); - let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); - let (_handle, _stream) = proto.split(); - let peer = PeerId::random(); - - if let Err(error::Error::PeerDoesntExist(peer_id)) = - notif.send_async_notification(&peer, vec![1, 3, 3, 7]).await - { - assert_eq!(peer, peer_id); - } else { - panic!("invalid error received from `send_sync_notification()`"); - } - } - - #[tokio::test] - async fn receive_notification() { - let (proto, mut notif) = notification_service("/proto/1".into()); - let (sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); - let (mut handle, _stream) = proto.split(); - let peer_id = PeerId::random(); - - // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); - - if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = - notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(handshake, vec![1, 3, 3, 7]); - let _ = result_tx.send(ValidationResult::Accept).unwrap(); - } else { - panic!("invalid event received"); - } - assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); - - // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); - - if let Some(NotificationEvent::NotificationStreamOpened { - peer, - role, - negotiated_fallback, - handshake, - }) = notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); - assert_eq!(handshake, vec![1, 3, 3, 7]); - } else { - panic!("invalid event received"); - } - - // notification is received - handle.report_notification_received(peer_id, vec![1, 3, 3, 8]).unwrap(); - - if let Some(NotificationEvent::NotificationReceived { peer, notification }) = - notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(notification, vec![1, 3, 3, 8]); - } else { - panic!("invalid event received"); - } - } - - #[tokio::test] - async fn backpressure_works() { - let (proto, mut notif) = notification_service("/proto/1".into()); - let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); - let (mut handle, _stream) = proto.split(); - let peer_id = PeerId::random(); - - // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); - - if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = - notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(handshake, vec![1, 3, 3, 7]); - let _ = result_tx.send(ValidationResult::Accept).unwrap(); - } else { - panic!("invalid event received"); - } - assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); - - // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); - - if let Some(NotificationEvent::NotificationStreamOpened { - peer, - role, - negotiated_fallback, - handshake, - }) = notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); - assert_eq!(handshake, vec![1, 3, 3, 7]); - } else { - panic!("invalid event received"); - } - - // fill the message buffer with messages - for i in 0..=ASYNC_NOTIFICATIONS_BUFFER_SIZE { - assert!( - futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, i as u8])) - .is_ready() - ); - } - - // try to send one more message and verify that the call blocks - assert!( - futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, 9])).is_pending() - ); - - // release one slot from the buffer for new message - assert_eq!( - async_rx.next().await, - Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 0] }) - ); - - // verify that a message can be sent - assert!( - futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, 9])).is_ready() - ); - } - - #[tokio::test] - async fn peer_disconnects_then_sync_notification_is_sent() { - let (proto, mut notif) = notification_service("/proto/1".into()); - let (sink, _, sync_rx) = NotificationsSink::new(PeerId::random()); - let (mut handle, _stream) = proto.split(); - let peer_id = PeerId::random(); - - // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); - - if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = - notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(handshake, vec![1, 3, 3, 7]); - let _ = result_tx.send(ValidationResult::Accept).unwrap(); - } else { - panic!("invalid event received"); - } - assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); - - // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); - - if let Some(NotificationEvent::NotificationStreamOpened { - peer, - role, - negotiated_fallback, - handshake, - }) = notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); - assert_eq!(handshake, vec![1, 3, 3, 7]); - } else { - panic!("invalid event received"); - } - - // report that a substream has been closed but don't poll `notif` to receive this - // information - handle.report_substream_closed(peer_id).unwrap(); - drop(sync_rx); - - // as per documentation, error is not reported but the notification is silently dropped - assert!(notif.send_sync_notification(&peer_id, vec![1, 3, 3, 7]).is_ok()); - } - - #[tokio::test] - async fn peer_disconnects_then_async_notification_is_sent() { - let (proto, mut notif) = notification_service("/proto/1".into()); - let (sink, async_rx, _) = NotificationsSink::new(PeerId::random()); - let (mut handle, _stream) = proto.split(); - let peer_id = PeerId::random(); - - // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); - - if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = - notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(handshake, vec![1, 3, 3, 7]); - let _ = result_tx.send(ValidationResult::Accept).unwrap(); - } else { - panic!("invalid event received"); - } - assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); - - // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); - - if let Some(NotificationEvent::NotificationStreamOpened { - peer, - role, - negotiated_fallback, - handshake, - }) = notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); - assert_eq!(handshake, vec![1, 3, 3, 7]); - } else { - panic!("invalid event received"); - } - - // report that a substream has been closed but don't poll `notif` to receive this - // information - handle.report_substream_closed(peer_id).unwrap(); - drop(async_rx); - - // as per documentation, error is not reported but the notification is silently dropped - if let Err(error::Error::ConnectionClosed) = - notif.send_async_notification(&peer_id, vec![1, 3, 3, 7]).await - { - } else { - panic!("invalid state after calling `send_async_notificatio()` on closed connection") - } - } - - #[tokio::test] - async fn cloned_service_opening_substream_works() { - let (proto, mut notif1) = notification_service("/proto/1".into()); - let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); - let (handle, _stream) = proto.split(); - let mut notif2 = notif1.clone().unwrap(); - let peer_id = PeerId::random(); - - // validate inbound substream - let mut result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); - - // verify that `stream1` also gets the event - if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = - notif1.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(handshake, vec![1, 3, 3, 7]); - let _ = result_tx.send(ValidationResult::Accept).unwrap(); - } else { - panic!("invalid event received"); - } - - // verify that because only one listener has thus far send their result, the result is - // pending - assert!(result_rx.try_recv().is_err()); - - // verify that `stream2` also gets the event - if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = - notif2.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(handshake, vec![1, 3, 3, 7]); - result_tx.send(ValidationResult::Accept).unwrap(); - } else { - panic!("invalid event received"); - } - - assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); - } - - #[tokio::test] - async fn cloned_service_one_service_rejects_substream() { - let (proto, mut notif1) = notification_service("/proto/1".into()); - let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); - let (handle, _stream) = proto.split(); - let mut notif2 = notif1.clone().unwrap(); - let mut notif3 = notif2.clone().unwrap(); - let peer_id = PeerId::random(); - - // validate inbound substream - let mut result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); - - for notif in vec![&mut notif1, &mut notif2] { - if let Some(NotificationEvent::ValidateInboundSubstream { - peer, - handshake, - result_tx, - }) = notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(handshake, vec![1, 3, 3, 7]); - let _ = result_tx.send(ValidationResult::Accept).unwrap(); - } else { - panic!("invalid event received"); - } - } - - // `notif3` has not yet sent their validation result - assert!(result_rx.try_recv().is_err()); - - if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = - notif3.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(handshake, vec![1, 3, 3, 7]); - let _ = result_tx.send(ValidationResult::Reject).unwrap(); - } else { - panic!("invalid event received"); - } - assert_eq!(result_rx.await.unwrap(), ValidationResult::Reject); - } - - #[tokio::test] - async fn cloned_service_opening_substream_sending_and_receiving_notifications_work() { - let (proto, mut notif1) = notification_service("/proto/1".into()); - let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); - let (mut handle, _stream) = proto.split(); - let mut notif2 = notif1.clone().unwrap(); - let mut notif3 = notif1.clone().unwrap(); - let peer_id = PeerId::random(); - - // validate inbound substream - let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); - - for notif in vec![&mut notif1, &mut notif2, &mut notif3] { - // accept the inbound substream for all services - if let Some(NotificationEvent::ValidateInboundSubstream { - peer, - handshake, - result_tx, - }) = notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(handshake, vec![1, 3, 3, 7]); - let _ = result_tx.send(ValidationResult::Accept).unwrap(); - } else { - panic!("invalid event received"); - } - } - assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); - - // report that then notification stream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); - - for notif in vec![&mut notif1, &mut notif2, &mut notif3] { - if let Some(NotificationEvent::NotificationStreamOpened { - peer, - role, - negotiated_fallback, - handshake, - }) = notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); - assert_eq!(handshake, vec![1, 3, 3, 7]); - } else { - panic!("invalid event received"); - } - } - // receive a notification from peer and verify all services receive it - handle.report_notification_received(peer_id, vec![1, 3, 3, 8]).unwrap(); - - for notif in vec![&mut notif1, &mut notif2, &mut notif3] { - if let Some(NotificationEvent::NotificationReceived { peer, notification }) = - notif.next_event().await - { - assert_eq!(peer_id, peer); - assert_eq!(notification, vec![1, 3, 3, 8]); - } else { - panic!("invalid event received"); - } - } - - for (i, notif) in vec![&mut notif1, &mut notif2, &mut notif3].iter().enumerate() { - // send notification from each service and verify peer receives it - notif.send_sync_notification(&peer_id, vec![1, 3, 3, i as u8]).unwrap(); - assert_eq!( - sync_rx.next().await, - Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, i as u8] }) - ); - } - - // close the substream for peer and verify all services receive the event - handle.report_substream_closed(peer_id).unwrap(); - - for notif in vec![&mut notif1, &mut notif2, &mut notif3] { - if let Some(NotificationEvent::NotificationStreamClosed { peer }) = - notif.next_event().await - { - assert_eq!(peer_id, peer); - } else { - panic!("invalid event received"); - } - } - } -} diff --git a/client/network/src/protocol/notifications/service/metrics.rs b/client/network/src/protocol/notifications/service/metrics.rs new file mode 100644 index 0000000000000..b46a8f75295fe --- /dev/null +++ b/client/network/src/protocol/notifications/service/metrics.rs @@ -0,0 +1,17 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs new file mode 100644 index 0000000000000..8fd31c2b7f49e --- /dev/null +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -0,0 +1,463 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Notification service implementation. + +use crate::{ + error, + protocol::notifications::handler::NotificationsSink, + service::traits::{NotificationEvent, NotificationService, ValidationResult}, + types::ProtocolName, +}; + +use futures::{ + stream::{FuturesUnordered, Stream}, + StreamExt, +}; +use libp2p::PeerId; +use tokio::sync::{mpsc, oneshot}; +use tokio_stream::wrappers::ReceiverStream; + +use sc_network_common::role::ObservedRole; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; + +use std::{ + collections::HashMap, + fmt::Debug, + sync::{Arc, Mutex}, +}; + +mod metrics; +#[cfg(test)] +mod tests; + +/// Logging target for the file. +const LOG_TARGET: &str = "notification-service"; + +/// Type representing subscribers of a notification protocol. +type Subscribers = Arc>>>; + +/// Inner notification event to deal with `NotificationsSinks` without exposing that +/// implementation detail to [`NotificationService`] consumers. +#[derive(Debug)] +enum InnerNotificationEvent { + /// Validate inbound substream. + ValidateInboundSubstream { + /// Peer ID. + peer: PeerId, + + /// Received handshake. + handshake: Vec, + + /// `oneshot::Sender` for sending validation result back to `Notifications` + result_tx: oneshot::Sender, + }, + + /// Remote identified by `PeerId` opened a substream and sent `Handshake`. + /// Validate `Handshake` and report status (accept/reject) to `Notifications`. + NotificationStreamOpened { + /// Peer ID. + peer: PeerId, + + /// Received handshake. + handshake: Vec, + + /// Role of the peer. + role: ObservedRole, + + /// Negotiated fallback. + negotiated_fallback: Option, + + /// Notification sink. + sink: NotificationsSink, + }, + + /// Substream was closed. + NotificationStreamClosed { + /// Peer Id. + peer: PeerId, + }, + + /// Notification was received from the substream. + NotificationReceived { + /// Peer ID. + peer: PeerId, + + /// Received notification. + notification: Vec, + }, +} + +/// Notification commands. +/// +/// Sent by the installed protocols to `Notifications` to open/close/modify substreams. +pub enum NotificationCommand { + /// Instruct `Notifications` to open a substream to peer. + OpenSubstream(PeerId), + + /// Instruct `Notifications` to close the substream to peer. + CloseSubstream(PeerId), + + /// Set handshake for the notifications protocol. + SetHandshake(Vec), +} + +/// Handle that is passed on to the notifications protocol. +#[derive(Debug)] +pub struct NotificationHandle { + /// Protocol name. + protocol: ProtocolName, + + /// TX channel for sending commands to `Notifications`. + tx: mpsc::Sender, + + /// RX channel for receiving events from `Notifications`. + rx: TracingUnboundedReceiver, + + /// All subscribers of `NotificationEvent`s. + subscribers: Subscribers, + + /// Connected peers. + peers: HashMap, +} + +impl NotificationHandle { + /// Create new [`NotificationHandle`]. + fn new( + protocol: ProtocolName, + tx: mpsc::Sender, + rx: TracingUnboundedReceiver, + subscribers: Arc>>>, + ) -> Self { + Self { protocol, tx, rx, subscribers, peers: HashMap::new() } + } +} + +#[async_trait::async_trait] +impl NotificationService for NotificationHandle { + /// Instruct `Notifications` to open a new substream for `peer`. + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + todo!("support for opening substreams not implemented yet"); + } + + /// Instruct `Notifications` to close substream for `peer`. + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + todo!("support for closing substreams not implemented yet"); + } + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification( + &self, + peer: &PeerId, + notification: Vec, + ) -> Result<(), error::Error> { + log::trace!(target: LOG_TARGET, "{}: send sync notification to {peer:?}", self.protocol); + + self.peers + .get(&peer) + // TODO: check what the current implementation does in case the peer doesn't exist + .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? + .send_sync_notification(notification); + Ok(()) + } + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + async fn send_async_notification( + &self, + peer: &PeerId, + notification: Vec, + ) -> Result<(), error::Error> { + log::trace!(target: LOG_TARGET, "{}: send async notification to {peer:?}", self.protocol); + + self.peers + .get(&peer) + // TODO: check what the current implementation does in case the peer doesn't exist + .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? + .reserve_notification() + .await + .map_err(|_| error::Error::ConnectionClosed)? + .send(notification) + .map_err(|_| error::Error::ChannelClosed) + } + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_hanshake(&mut self, handshake: Vec) -> Result<(), ()> { + log::trace!(target: LOG_TARGET, "{}: set handshake to {handshake:?}", self.protocol); + + self.tx.send(NotificationCommand::SetHandshake(handshake)).await.map_err(|_| ()) + } + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option { + match self.rx.next().await? { + InnerNotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => + Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }), + InnerNotificationEvent::NotificationStreamOpened { + peer, + handshake, + role, + negotiated_fallback, + sink, + } => { + self.peers.insert(peer, sink); + Some(NotificationEvent::NotificationStreamOpened { + peer, + handshake, + role, + negotiated_fallback, + }) + }, + InnerNotificationEvent::NotificationStreamClosed { peer } => { + self.peers.remove(&peer); + Some(NotificationEvent::NotificationStreamClosed { peer }) + }, + InnerNotificationEvent::NotificationReceived { peer, notification } => + Some(NotificationEvent::NotificationReceived { peer, notification }), + } + } + + // Clone [`NotificationService`] + fn clone(&mut self) -> Result, ()> { + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); + subscribers.push(event_tx); + + Ok(Box::new(NotificationHandle { + protocol: self.protocol.clone(), + tx: self.tx.clone(), + rx: event_rx, + peers: self.peers.clone(), + subscribers: self.subscribers.clone(), + })) + } + + /// Get protocol name. + fn protocol(&self) -> &ProtocolName { + &self.protocol + } +} + +/// Channel pair which allows `Notifications` to interact with a protocol. +#[derive(Debug)] +pub struct ProtocolHandlePair { + /// Protocol name. + protocol: ProtocolName, + + /// Subscribers of the notification protocol events. + subscribers: Subscribers, + + // Receiver for notification commands received from the protocol implementation. + rx: mpsc::Receiver, +} + +impl ProtocolHandlePair { + /// Create new [`ProtocolHandlePair`]. + fn new( + protocol: ProtocolName, + subscribers: Subscribers, + rx: mpsc::Receiver, + ) -> Self { + Self { protocol, subscribers, rx } + } + + /// Consume `self` and split [`ProtocolHandlePair`] into a handle which allows it to send events + /// to the protocol into a stream of commands received from the protocol. + pub fn split( + self, + ) -> (ProtocolHandle, Box + Send + Unpin>) { + ( + ProtocolHandle::new(self.protocol, self.subscribers), + Box::new(ReceiverStream::new(self.rx)), + ) + } +} + +/// Handle that is passed on to `Notifications` and allows it to directly communicate +/// with the protocol. +#[derive(Debug)] +pub struct ProtocolHandle { + /// Protocol name. + protocol: ProtocolName, + + /// Subscribers of the notification protocol. + subscribers: Subscribers, + + /// Number of connected peers. + num_peers: usize, +} + +impl ProtocolHandle { + fn new(protocol: ProtocolName, subscribers: Subscribers) -> Self { + Self { protocol, subscribers, num_peers: 0usize } + } + + /// Report to the protocol that a substream has been opened and it must be validated by the + /// protocol. + /// + /// Return `oneshot::Receiver` which allows `Notifications` to poll for the validation result + /// from protocol. + pub fn report_incoming_substream( + &self, + peer: PeerId, + handshake: Vec, + ) -> Result, ()> { + let subscribers = self.subscribers.lock().map_err(|_| ())?; + + log::trace!( + target: LOG_TARGET, + "{}: report incoming substream for {peer}, handshake {handshake:?}", + self.protocol + ); + + // if there is only one subscriber, `Notifications` can wait directly on the + // `oneshot::channel()`'s RX half without indirection + if subscribers.len() == 1 { + let (result_tx, rx) = oneshot::channel(); + return subscribers[0] + .unbounded_send(InnerNotificationEvent::ValidateInboundSubstream { + peer, + handshake, + result_tx, + }) + .map(|_| rx) + .map_err(|_| ()) + } + + // if there are multiple subscribers, create a task which waits for all of the + // validations to finish and returns the combined result to `Notifications` + let mut results: FuturesUnordered<_> = subscribers + .iter() + .filter_map(|subscriber| { + let (result_tx, rx) = oneshot::channel(); + + subscriber + .unbounded_send(InnerNotificationEvent::ValidateInboundSubstream { + peer, + handshake: handshake.clone(), + result_tx, + }) + .is_ok() + .then_some(rx) + }) + .collect(); + + let (tx, rx) = oneshot::channel(); + tokio::spawn(async move { + while let Some(event) = results.next().await { + match event { + Err(_) | Ok(ValidationResult::Reject) => + return tx.send(ValidationResult::Reject), + Ok(ValidationResult::Accept) => {}, + } + } + + return tx.send(ValidationResult::Accept) + }); + + Ok(rx) + } + + /// Report to the protocol that a substream has been opened and that it can now use the handle + /// to send notifications to the remote peer. + pub fn report_substream_opened( + &mut self, + peer: PeerId, + handshake: Vec, + role: ObservedRole, + negotiated_fallback: Option, + sink: NotificationsSink, + ) -> Result<(), ()> { + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + + log::trace!(target: LOG_TARGET, "{}: substream opened for {peer:?}", self.protocol); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationStreamOpened { + peer, + handshake: handshake.clone(), + role: role.clone(), + negotiated_fallback: negotiated_fallback.clone(), + sink: sink.clone(), + }) + .is_ok() + }); + self.num_peers += 1; + + Ok(()) + } + + /// Substream was closed. + pub fn report_substream_closed(&mut self, peer: PeerId) -> Result<(), ()> { + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + + log::trace!(target: LOG_TARGET, "{}: substream closed for {peer:?}", self.protocol); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationStreamClosed { peer }) + .is_ok() + }); + self.num_peers -= 1; + + Ok(()) + } + + /// Notification was received from the substream. + pub fn report_notification_received( + &mut self, + peer: PeerId, + notification: Vec, + ) -> Result<(), ()> { + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + + log::trace!(target: LOG_TARGET, "{}: notification received from {peer:?}", self.protocol); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationReceived { + peer, + notification: notification.clone(), + }) + .is_ok() + }); + + Ok(()) + } + + /// Get the number of connected peers. + pub fn num_peers(&self) -> usize { + self.num_peers + } +} + +/// Create new (protocol, notification) handle pair. +/// +/// Handle pair allows `Notifications` and the protocol to communicate with each other directly. +pub fn notification_service( + protocol: ProtocolName, +) -> (ProtocolHandlePair, Box) { + let (cmd_tx, cmd_rx) = mpsc::channel(64); // TODO: zzz + let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); + let subscribers = Arc::new(Mutex::new(vec![event_tx])); + + ( + ProtocolHandlePair::new(protocol.clone(), subscribers.clone(), cmd_rx), + Box::new(NotificationHandle::new(protocol.clone(), cmd_tx, event_rx, subscribers)), + ) +} diff --git a/client/network/src/protocol/notifications/service/tests.rs b/client/network/src/protocol/notifications/service/tests.rs new file mode 100644 index 0000000000000..b770c7c8c1096 --- /dev/null +++ b/client/network/src/protocol/notifications/service/tests.rs @@ -0,0 +1,579 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use super::*; +use crate::protocol::notifications::handler::{ + NotificationsSinkMessage, ASYNC_NOTIFICATIONS_BUFFER_SIZE, +}; + +#[tokio::test] +async fn validate_and_accept_substream() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (handle, _stream) = proto.split(); + + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + + assert_eq!(rx.await.unwrap(), ValidationResult::Accept); +} + +#[tokio::test] +async fn substream_opened() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, _, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + + let peer_id = PeerId::random(); + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + handshake, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); + } else { + panic!("invalid event received"); + } +} + +#[tokio::test] +async fn send_sync_notification() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + handshake, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); + } else { + panic!("invalid event received"); + } + + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]).unwrap(); + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }) + ); +} + +#[tokio::test] +async fn send_async_notification() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + handshake, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); + } else { + panic!("invalid event received"); + } + + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }) + ); +} + +#[tokio::test] +async fn send_sync_notification_to_non_existent_peer() { + let (proto, notif) = notification_service("/proto/1".into()); + let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); + let (_handle, _stream) = proto.split(); + let peer = PeerId::random(); + + if let Err(error::Error::PeerDoesntExist(peer_id)) = + notif.send_sync_notification(&peer, vec![1, 3, 3, 7]) + { + assert_eq!(peer, peer_id); + } else { + panic!("invalid error received from `send_sync_notification()`"); + } +} + +#[tokio::test] +async fn send_async_notification_to_non_existent_peer() { + let (proto, notif) = notification_service("/proto/1".into()); + let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); + let (_handle, _stream) = proto.split(); + let peer = PeerId::random(); + + if let Err(error::Error::PeerDoesntExist(peer_id)) = + notif.send_async_notification(&peer, vec![1, 3, 3, 7]).await + { + assert_eq!(peer, peer_id); + } else { + panic!("invalid error received from `send_sync_notification()`"); + } +} + +#[tokio::test] +async fn receive_notification() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + handshake, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); + } else { + panic!("invalid event received"); + } + + // notification is received + handle.report_notification_received(peer_id, vec![1, 3, 3, 8]).unwrap(); + + if let Some(NotificationEvent::NotificationReceived { peer, notification }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(notification, vec![1, 3, 3, 8]); + } else { + panic!("invalid event received"); + } +} + +#[tokio::test] +async fn backpressure_works() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + handshake, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); + } else { + panic!("invalid event received"); + } + + // fill the message buffer with messages + for i in 0..=ASYNC_NOTIFICATIONS_BUFFER_SIZE { + assert!(futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, i as u8])) + .is_ready()); + } + + // try to send one more message and verify that the call blocks + assert!(futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, 9])).is_pending()); + + // release one slot from the buffer for new message + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 0] }) + ); + + // verify that a message can be sent + assert!(futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, 9])).is_ready()); +} + +#[tokio::test] +async fn peer_disconnects_then_sync_notification_is_sent() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, _, sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + handshake, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); + } else { + panic!("invalid event received"); + } + + // report that a substream has been closed but don't poll `notif` to receive this + // information + handle.report_substream_closed(peer_id).unwrap(); + drop(sync_rx); + + // as per documentation, error is not reported but the notification is silently dropped + assert!(notif.send_sync_notification(&peer_id, vec![1, 3, 3, 7]).is_ok()); +} + +#[tokio::test] +async fn peer_disconnects_then_async_notification_is_sent() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + handshake, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); + } else { + panic!("invalid event received"); + } + + // report that a substream has been closed but don't poll `notif` to receive this + // information + handle.report_substream_closed(peer_id).unwrap(); + drop(async_rx); + + // as per documentation, error is not reported but the notification is silently dropped + if let Err(error::Error::ConnectionClosed) = + notif.send_async_notification(&peer_id, vec![1, 3, 3, 7]).await + { + } else { + panic!("invalid state after calling `send_async_notificatio()` on closed connection") + } +} + +#[tokio::test] +async fn cloned_service_opening_substream_works() { + let (proto, mut notif1) = notification_service("/proto/1".into()); + let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); + let (handle, _stream) = proto.split(); + let mut notif2 = notif1.clone().unwrap(); + let peer_id = PeerId::random(); + + // validate inbound substream + let mut result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + // verify that `stream1` also gets the event + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif1.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + + // verify that because only one listener has thus far send their result, the result is + // pending + assert!(result_rx.try_recv().is_err()); + + // verify that `stream2` also gets the event + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif2.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); +} + +#[tokio::test] +async fn cloned_service_one_service_rejects_substream() { + let (proto, mut notif1) = notification_service("/proto/1".into()); + let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); + let (handle, _stream) = proto.split(); + let mut notif2 = notif1.clone().unwrap(); + let mut notif3 = notif2.clone().unwrap(); + let peer_id = PeerId::random(); + + // validate inbound substream + let mut result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + for notif in vec![&mut notif1, &mut notif2] { + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + } + + // `notif3` has not yet sent their validation result + assert!(result_rx.try_recv().is_err()); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif3.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Reject).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Reject); +} + +#[tokio::test] +async fn cloned_service_opening_substream_sending_and_receiving_notifications_work() { + let (proto, mut notif1) = notification_service("/proto/1".into()); + let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let mut notif2 = notif1.clone().unwrap(); + let mut notif3 = notif1.clone().unwrap(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + // accept the inbound substream for all services + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that then notification stream has been opened + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + handshake, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); + } else { + panic!("invalid event received"); + } + } + // receive a notification from peer and verify all services receive it + handle.report_notification_received(peer_id, vec![1, 3, 3, 8]).unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + if let Some(NotificationEvent::NotificationReceived { peer, notification }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(notification, vec![1, 3, 3, 8]); + } else { + panic!("invalid event received"); + } + } + + for (i, notif) in vec![&mut notif1, &mut notif2, &mut notif3].iter().enumerate() { + // send notification from each service and verify peer receives it + notif.send_sync_notification(&peer_id, vec![1, 3, 3, i as u8]).unwrap(); + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, i as u8] }) + ); + } + + // close the substream for peer and verify all services receive the event + handle.report_substream_closed(peer_id).unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + if let Some(NotificationEvent::NotificationStreamClosed { peer }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + } else { + panic!("invalid event received"); + } + } +} From 1fd9a089014a0cba5cbb58a1d0d5cc9e17d6b235 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 23 May 2023 11:29:27 +0300 Subject: [PATCH 17/43] Add getter for peer's `NotificationsSink` --- .../src/protocol/notifications/service/mod.rs | 5 ++ .../protocol/notifications/service/tests.rs | 78 +++++++++++++++++++ client/network/src/service/traits.rs | 4 + 3 files changed, 87 insertions(+) diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index 8fd31c2b7f49e..a82633ab014f5 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -250,6 +250,11 @@ impl NotificationService for NotificationHandle { fn protocol(&self) -> &ProtocolName { &self.protocol } + + /// Get notification sink of the peer. + fn notification_sink(&self, peer: &PeerId) -> Option { + self.peers.get(peer).map(|sink| sink.clone()) + } } /// Channel pair which allows `Notifications` to interact with a protocol. diff --git a/client/network/src/protocol/notifications/service/tests.rs b/client/network/src/protocol/notifications/service/tests.rs index b770c7c8c1096..8867b2098c7a3 100644 --- a/client/network/src/protocol/notifications/service/tests.rs +++ b/client/network/src/protocol/notifications/service/tests.rs @@ -577,3 +577,81 @@ async fn cloned_service_opening_substream_sending_and_receiving_notifications_wo } } } + +#[tokio::test] +async fn sending_notifications_using_notifications_sink_works() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, mut async_rx, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + handshake, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); + } else { + panic!("invalid event received"); + } + + // get a copy of the notification sink and send a synchronous notification using. + let sink = notif.notification_sink(&peer_id).unwrap(); + sink.send_sync_notification(vec![1, 3, 3, 6]); + + // send an asynchronous notification using the acquired notifications sink. + let sender = sink.reserve_notification().await.unwrap(); + sender.send(vec![1, 3, 3, 7]).unwrap(); + + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 6] }), + ); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 7] }), + ); + + // send notifications using the stored notification sink as well. + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]).unwrap(); + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }), + ); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }), + ); +} + +#[test] +fn try_to_get_notifications_sink_for_non_existent_peer() { + let (_proto, notif) = notification_service("/proto/1".into()); + assert!(notif.notification_sink(&PeerId::random()).is_none()); +} diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 2a1b1a7e058ce..ef0f2767f5fad 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -27,6 +27,7 @@ use crate::{ request_responses::{IfDisconnected, RequestFailure}, service::signature::Signature, types::ProtocolName, + NotificationsSink, }; use futures::{channel::oneshot, Stream}; @@ -770,4 +771,7 @@ pub trait NotificationService: Debug + Send { /// Get protocol name of the `NotificationService`. fn protocol(&self) -> &ProtocolName; + + /// Get notification sink of the peer. + fn notification_sink(&self, peer: &PeerId) -> Option; } From b19ccb8dcf134b5fbb1859d4ea2be10aaa960ed0 Mon Sep 17 00:00:00 2001 From: Aaro Altonen <48052676+altonen@users.noreply.github.com> Date: Thu, 25 May 2023 12:38:16 +0300 Subject: [PATCH 18/43] Apply suggestions from code review Co-authored-by: Dmitry Markin --- client/consensus/grandpa/src/communication/tests.rs | 9 +++------ client/network/src/config.rs | 2 +- client/network/src/protocol/notifications/service/mod.rs | 2 +- client/network/sync/src/engine.rs | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/client/consensus/grandpa/src/communication/tests.rs b/client/consensus/grandpa/src/communication/tests.rs index 1bcb61e5c2e7d..1df67941c53c3 100644 --- a/client/consensus/grandpa/src/communication/tests.rs +++ b/client/consensus/grandpa/src/communication/tests.rs @@ -216,9 +216,6 @@ pub(crate) struct TestNotificationService { #[async_trait::async_trait] impl NotificationService for TestNotificationService { /// Instruct `Notifications` to open a new substream for `peer`. - /// - /// `dial_if_disconnected` informs `Notifications` whether to dial - // the peer if there is currently no active connection to it. async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { unimplemented!(); } @@ -456,7 +453,7 @@ fn good_commit_leads_to_relay() { // waiting for events from the network. Send it events that inform that // a notification stream was opened and that a notification was received. // - // since each protocol has its own notification stream, events need not be filtered. + // Since each protocol has its own notification stream, events need not be filtered. let sender_id = id; let send_message = async move { @@ -486,7 +483,7 @@ fn good_commit_leads_to_relay() { }, ); - // Announce its local set has being on the current set id through a neighbor + // Announce its local set being on the current set id through a neighbor // packet, otherwise it won't be eligible to receive the commit let _ = { let update = gossip::VersionedNeighborPacket::V1(gossip::NeighborPacket { @@ -607,7 +604,7 @@ fn bad_commit_leads_to_report() { // waiting for events from the network. Send it events that inform that // a notification stream was opened and that a notification was received. // - // since each protocol has its own notification stream, events need not be filtered. + // Since each protocol has its own notification stream, events need not be filtered. let sender_id = id; let send_message = async move { diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 20dd6799e259f..f0e545f57468d 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -551,7 +551,7 @@ impl NonDefaultSetConfig { &self.protocol_name } - /// Get refer + /// Get reference to fallback protocol names. pub fn fallback_names(&self) -> impl Iterator { self.fallback_names.iter() } diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index a82633ab014f5..3c293f653d2c2 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -281,7 +281,7 @@ impl ProtocolHandlePair { } /// Consume `self` and split [`ProtocolHandlePair`] into a handle which allows it to send events - /// to the protocol into a stream of commands received from the protocol. + /// to the protocol and a stream of commands received from the protocol. pub fn split( self, ) -> (ProtocolHandle, Box + Send + Unpin>) { diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index 0fed553c7ba5f..05399f140f800 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -771,7 +771,7 @@ where if !self.peers.contains_key(&peer) { log::trace!( target: LOG_TARGET, - "seceived notification from {peer} who had been earlier refused by `SyncingEngine`", + "received notification from {peer} who had been earlier refused by `SyncingEngine`", ); continue } From 37db9f632a0f833ec7839b2f19d234041d3bb458 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Fri, 26 May 2023 14:36:02 +0300 Subject: [PATCH 19/43] Get peer count from `SyncingEngine` in `sc-network` tests --- .../grandpa/src/communication/tests.rs | 6 +- client/network-gossip/src/bridge.rs | 6 +- client/network/src/protocol.rs | 2 +- .../src/protocol/notifications/behaviour.rs | 17 +++- client/network/test/src/lib.rs | 41 +++++---- client/network/test/src/sync.rs | 86 +++++++++---------- 6 files changed, 87 insertions(+), 71 deletions(-) diff --git a/client/consensus/grandpa/src/communication/tests.rs b/client/consensus/grandpa/src/communication/tests.rs index 1df67941c53c3..6227fe80ecaf0 100644 --- a/client/consensus/grandpa/src/communication/tests.rs +++ b/client/consensus/grandpa/src/communication/tests.rs @@ -32,7 +32,7 @@ use sc_network::{ types::ProtocolName, Multiaddr, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, NetworkSyncForkRequest, NotificationSenderError, NotificationSenderT as NotificationSender, - PeerId, ReputationChange, + NotificationsSink, PeerId, ReputationChange, }; use sc_network_common::{ role::ObservedRole, @@ -261,6 +261,10 @@ impl NotificationService for TestNotificationService { fn protocol(&self) -> &ProtocolName { unimplemented!(); } + + fn notification_sink(&self, _peer: &PeerId) -> Option { + unimplemented!(); + } } pub(crate) struct Tester { diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index efe1d2da56b56..139e2d05f407f 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -339,7 +339,7 @@ mod tests { use sc_network::{ config::MultiaddrWithPeerId, service::traits::NotificationEvent, Event, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, NotificationSenderError, - NotificationSenderT as NotificationSender, NotificationService, + NotificationSenderT as NotificationSender, NotificationService, NotificationsSink, }; use sc_network_common::{role::ObservedRole, sync::SyncEventStream}; use sp_runtime::{ @@ -546,6 +546,10 @@ mod tests { fn protocol(&self) -> &ProtocolName { unimplemented!(); } + + fn notification_sink(&self, _peer: &PeerId) -> Option { + unimplemented!(); + } } struct AllowAll; diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 8c8a1dd7e44d2..902b6ee8555a4 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -24,7 +24,7 @@ use crate::{ use bytes::Bytes; use codec::{DecodeAll, Encode}; -use futures::{stream::FuturesUnordered, StreamExt}; +use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; use libp2p::{ core::Endpoint, swarm::{ diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index e4cea321f10b3..bbb74d3b6fc7b 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -1050,7 +1050,13 @@ impl Notifications { match mem::replace(state, PeerState::Poisoned) { // Incoming => Enabled - PeerState::Incoming { ref mut accepted, backoff_until, connections } => { + PeerState::Incoming { + ref mut accepted, + backoff_until, + connections, + incoming_index, + .. + } => { trace!( target: LOG_TARGET, "PSM => Accept({:?}, {}, {:?}): Send substream for validation to protocol.", @@ -1075,7 +1081,12 @@ impl Notifications { }, } - *state = PeerState::Incoming { backoff_until, accepted: true, connections }; + *state = PeerState::Incoming { + backoff_until, + accepted: true, + connections, + incoming_index, + }; }, // Any state other than `Incoming` is invalid. @@ -1682,7 +1693,7 @@ impl NetworkBehaviour for Notifications { mut connections, backoff_until, incoming_index, - accepeted, + accepted, } => { debug_assert!(connections .iter() diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 5bc57d53d8bff..c7ac5409b1dd1 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -266,8 +266,8 @@ where } /// Returns the number of peers we're connected to. - pub fn num_peers(&self) -> usize { - self.network.num_connected_peers() + pub async fn num_peers(&self) -> usize { + self.sync_service.status().await.unwrap().num_connected_peers as usize } /// Returns the number of downloaded blocks. @@ -1000,20 +1000,6 @@ where tokio::spawn(f); } - /// Polls the testnet until all peers are connected to each other. - /// - /// Must be executed in a task context. - fn poll_until_connected(&mut self, cx: &mut FutureContext) -> Poll<()> { - self.poll(cx); - - let num_peers = self.peers().len(); - if self.peers().iter().all(|p| p.num_peers() == num_peers - 1) { - return Poll::Ready(()) - } - - Poll::Pending - } - async fn is_in_sync(&mut self) -> bool { let mut highest = None; let peers = self.peers_mut(); @@ -1091,10 +1077,27 @@ where } /// Run the network until all peers are connected to each other. - /// - /// Calls `poll_until_connected` repeatedly with the runtime passed as parameter. async fn run_until_connected(&mut self) { - futures::future::poll_fn::<(), _>(|cx| self.poll_until_connected(cx)).await; + let num_peers = self.peers().len(); + let sync_services = + self.peers().iter().map(|info| info.sync_service.clone()).collect::>(); + + 'outer: loop { + for sync_service in &sync_services { + if sync_service.status().await.unwrap().num_connected_peers as usize != + num_peers - 1 + { + futures::future::poll_fn::<(), _>(|cx| { + self.poll(cx); + Poll::Ready(()) + }) + .await; + continue 'outer + } + } + + break + } } /// Polls the testnet. Processes all the pending actions. diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 81707445dc9d3..4c509b93ab60f 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -44,16 +44,16 @@ async fn sync_peers_works() { sp_tracing::try_init_simple(); let mut net = TestNet::new(3); - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - for peer in 0..3 { - if net.peer(peer).num_peers() != 2 { - return Poll::Pending - } - } - Poll::Ready(()) - }) - .await; + while net.peer(0).num_peers().await != 2 && + net.peer(1).num_peers().await != 2 && + net.peer(2).num_peers().await != 2 + { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); + Poll::Ready(()) + }) + .await; + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -412,15 +412,13 @@ async fn can_sync_small_non_best_forks() { assert!(net.peer(1).client().header(small_hash).unwrap().is_none()); // poll until the two nodes connect, otherwise announcing the block will not work - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { - Poll::Pending - } else { + while net.peer(0).num_peers().await == 0 || net.peer(1).num_peers().await == 0 { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); Poll::Ready(()) - } - }) - .await; + }) + .await; + } // synchronization: 0 synced to longer chain and 1 didn't sync to small chain. @@ -465,6 +463,7 @@ async fn can_sync_forks_ahead_of_the_best_chain() { net.peer(1).push_blocks(1, false); net.run_until_connected().await; + // Peer 0 is on 2-block fork which is announced with is_best=false let fork_hash = net .peer(0) @@ -516,15 +515,13 @@ async fn can_sync_explicit_forks() { assert!(net.peer(1).client().header(small_hash).unwrap().is_none()); // poll until the two nodes connect, otherwise announcing the block will not work - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { - Poll::Pending - } else { + while net.peer(0).num_peers().await == 0 || net.peer(1).num_peers().await == 0 { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); Poll::Ready(()) - } - }) - .await; + }) + .await; + } // synchronization: 0 synced to longer chain and 1 didn't sync to small chain. @@ -610,15 +607,14 @@ async fn full_sync_requires_block_body() { net.peer(0).push_headers(1); // Wait for nodes to connect - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { - Poll::Pending - } else { + while net.peer(0).num_peers().await == 0 || net.peer(1).num_peers().await == 0 { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); Poll::Ready(()) - } - }) - .await; + }) + .await; + } + net.run_until_idle().await; assert_eq!(net.peer(1).client.info().best_number, 0); } @@ -914,18 +910,16 @@ async fn block_announce_data_is_propagated() { }); // Wait until peer 1 is connected to both nodes. - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(1).num_peers() == 2 && - net.peer(0).num_peers() == 1 && - net.peer(2).num_peers() == 1 - { + while net.peer(1).num_peers().await != 2 || + net.peer(0).num_peers().await != 1 || + net.peer(2).num_peers().await != 1 + { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); Poll::Ready(()) - } else { - Poll::Pending - } - }) - .await; + }) + .await; + } let block_hash = net .peer(0) @@ -1007,7 +1001,7 @@ async fn multiple_requests_are_accepted_as_long_as_they_are_not_fulfilled() { tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; net.peer(0).push_blocks(1, false); net.run_until_sync().await; - assert_eq!(1, net.peer(0).num_peers()); + assert_eq!(1, net.peer(0).num_peers().await); } let hashof10 = hashes[9]; From 4e2ad898ee0771a4da816bdb25360ae697d5d6f0 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Fri, 26 May 2023 15:50:55 +0300 Subject: [PATCH 20/43] Apply review comments --- client/network/src/config.rs | 12 ++++-------- client/network/src/protocol.rs | 4 ++-- .../network/src/protocol/notifications/behaviour.rs | 5 +++++ .../src/protocol/notifications/service/mod.rs | 3 +-- client/network/sync/src/engine.rs | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/client/network/src/config.rs b/client/network/src/config.rs index f0e545f57468d..96f6fc27c9ed7 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -518,7 +518,7 @@ pub struct NonDefaultSetConfig { /// `ProtocolHandle` is given to `Notifications` when it initializes itself. This handle allows /// `Notifications ` to communicate with the protocol directly without relaying events through /// `sc-network.` - protocol_handle_pair: Option, + protocol_handle_pair: ProtocolHandlePair, } impl NonDefaultSetConfig { @@ -540,7 +540,7 @@ impl NonDefaultSetConfig { fallback_names, handshake, set_config, - protocol_handle_pair: Some(protocol_handle_pair), + protocol_handle_pair, }, notification_service, ) @@ -572,12 +572,8 @@ impl NonDefaultSetConfig { } /// Take `ProtocolHandlePair` from `NonDefaultSetConfig` - // TODO: consume self instead? - pub(crate) fn take_protocol_handle(&mut self) -> ProtocolHandlePair { - std::mem::take(&mut self.protocol_handle_pair).expect( - "`take_protocol_handle()` is called only once for each `NonDefaultSetConfig`\ - during protocol initialization so it must exist. qed", - ) + pub(crate) fn take_protocol_handle(self) -> ProtocolHandlePair { + self.protocol_handle_pair } /// Modifies the configuration to allow non-reserved nodes. diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 902b6ee8555a4..54860d7034e9a 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -103,7 +103,7 @@ impl Protocol { roles: Roles, network_config: &config::NetworkConfiguration, notification_protocols: Vec, - mut block_announces_protocol: config::NonDefaultSetConfig, + block_announces_protocol: config::NonDefaultSetConfig, _tx: TracingUnboundedSender>, ) -> error::Result<(Self, sc_peerset::PeersetHandle, Vec<(PeerId, Multiaddr)>)> { let mut known_addresses = Vec::new(); @@ -185,7 +185,7 @@ impl Protocol { }, block_announces_protocol.take_protocol_handle(), )) - .chain(notification_protocols.into_iter().map(|mut s| { + .chain(notification_protocols.into_iter().map(|s| { ( notifications::ProtocolConfig { name: s.protocol_name().clone(), diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index bbb74d3b6fc7b..53854348ee443 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -2720,6 +2720,7 @@ mod tests { // we rely on implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(sc_peerset::IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); } @@ -2802,6 +2803,7 @@ mod tests { // we rely on the implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(sc_peerset::IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); // disconnect peer and verify that the state is `Disabled` @@ -2923,6 +2925,7 @@ mod tests { )); notif.peerset_report_accept(sc_peerset::IncomingIndex(0)); + notif.protocol_report_accept(sc_peerset::IncomingIndex(0)); assert_eq!(notif.incoming.len(), 0); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(PeerState::Disabled { .. }))); } @@ -3176,6 +3179,7 @@ mod tests { // We rely on the implementation detail that incoming indices are counted // from 0 to not mock the `Peerset`. notif.peerset_report_accept(sc_peerset::IncomingIndex(0)); + notif.protocol_report_accept(sc_peerset::IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); // open new substream @@ -4042,6 +4046,7 @@ mod tests { // we rely on the implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(sc_peerset::IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); let event = conn_yielder.open_substream(peer, 0, connected, vec![1, 2, 3, 4]); diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index 3c293f653d2c2..3af08c9075303 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -68,8 +68,7 @@ enum InnerNotificationEvent { result_tx: oneshot::Sender, }, - /// Remote identified by `PeerId` opened a substream and sent `Handshake`. - /// Validate `Handshake` and report status (accept/reject) to `Notifications`. + /// Notification substream open to `peer`. NotificationStreamOpened { /// Peer ID. peer: PeerId, diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index 05399f140f800..7865be95ca652 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -807,7 +807,7 @@ where /// Returns a result if the handshake of this peer was indeed accepted. pub fn on_sync_peer_disconnected(&mut self, peer: PeerId) { if !self.peers.remove(&peer).is_some() { - log::debug!(target: LOG_TARGET, "{peer} does not exist in `SyncingEngine`"); + log::error!(target: LOG_TARGET, "{peer} does not exist in `SyncingEngine`"); return } From 182fdc71933b79d1338222af9c21f409e36af74a Mon Sep 17 00:00:00 2001 From: Aaro Altonen <48052676+altonen@users.noreply.github.com> Date: Fri, 26 May 2023 15:58:17 +0300 Subject: [PATCH 21/43] Apply suggestions from code review Co-authored-by: Dmitry Markin --- client/network/src/protocol/notifications/behaviour.rs | 2 +- client/network/src/protocol/notifications/service/tests.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 53854348ee443..3d3117e850c13 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -2279,7 +2279,7 @@ impl NetworkBehaviour for Notifications { self.protocol_report_reject(index); }, Err(_) => { - error!(target: "sub-libp2p", "Protocol ha shut down"); + error!(target: "sub-libp2p", "Protocol has shut down"); break }, } diff --git a/client/network/src/protocol/notifications/service/tests.rs b/client/network/src/protocol/notifications/service/tests.rs index 8867b2098c7a3..39604da6c4355 100644 --- a/client/network/src/protocol/notifications/service/tests.rs +++ b/client/network/src/protocol/notifications/service/tests.rs @@ -427,7 +427,7 @@ async fn cloned_service_opening_substream_works() { // validate inbound substream let mut result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); - // verify that `stream1` also gets the event + // verify that `notif1` gets the event if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = notif1.next_event().await { @@ -442,7 +442,7 @@ async fn cloned_service_opening_substream_works() { // pending assert!(result_rx.try_recv().is_err()); - // verify that `stream2` also gets the event + // verify that `notif2` also gets the event if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = notif2.next_event().await { From 440402cbdfb84fdae04fda913ef14d18978ab8da Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Fri, 26 May 2023 17:37:35 +0300 Subject: [PATCH 22/43] Introduce `NotificationEvent::NotificationSinkReplaced` --- client/consensus/beefy/src/tests.rs | 2 + client/network-gossip/src/bridge.rs | 1 + .../src/protocol/notifications/behaviour.rs | 8 +- .../src/protocol/notifications/service/mod.rs | 41 +++++- .../protocol/notifications/service/tests.rs | 128 ++++++++++++++++++ client/network/src/service/traits.rs | 11 +- client/network/statement/src/lib.rs | 1 + client/network/sync/src/engine.rs | 1 + client/network/test/src/service.rs | 1 + client/network/transactions/src/lib.rs | 1 + 10 files changed, 192 insertions(+), 3 deletions(-) diff --git a/client/consensus/beefy/src/tests.rs b/client/consensus/beefy/src/tests.rs index 436b3adbd3f0a..b7713d90f8d4a 100644 --- a/client/consensus/beefy/src/tests.rs +++ b/client/consensus/beefy/src/tests.rs @@ -1021,7 +1021,9 @@ async fn should_initialize_voter_at_genesis() { assert_eq!(state, persisted_state); } +// TODO(aaro): fix #[tokio::test] +#[ignore] async fn should_initialize_voter_at_custom_genesis() { let keys = &[BeefyKeyring::Alice]; let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap(); diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 139e2d05f407f..1ab35766fb48f 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -223,6 +223,7 @@ impl Future for GossipEngine { ); this.forwarding_state = ForwardingState::Busy(to_forward.into()); }, + NotificationEvent::NotificationSinkReplaced { .. } => {}, }, // The network event stream closed. Do the same for [`GossipValidator`]. Poll::Ready(None) => { diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 3d3117e850c13..28c48c0137b3c 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -1506,9 +1506,15 @@ impl NetworkBehaviour for Notifications { let event = NotificationsOut::CustomProtocolReplaced { peer_id, set_id, - notifications_sink: replacement_sink, + notifications_sink: replacement_sink.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); + // TODO(aaro): check error + let _ = self.protocol_handles[usize::from(set_id)] + .report_notification_sink_replaced( + peer_id, + replacement_sink, + ); } } else { trace!( diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index 3af08c9075303..3be58c015e30f 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -88,7 +88,7 @@ enum InnerNotificationEvent { /// Substream was closed. NotificationStreamClosed { - /// Peer Id. + /// Peer ID. peer: PeerId, }, @@ -100,6 +100,15 @@ enum InnerNotificationEvent { /// Received notification. notification: Vec, }, + + /// Notification sink has been replaced. + NotificationSinkReplaced { + /// Peer ID. + peer: PeerId, + + /// Notification sink. + sink: NotificationsSink, + }, } /// Notification commands. @@ -227,6 +236,10 @@ impl NotificationService for NotificationHandle { }, InnerNotificationEvent::NotificationReceived { peer, notification } => Some(NotificationEvent::NotificationReceived { peer, notification }), + InnerNotificationEvent::NotificationSinkReplaced { peer, sink } => { + self.peers.insert(peer, sink.clone()); + Some(NotificationEvent::NotificationSinkReplaced { peer, sink }) + }, } } @@ -444,6 +457,32 @@ impl ProtocolHandle { Ok(()) } + /// Notification sink was replaced. + pub fn report_notification_sink_replaced( + &mut self, + peer: PeerId, + sink: NotificationsSink, + ) -> Result<(), ()> { + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + + log::trace!( + target: LOG_TARGET, + "{}: notification sink replaced for {peer:?}", + self.protocol + ); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationSinkReplaced { + peer, + sink: sink.clone(), + }) + .is_ok() + }); + + Ok(()) + } + /// Get the number of connected peers. pub fn num_peers(&self) -> usize { self.num_peers diff --git a/client/network/src/protocol/notifications/service/tests.rs b/client/network/src/protocol/notifications/service/tests.rs index 39604da6c4355..c8e74bb5b08ee 100644 --- a/client/network/src/protocol/notifications/service/tests.rs +++ b/client/network/src/protocol/notifications/service/tests.rs @@ -655,3 +655,131 @@ fn try_to_get_notifications_sink_for_non_existent_peer() { let (_proto, notif) = notification_service("/proto/1".into()); assert!(notif.notification_sink(&PeerId::random()).is_none()); } + +#[tokio::test] +async fn notification_sink_replaced() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, mut async_rx, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + role, + negotiated_fallback, + handshake, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(role, ObservedRole::Full); + assert_eq!(handshake, vec![1, 3, 3, 7]); + } else { + panic!("invalid event received"); + } + + // get a copy of the notification sink and send a synchronous notification using. + let sink = notif.notification_sink(&peer_id).unwrap(); + sink.send_sync_notification(vec![1, 3, 3, 6]); + + // send an asynchronous notification using the acquired notifications sink. + let sender = sink.reserve_notification().await.unwrap(); + sender.send(vec![1, 3, 3, 7]).unwrap(); + + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 6] }), + ); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 7] }), + ); + + // send notifications using the stored notification sink as well. + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]).unwrap(); + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }), + ); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }), + ); + + // the initial connection was closed and `Notifications` switched to secondary connection + // and emitted `CustomProtocolReplaced` which informs the local `NotificationService` that + // the notification sink was replaced. + let (new_sink, mut new_async_rx, mut new_sync_rx) = NotificationsSink::new(PeerId::random()); + handle.report_notification_sink_replaced(peer_id, new_sink).unwrap(); + + let new_received_sink = + if let Some(NotificationEvent::NotificationSinkReplaced { peer: _, sink }) = + notif.next_event().await + { + drop(sync_rx); + drop(async_rx); + sink + } else { + panic!("invalid event received"); + }; + + // verify that using the `NotificationService` API automatically results in using the correct + // sink + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]).unwrap(); + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + + assert_eq!( + new_sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }), + ); + assert_eq!( + new_async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }), + ); + + // try to use the old sink to send a synchronous notification but nothing is obviously received + // and no error is reported as defined in the documentation + sink.send_sync_notification(vec![1, 3, 3, 6]); + assert!(new_sync_rx.try_next().is_err()); + + // try to use the old sink to send an asynchronous notification but this time an error is + // returned as the connection has closed. + assert!(sink.reserve_notification().await.is_err()); + + // now send two notifications using the new sink + new_received_sink.send_sync_notification(vec![1, 3, 3, 6]); + + // send an asynchronous notification using the acquired notifications sink. + let sender = new_received_sink.reserve_notification().await.unwrap(); + sender.send(vec![1, 3, 3, 7]).unwrap(); + + assert_eq!( + new_sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 6] }), + ); + assert_eq!( + new_async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 7] }), + ); +} diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 140edc2de9315..f4e959acbeb8f 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -655,12 +655,21 @@ pub enum NotificationEvent { /// Notification was received from the substream. NotificationReceived { - /// Peer IDJ. + /// Peer ID. peer: PeerId, /// Received notification. notification: Vec, }, + + /// Notification sink was replaced. + NotificationSinkReplaced { + /// Peer ID. + peer: PeerId, + + /// Notification sink. + sink: NotificationsSink, + }, } /// Notification service diff --git a/client/network/statement/src/lib.rs b/client/network/statement/src/lib.rs index 350c4d4994994..f1faf4c3c8dca 100644 --- a/client/network/statement/src/lib.rs +++ b/client/network/statement/src/lib.rs @@ -342,6 +342,7 @@ where log::debug!(target: LOG_TARGET, "Failed to decode statement list from {peer}"); } }, + NotificationEvent::NotificationSinkReplaced { .. } => {}, } } diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index 4fe4326f4b007..8b6878030da25 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -827,6 +827,7 @@ where self.process_block_announce_validation_result(res) } }, + NotificationEvent::NotificationSinkReplaced { .. } => {}, } } diff --git a/client/network/test/src/service.rs b/client/network/test/src/service.rs index 8277626af7d48..06de36f740959 100644 --- a/client/network/test/src/service.rs +++ b/client/network/test/src/service.rs @@ -410,6 +410,7 @@ async fn notifications_state_consistent() { .send_sync_notification(&node1.local_peer_id(), b"hello world".to_vec()); } }, + _ => {}, }; } } diff --git a/client/network/transactions/src/lib.rs b/client/network/transactions/src/lib.rs index 1f86ff5fb54a9..9ba20f8f22a32 100644 --- a/client/network/transactions/src/lib.rs +++ b/client/network/transactions/src/lib.rs @@ -349,6 +349,7 @@ where warn!(target: "sub-libp2p", "Failed to decode transactions list"); } }, + NotificationEvent::NotificationSinkReplaced { .. } => {}, } } From 96d2250006555f447ca2942ee7b7f11946a86c9c Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Fri, 26 May 2023 19:48:11 +0300 Subject: [PATCH 23/43] Refactor substream acceptance in `Notifications` --- .../src/protocol/notifications/behaviour.rs | 204 +++++------------- 1 file changed, 49 insertions(+), 155 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 28c48c0137b3c..4387f2f9c60eb 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -261,9 +261,6 @@ enum PeerState { /// If `Some`, any dial attempts to this peer are delayed until the given `Instant`. backoff_until: Option, - /// Whether the substream has been accepted by `Peerset`. - accepted: bool, - /// Incoming index tracking this connection. incoming_index: sc_peerset::IncomingIndex, @@ -900,33 +897,53 @@ impl Notifications { } } - fn protocol_report_accept(&mut self, index: sc_peerset::IncomingIndex) { + /// Substream has been accepted by the `ProtocolController` and must now be sent + /// to the protocol for validation. + fn peerset_report_preaccept(&mut self, index: sc_peerset::IncomingIndex) { + if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { + trace!( + target: LOG_TARGET, + "PSM => Preaccept({:?}): Sent to protocol for validation", + index + ); + let incoming = &self.incoming[pos]; + + match self.protocol_handles[usize::from(incoming.set_id)] + .report_incoming_substream(incoming.peer_id, incoming.handshake.clone()) + { + Ok(rx) => self + .pending_inbound_validations + .push(Box::pin(async move { (rx.await, index) })), + Err(err) => { + error!(target: LOG_TARGET, "protocol has exited: {err:?}"); + debug_assert!(false); + }, + } + } else { + error!(target: LOG_TARGET, "PSM => Preaccept({:?}): Invalid index", index); + } + } + + /// Function that is called when the peerset wants us to accept a connection + /// request from a peer. + fn peerset_report_accept(&mut self, index: sc_peerset::IncomingIndex) { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) } else { - error!(target: LOG_TARGET, "Protocol => Accept({:?}): Invalid index", index); + error!(target: "sub-libp2p", "PSM => Accept({:?}): Invalid index", index); return }; if !incoming.alive { - trace!( - target: LOG_TARGET, - "Protocol => Accept({:?}, {}, {:?}): Obsolete incoming", - index, - incoming.peer_id, - incoming.set_id - ); + trace!(target: "sub-libp2p", "PSM => Accept({:?}, {}, {:?}): Obsolete incoming", + index, incoming.peer_id, incoming.set_id); match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { Some(PeerState::DisabledPendingEnable { .. }) | Some(PeerState::Enabled { .. }) => { }, _ => { - trace!( - target: LOG_TARGET, - "Protocol <= Dropped({}, {:?})", - incoming.peer_id, - incoming.set_id - ); + trace!(target: "sub-libp2p", "PSM <= Dropped({}, {:?})", + incoming.peer_id, incoming.set_id); self.peerset.dropped(incoming.set_id, incoming.peer_id, DropReason::Unknown); }, } @@ -971,13 +988,8 @@ impl Notifications { .iter_mut() .filter(|(_, s)| matches!(s, ConnectionState::OpenDesiredByRemote)) { - trace!( - target: LOG_TARGET, - "Handler({:?}, {:?}) <= Open({:?})", - incoming.peer_id, - *connec_id, - incoming.set_id - ); + trace!(target: "sub-libp2p", "Handler({:?}, {:?}) <= Open({:?})", + incoming.peer_id, *connec_id, incoming.set_id); self.events.push_back(ToSwarm::NotifyHandler { peer_id: incoming.peer_id, handler: NotifyHandler::One(*connec_id), @@ -991,110 +1003,9 @@ impl Notifications { // Any state other than `Incoming` is invalid. peer => { - error!( - target: LOG_TARGET, - "State mismatch in libp2p: Expected alive incoming. Got {:?}.", peer - ); - debug_assert!(false); - }, - } - } - - /// Protocol rejected inbound substream. - fn protocol_report_reject(&mut self, index: sc_peerset::IncomingIndex) { - self.peerset_report_reject(index); - } - - /// Function that is called when the peerset wants us to accept a connection - /// request from a peer. - fn peerset_report_accept(&mut self, index: sc_peerset::IncomingIndex) { - let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) - { - self.incoming.remove(pos) - } else { - error!(target: LOG_TARGET, "PSM => Accept({:?}): Invalid index", index); - return - }; - - if !incoming.alive { - trace!( - target: LOG_TARGET, - "PSM => Accept({:?}, {}, {:?}): Obsolete incoming", - index, - incoming.peer_id, - incoming.set_id - ); - match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { - Some(PeerState::DisabledPendingEnable { .. }) | Some(PeerState::Enabled { .. }) => { - }, - _ => { - trace!( - target: LOG_TARGET, - "PSM <= Dropped({}, {:?})", - incoming.peer_id, - incoming.set_id - ); - self.peerset.dropped(incoming.set_id, incoming.peer_id, DropReason::Unknown); - }, - } - return - } - - let state = match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { - Some(s) => s, - None => { - debug_assert!(false); - return - }, - }; - - match mem::replace(state, PeerState::Poisoned) { - // Incoming => Enabled - PeerState::Incoming { - ref mut accepted, - backoff_until, - connections, - incoming_index, - .. - } => { - trace!( - target: LOG_TARGET, - "PSM => Accept({:?}, {}, {:?}): Send substream for validation to protocol.", - index, - incoming.peer_id, - incoming.set_id - ); - - match self.protocol_handles[usize::from(incoming.set_id)] - .report_incoming_substream(incoming.peer_id, incoming.handshake.clone()) - { - Ok(rx) => { - *accepted = true; - self.pending_inbound_validations - .push(Box::pin(async move { (rx.await, index) })); - self.incoming.push(incoming); - }, - Err(err) => { - error!(target: LOG_TARGET, "protocol has exited: {err:?}"); - debug_assert!(false); - return - }, - } - - *state = PeerState::Incoming { - backoff_until, - accepted: true, - connections, - incoming_index, - }; - }, - - // Any state other than `Incoming` is invalid. - peer => { - error!( - target: LOG_TARGET, - "State mismatch in libp2p: Expected alive incoming. Got {:?}.", peer - ); + error!(target: "sub-libp2p", + "State mismatch in libp2p: Expected alive incoming. Got {:?}.", + peer); debug_assert!(false); }, } @@ -1466,7 +1377,6 @@ impl NetworkBehaviour for Notifications { connections, backoff_until, incoming_index, - accepted: false, }; } }, @@ -1695,12 +1605,7 @@ impl NetworkBehaviour for Notifications { match mem::replace(entry.get_mut(), PeerState::Poisoned) { // Incoming => Incoming - PeerState::Incoming { - mut connections, - backoff_until, - incoming_index, - accepted, - } => { + PeerState::Incoming { mut connections, backoff_until, incoming_index } => { debug_assert!(connections .iter() .any(|(_, s)| matches!(s, ConnectionState::OpenDesiredByRemote))); @@ -1728,12 +1633,8 @@ impl NetworkBehaviour for Notifications { debug_assert!(false); } - *entry.into_mut() = PeerState::Incoming { - connections, - backoff_until, - incoming_index, - accepted, - }; + *entry.into_mut() = + PeerState::Incoming { connections, backoff_until, incoming_index }; }, PeerState::Enabled { mut connections } => { @@ -1803,7 +1704,6 @@ impl NetworkBehaviour for Notifications { connections, backoff_until, incoming_index: incoming_id, - accepted: false, }; } else { // Connections in `OpeningThenClosing` and `Closing` state can be @@ -2235,7 +2135,7 @@ impl NetworkBehaviour for Notifications { loop { match futures::Stream::poll_next(Pin::new(&mut self.peerset), cx) { Poll::Ready(Some(sc_peerset::Message::Accept(index))) => { - self.peerset_report_accept(index); + self.peerset_report_preaccept(index); }, Poll::Ready(Some(sc_peerset::Message::Reject(index))) => { self.peerset_report_reject(index); @@ -2279,10 +2179,11 @@ impl NetworkBehaviour for Notifications { { match result { Ok(ValidationResult::Accept) => { - self.protocol_report_accept(index); + self.peerset_report_accept(index); }, Ok(ValidationResult::Reject) => { - self.protocol_report_reject(index); + // TODO(aaro): remove connection from peerset + self.peerset_report_reject(index); }, Err(_) => { error!(target: "sub-libp2p", "Protocol has shut down"); @@ -2569,12 +2470,10 @@ mod tests { }, ); - if let Some(&PeerState::Incoming { - ref connections, backoff_until: None, accepted, .. - }) = notif.peers.get(&(peer, 0.into())) + if let Some(&PeerState::Incoming { ref connections, backoff_until: None, .. }) = + notif.peers.get(&(peer, 0.into())) { assert_eq!(connections.len(), 1); - assert_eq!(accepted, false); assert_eq!(connections[0], (conn, ConnectionState::OpenDesiredByRemote)); } else { panic!("invalid state"); @@ -2726,7 +2625,6 @@ mod tests { // we rely on implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` notif.peerset_report_accept(IncomingIndex(0)); - notif.protocol_report_accept(sc_peerset::IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); } @@ -2809,7 +2707,6 @@ mod tests { // we rely on the implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` notif.peerset_report_accept(IncomingIndex(0)); - notif.protocol_report_accept(sc_peerset::IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); // disconnect peer and verify that the state is `Disabled` @@ -2931,7 +2828,6 @@ mod tests { )); notif.peerset_report_accept(sc_peerset::IncomingIndex(0)); - notif.protocol_report_accept(sc_peerset::IncomingIndex(0)); assert_eq!(notif.incoming.len(), 0); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(PeerState::Disabled { .. }))); } @@ -3185,7 +3081,6 @@ mod tests { // We rely on the implementation detail that incoming indices are counted // from 0 to not mock the `Peerset`. notif.peerset_report_accept(sc_peerset::IncomingIndex(0)); - notif.protocol_report_accept(sc_peerset::IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); // open new substream @@ -4052,7 +3947,6 @@ mod tests { // we rely on the implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` notif.peerset_report_accept(IncomingIndex(0)); - notif.protocol_report_accept(sc_peerset::IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); let event = conn_yielder.open_substream(peer, 0, connected, vec![1, 2, 3, 4]); From ea56743ffbedb0b367a87d2f32639a347aee5cdd Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Mon, 5 Jun 2023 11:51:04 +0300 Subject: [PATCH 24/43] Rename `Peerset` functions --- .../src/protocol/notifications/behaviour.rs | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 44cdcea1482af..c473fcbfe356f 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -926,7 +926,7 @@ impl Notifications { /// Function that is called when the peerset wants us to accept a connection /// request from a peer. - fn peerset_report_accept(&mut self, index: crate::peerset::IncomingIndex) { + fn protocol_report_accept(&mut self, index: crate::peerset::IncomingIndex) { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) @@ -1012,7 +1012,7 @@ impl Notifications { } /// Function that is called when the peerset wants us to reject an incoming peer. - fn peerset_report_reject(&mut self, index: crate::peerset::IncomingIndex) { + fn protocol_report_reject(&mut self, index: crate::peerset::IncomingIndex) { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) @@ -2139,7 +2139,7 @@ impl NetworkBehaviour for Notifications { self.peerset_report_preaccept(index); }, Poll::Ready(Some(crate::peerset::Message::Reject(index))) => { - self.peerset_report_reject(index); + self.protocol_report_reject(index); }, Poll::Ready(Some(crate::peerset::Message::Connect { peer_id, set_id, .. })) => { self.peerset_report_connect(peer_id, set_id); @@ -2180,11 +2180,11 @@ impl NetworkBehaviour for Notifications { { match result { Ok(ValidationResult::Accept) => { - self.peerset_report_accept(index); + self.protocol_report_accept(index); }, Ok(ValidationResult::Reject) => { // TODO(aaro): remove connection from peerset - self.peerset_report_reject(index); + self.protocol_report_reject(index); }, Err(_) => { error!(target: "sub-libp2p", "Protocol has shut down"); @@ -2624,7 +2624,7 @@ mod tests { // attempt to connect to the peer and verify that the peer state is `Enabled`; // we rely on implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); } @@ -2706,7 +2706,7 @@ mod tests { ); // we rely on the implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); // disconnect peer and verify that the state is `Disabled` @@ -2827,7 +2827,7 @@ mod tests { IncomingPeer { alive: false, incoming_id: crate::peerset::IncomingIndex(0), .. }, )); - notif.peerset_report_accept(crate::peerset::IncomingIndex(0)); + notif.protocol_report_accept(crate::peerset::IncomingIndex(0)); assert_eq!(notif.incoming.len(), 0); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(PeerState::Disabled { .. }))); } @@ -3080,7 +3080,7 @@ mod tests { // We rely on the implementation detail that incoming indices are counted // from 0 to not mock the `Peerset`. - notif.peerset_report_accept(crate::peerset::IncomingIndex(0)); + notif.protocol_report_accept(crate::peerset::IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); // open new substream @@ -3946,7 +3946,7 @@ mod tests { // we rely on the implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); let event = conn_yielder.open_substream(peer, 0, connected, vec![1, 2, 3, 4]); @@ -4277,7 +4277,7 @@ mod tests { #[test] #[should_panic] #[cfg(debug_assertions)] - fn peerset_report_accept_incoming_peer() { + fn protocol_report_accept_incoming_peer() { let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -4315,13 +4315,13 @@ mod tests { )); notif.peers.remove(&(peer, set_id)); - notif.peerset_report_accept(crate::peerset::IncomingIndex(0)); + notif.protocol_report_accept(crate::peerset::IncomingIndex(0)); } #[test] #[should_panic] #[cfg(debug_assertions)] - fn peerset_report_accept_not_incoming_peer() { + fn protocol_report_accept_not_incoming_peer() { let (mut notif, _peerset, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -4367,7 +4367,7 @@ mod tests { assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); notif.incoming[0].alive = true; - notif.peerset_report_accept(crate::peerset::IncomingIndex(0)); + notif.protocol_report_accept(crate::peerset::IncomingIndex(0)); } #[test] @@ -4408,7 +4408,7 @@ mod tests { fn accept_non_existent_connection() { let (mut notif, _peerset, _notif_service) = development_notifs(); - notif.peerset_report_accept(0.into()); + notif.protocol_report_accept(0.into()); assert!(notif.peers.is_empty()); assert!(notif.incoming.is_empty()); @@ -4418,7 +4418,7 @@ mod tests { fn reject_non_existent_connection() { let (mut notif, _peerset, _notif_service) = development_notifs(); - notif.peerset_report_reject(0.into()); + notif.protocol_report_reject(0.into()); assert!(notif.peers.is_empty()); assert!(notif.incoming.is_empty()); @@ -4458,7 +4458,7 @@ mod tests { assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); notif.incoming[0].alive = false; - notif.peerset_report_reject(0.into()); + notif.protocol_report_reject(0.into()); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); } @@ -4503,7 +4503,7 @@ mod tests { )); notif.peers.remove(&(peer, set_id)); - notif.peerset_report_reject(0.into()); + notif.protocol_report_reject(0.into()); } #[test] From 10977383f099f7ee743f40f301fe211049026a35 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 6 Jun 2023 12:41:17 +0300 Subject: [PATCH 25/43] Add metrics --- client/network/src/config.rs | 2 +- .../src/protocol/notifications/behaviour.rs | 2 +- .../protocol/notifications/service/metrics.rs | 101 ++++++++++++++++++ .../src/protocol/notifications/service/mod.rs | 46 +++++--- .../protocol/notifications/service/tests.rs | 32 +++--- .../src/protocol/notifications/tests.rs | 2 +- 6 files changed, 154 insertions(+), 31 deletions(-) diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 96f6fc27c9ed7..9395c3b2e8bcf 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -532,7 +532,7 @@ impl NonDefaultSetConfig { set_config: SetConfig, ) -> (Self, Box) { let (protocol_handle_pair, notification_service) = - notification_service(protocol_name.clone()); + notification_service(protocol_name.clone(), None); // TODO(aaro): fix, needs `FullNetworkConfig` ( Self { protocol_name, diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index c473fcbfe356f..9fca3fca075e0 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -2316,7 +2316,7 @@ mod tests { Box, ) { let (protocol_handle_pair, notif_service) = - crate::protocol::notifications::service::notification_service("/proto/1".into()); + crate::protocol::notifications::service::notification_service("/proto/1".into(), None); let (peerset, peerset_handle) = { let mut sets = Vec::with_capacity(1); diff --git a/client/network/src/protocol/notifications/service/metrics.rs b/client/network/src/protocol/notifications/service/metrics.rs index b46a8f75295fe..dd5bcb230e7bf 100644 --- a/client/network/src/protocol/notifications/service/metrics.rs +++ b/client/network/src/protocol/notifications/service/metrics.rs @@ -15,3 +15,104 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + +use crate::types::ProtocolName; + +use prometheus_endpoint::{ + self as prometheus, CounterVec, HistogramOpts, HistogramVec, Opts, PrometheusError, Registry, + U64, +}; + +/// Notification metrics. +#[derive(Debug, Clone)] +pub struct Metrics { + pub notifications_sizes: HistogramVec, + pub notifications_streams_closed_total: CounterVec, + pub notifications_streams_opened_total: CounterVec, +} + +impl Metrics { + fn register(registry: &Registry) -> Result { + Ok(Self { + notifications_sizes: prometheus::register( + HistogramVec::new( + HistogramOpts { + common_opts: Opts::new( + "substrate_sub_libp2p_notifications_sizes", + "Sizes of the notifications send to and received from all nodes", + ), + buckets: prometheus::exponential_buckets(64.0, 4.0, 8) + .expect("parameters are always valid values; qed"), + }, + &["direction", "protocol"], + )?, + registry, + )?, + notifications_streams_closed_total: prometheus::register( + CounterVec::new( + Opts::new( + "substrate_sub_libp2p_notifications_streams_closed_total", + "Total number of notification substreams that have been closed", + ), + &["protocol"], + )?, + registry, + )?, + notifications_streams_opened_total: prometheus::register( + CounterVec::new( + Opts::new( + "substrate_sub_libp2p_notifications_streams_opened_total", + "Total number of notification substreams that have been opened", + ), + &["protocol"], + )?, + registry, + )?, + }) + } +} + +/// Register metrics. +pub fn register(registry: &Registry) -> Result { + Metrics::register(registry) +} + +/// Register opened substream to Prometheus. +pub fn register_substream_opened(metrics: &Option, protocol: &ProtocolName) { + if let Some(metrics) = metrics { + metrics.notifications_streams_opened_total.with_label_values(&[&protocol]).inc(); + } +} + +/// Register opened substream to Prometheus. +pub fn register_substream_closed(metrics: &Option, protocol: &ProtocolName) { + if let Some(metrics) = metrics { + metrics + .notifications_streams_closed_total + .with_label_values(&[&protocol[..]]) + .inc(); + } +} + +/// Register opened substream to Prometheus. +pub fn _register_notification_sent( + _metrics: &Option, + _protocol: &ProtocolName, + _size: usize, +) { + todo!(); +} + +/// Register opened substream to Prometheus. +pub fn register_notification_received( + metrics: &Option, + protocol: &ProtocolName, + size: usize, +) { + if let Some(metrics) = metrics { + metrics + .notifications_sizes + .with_label_values(&["in", protocol]) + .observe(size as f64); + } +} diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index 3be58c015e30f..7454a65cbef18 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -30,6 +30,7 @@ use futures::{ StreamExt, }; use libp2p::PeerId; +use prometheus_endpoint::Registry; use tokio::sync::{mpsc, oneshot}; use tokio_stream::wrappers::ReceiverStream; @@ -142,6 +143,9 @@ pub struct NotificationHandle { /// Connected peers. peers: HashMap, + + /// Prometheus metrics. + metrics: Option, } impl NotificationHandle { @@ -151,8 +155,9 @@ impl NotificationHandle { tx: mpsc::Sender, rx: TracingUnboundedReceiver, subscribers: Arc>>>, + metrics: Option, ) -> Self { - Self { protocol, tx, rx, subscribers, peers: HashMap::new() } + Self { protocol, tx, rx, subscribers, peers: HashMap::new(), metrics } } } @@ -178,7 +183,7 @@ impl NotificationService for NotificationHandle { self.peers .get(&peer) - // TODO: check what the current implementation does in case the peer doesn't exist + // TODO(aaro): check what the current implementation does in case the peer doesn't exist .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? .send_sync_notification(notification); Ok(()) @@ -194,7 +199,7 @@ impl NotificationService for NotificationHandle { self.peers .get(&peer) - // TODO: check what the current implementation does in case the peer doesn't exist + // TODO(aaro): check what the current implementation does in case the peer doesn't exist .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? .reserve_notification() .await @@ -255,6 +260,7 @@ impl NotificationService for NotificationHandle { rx: event_rx, peers: self.peers.clone(), subscribers: self.subscribers.clone(), + metrics: self.metrics.clone(), })) } @@ -280,6 +286,9 @@ pub struct ProtocolHandlePair { // Receiver for notification commands received from the protocol implementation. rx: mpsc::Receiver, + + /// Prometheus metrics. + metrics: Option, } impl ProtocolHandlePair { @@ -288,8 +297,9 @@ impl ProtocolHandlePair { protocol: ProtocolName, subscribers: Subscribers, rx: mpsc::Receiver, + metrics: Option, ) -> Self { - Self { protocol, subscribers, rx } + Self { protocol, subscribers, rx, metrics } } /// Consume `self` and split [`ProtocolHandlePair`] into a handle which allows it to send events @@ -298,7 +308,7 @@ impl ProtocolHandlePair { self, ) -> (ProtocolHandle, Box + Send + Unpin>) { ( - ProtocolHandle::new(self.protocol, self.subscribers), + ProtocolHandle::new(self.protocol, self.subscribers, self.metrics), Box::new(ReceiverStream::new(self.rx)), ) } @@ -316,11 +326,18 @@ pub struct ProtocolHandle { /// Number of connected peers. num_peers: usize, + + /// Prometheus metrics. + metrics: Option, } impl ProtocolHandle { - fn new(protocol: ProtocolName, subscribers: Subscribers) -> Self { - Self { protocol, subscribers, num_peers: 0usize } + fn new( + protocol: ProtocolName, + subscribers: Subscribers, + metrics: Option, + ) -> Self { + Self { protocol, subscribers, num_peers: 0usize, metrics } } /// Report to the protocol that a substream has been opened and it must be validated by the @@ -399,8 +416,9 @@ impl ProtocolHandle { negotiated_fallback: Option, sink: NotificationsSink, ) -> Result<(), ()> { - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + metrics::register_substream_opened(&self.metrics, &self.protocol); + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; log::trace!(target: LOG_TARGET, "{}: substream opened for {peer:?}", self.protocol); subscribers.retain(|subscriber| { @@ -421,8 +439,9 @@ impl ProtocolHandle { /// Substream was closed. pub fn report_substream_closed(&mut self, peer: PeerId) -> Result<(), ()> { - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + metrics::register_substream_closed(&self.metrics, &self.protocol); + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; log::trace!(target: LOG_TARGET, "{}: substream closed for {peer:?}", self.protocol); subscribers.retain(|subscriber| { @@ -441,8 +460,9 @@ impl ProtocolHandle { peer: PeerId, notification: Vec, ) -> Result<(), ()> { - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + metrics::register_notification_received(&self.metrics, &self.protocol, notification.len()); + let mut subscribers = self.subscribers.lock().map_err(|_| ())?; log::trace!(target: LOG_TARGET, "{}: notification received from {peer:?}", self.protocol); subscribers.retain(|subscriber| { @@ -494,13 +514,15 @@ impl ProtocolHandle { /// Handle pair allows `Notifications` and the protocol to communicate with each other directly. pub fn notification_service( protocol: ProtocolName, + metrics: Option, ) -> (ProtocolHandlePair, Box) { let (cmd_tx, cmd_rx) = mpsc::channel(64); // TODO: zzz let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); let subscribers = Arc::new(Mutex::new(vec![event_tx])); + let metrics = metrics.map_or(None, |registry| metrics::register(®istry).ok()); ( - ProtocolHandlePair::new(protocol.clone(), subscribers.clone(), cmd_rx), - Box::new(NotificationHandle::new(protocol.clone(), cmd_tx, event_rx, subscribers)), + ProtocolHandlePair::new(protocol.clone(), subscribers.clone(), cmd_rx, metrics.clone()), + Box::new(NotificationHandle::new(protocol.clone(), cmd_tx, event_rx, subscribers, metrics)), ) } diff --git a/client/network/src/protocol/notifications/service/tests.rs b/client/network/src/protocol/notifications/service/tests.rs index c8e74bb5b08ee..1b7b2a4bb8922 100644 --- a/client/network/src/protocol/notifications/service/tests.rs +++ b/client/network/src/protocol/notifications/service/tests.rs @@ -23,7 +23,7 @@ use crate::protocol::notifications::handler::{ #[tokio::test] async fn validate_and_accept_substream() { - let (proto, mut notif) = notification_service("/proto/1".into()); + let (proto, mut notif) = notification_service("/proto/1".into(), None); let (handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -44,7 +44,7 @@ async fn validate_and_accept_substream() { #[tokio::test] async fn substream_opened() { - let (proto, mut notif) = notification_service("/proto/1".into()); + let (proto, mut notif) = notification_service("/proto/1".into(), None); let (sink, _, _) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); @@ -71,7 +71,7 @@ async fn substream_opened() { #[tokio::test] async fn send_sync_notification() { - let (proto, mut notif) = notification_service("/proto/1".into()); + let (proto, mut notif) = notification_service("/proto/1".into(), None); let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -119,7 +119,7 @@ async fn send_sync_notification() { #[tokio::test] async fn send_async_notification() { - let (proto, mut notif) = notification_service("/proto/1".into()); + let (proto, mut notif) = notification_service("/proto/1".into(), None); let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -167,7 +167,7 @@ async fn send_async_notification() { #[tokio::test] async fn send_sync_notification_to_non_existent_peer() { - let (proto, notif) = notification_service("/proto/1".into()); + let (proto, notif) = notification_service("/proto/1".into(), None); let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); let (_handle, _stream) = proto.split(); let peer = PeerId::random(); @@ -183,7 +183,7 @@ async fn send_sync_notification_to_non_existent_peer() { #[tokio::test] async fn send_async_notification_to_non_existent_peer() { - let (proto, notif) = notification_service("/proto/1".into()); + let (proto, notif) = notification_service("/proto/1".into(), None); let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); let (_handle, _stream) = proto.split(); let peer = PeerId::random(); @@ -199,7 +199,7 @@ async fn send_async_notification_to_non_existent_peer() { #[tokio::test] async fn receive_notification() { - let (proto, mut notif) = notification_service("/proto/1".into()); + let (proto, mut notif) = notification_service("/proto/1".into(), None); let (sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -253,7 +253,7 @@ async fn receive_notification() { #[tokio::test] async fn backpressure_works() { - let (proto, mut notif) = notification_service("/proto/1".into()); + let (proto, mut notif) = notification_service("/proto/1".into(), None); let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -313,7 +313,7 @@ async fn backpressure_works() { #[tokio::test] async fn peer_disconnects_then_sync_notification_is_sent() { - let (proto, mut notif) = notification_service("/proto/1".into()); + let (proto, mut notif) = notification_service("/proto/1".into(), None); let (sink, _, sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -363,7 +363,7 @@ async fn peer_disconnects_then_sync_notification_is_sent() { #[tokio::test] async fn peer_disconnects_then_async_notification_is_sent() { - let (proto, mut notif) = notification_service("/proto/1".into()); + let (proto, mut notif) = notification_service("/proto/1".into(), None); let (sink, async_rx, _) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -418,7 +418,7 @@ async fn peer_disconnects_then_async_notification_is_sent() { #[tokio::test] async fn cloned_service_opening_substream_works() { - let (proto, mut notif1) = notification_service("/proto/1".into()); + let (proto, mut notif1) = notification_service("/proto/1".into(), None); let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); let (handle, _stream) = proto.split(); let mut notif2 = notif1.clone().unwrap(); @@ -458,7 +458,7 @@ async fn cloned_service_opening_substream_works() { #[tokio::test] async fn cloned_service_one_service_rejects_substream() { - let (proto, mut notif1) = notification_service("/proto/1".into()); + let (proto, mut notif1) = notification_service("/proto/1".into(), None); let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); let (handle, _stream) = proto.split(); let mut notif2 = notif1.clone().unwrap(); @@ -497,7 +497,7 @@ async fn cloned_service_one_service_rejects_substream() { #[tokio::test] async fn cloned_service_opening_substream_sending_and_receiving_notifications_work() { - let (proto, mut notif1) = notification_service("/proto/1".into()); + let (proto, mut notif1) = notification_service("/proto/1".into(), None); let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let mut notif2 = notif1.clone().unwrap(); @@ -580,7 +580,7 @@ async fn cloned_service_opening_substream_sending_and_receiving_notifications_wo #[tokio::test] async fn sending_notifications_using_notifications_sink_works() { - let (proto, mut notif) = notification_service("/proto/1".into()); + let (proto, mut notif) = notification_service("/proto/1".into(), None); let (sink, mut async_rx, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -652,13 +652,13 @@ async fn sending_notifications_using_notifications_sink_works() { #[test] fn try_to_get_notifications_sink_for_non_existent_peer() { - let (_proto, notif) = notification_service("/proto/1".into()); + let (_proto, notif) = notification_service("/proto/1".into(), None); assert!(notif.notification_sink(&PeerId::random()).is_none()); } #[tokio::test] async fn notification_sink_replaced() { - let (proto, mut notif) = notification_service("/proto/1".into()); + let (proto, mut notif) = notification_service("/proto/1".into(), None); let (sink, mut async_rx, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); diff --git a/client/network/src/protocol/notifications/tests.rs b/client/network/src/protocol/notifications/tests.rs index b513ba0b6dda6..70745b17bc1b5 100644 --- a/client/network/src/protocol/notifications/tests.rs +++ b/client/network/src/protocol/notifications/tests.rs @@ -84,7 +84,7 @@ fn build_nodes() -> (Swarm, Swarm) { }], }); let (protocol_handle_pair, _notif_service) = - crate::protocol::notifications::service::notification_service("/foo".into()); + crate::protocol::notifications::service::notification_service("/foo".into(), None); let behaviour = CustomProtoWithAddr { inner: Notifications::new( From 711761d85d4d9ea63a971eaf31d4d19095218054 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Wed, 14 Jun 2023 17:46:17 +0300 Subject: [PATCH 26/43] Introduce `MessageSink` The previous implementation of exposing `NotificationsSink` to the protocol was too leaky. Introduce new `MessageSink` abstraction which provides a clean send interface and internally handles notification sink replaces so the protocol doesn't have replace the sink when it's replaced. --- .../src/protocol/notifications/service/mod.rs | 145 +++++++++++++----- .../protocol/notifications/service/tests.rs | 48 +++--- client/network/src/service/traits.rs | 39 +++-- client/network/sync/src/engine.rs | 1 - client/network/transactions/src/lib.rs | 1 - 5 files changed, 151 insertions(+), 83 deletions(-) diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index 7454a65cbef18..5a43414118a70 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -21,7 +21,7 @@ use crate::{ error, protocol::notifications::handler::NotificationsSink, - service::traits::{NotificationEvent, NotificationService, ValidationResult}, + service::traits::{MessageSink, NotificationEvent, NotificationService, ValidationResult}, types::ProtocolName, }; @@ -30,6 +30,7 @@ use futures::{ StreamExt, }; use libp2p::PeerId; +use parking_lot::Mutex; use prometheus_endpoint::Registry; use tokio::sync::{mpsc, oneshot}; use tokio_stream::wrappers::ReceiverStream; @@ -37,11 +38,7 @@ use tokio_stream::wrappers::ReceiverStream; use sc_network_common::role::ObservedRole; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; -use std::{ - collections::HashMap, - fmt::Debug, - sync::{Arc, Mutex}, -}; +use std::{collections::HashMap, fmt::Debug, sync::Arc}; mod metrics; #[cfg(test)] @@ -53,6 +50,32 @@ const LOG_TARGET: &str = "notification-service"; /// Type representing subscribers of a notification protocol. type Subscribers = Arc>>>; +/// Type represending a distributable message sink. +/// +/// See documentation for [`PeerContext`] for more details. +type NotificationSink = Arc>; + +#[async_trait::async_trait] +impl MessageSink for NotificationSink { + /// Send synchronous `notification` to the peer associated with this [`MessageService`]. + fn send_sync_notification(&self, notification: Vec) { + self.lock().send_sync_notification(notification); + } + + /// Send an asynchronous `notification` to to the peer associated with this [`MessageService`], + /// allowing sender to exercise backpressure. + /// + /// Returns an error if the peer does not exist. + async fn send_async_notification(&self, notification: Vec) -> Result<(), error::Error> { + // notification sink must be cloned because the lock cannot be held across `.await` + // this makes the implementation less efficient but not prohibitively so as the same + // method is also used by `NetworkService` when sending notifications. + let sink = self.lock().clone(); + let sink = sink.reserve_notification().await.map_err(|_| error::Error::ConnectionClosed)?; + sink.send(notification).map_err(|_| error::Error::ChannelClosed) + } +} + /// Inner notification event to deal with `NotificationsSinks` without exposing that /// implementation detail to [`NotificationService`] consumers. #[derive(Debug)] @@ -126,6 +149,23 @@ pub enum NotificationCommand { SetHandshake(Vec), } +/// Context assigned to each peer. +/// +/// Contains `NotificationsSink` used by [`NotificationService`] to send notifications +/// and an additional, distributable `NotificationsSink` which the protocol may acquire +/// if it wishes to send notifications through `NotificationsSink` directly. +/// +/// The distributable `NoticationsSink` is wrapped in an `Arc>` to allow +/// `NotificationsService` to swap underlying sink in case the sink is replaced. +#[derive(Debug, Clone)] +struct PeerContext { + /// Sink for sending notificaitons. + sink: NotificationsSink, + + /// Distributable notification sink. + shared_sink: NotificationSink, +} + /// Handle that is passed on to the notifications protocol. #[derive(Debug)] pub struct NotificationHandle { @@ -142,7 +182,7 @@ pub struct NotificationHandle { subscribers: Subscribers, /// Connected peers. - peers: HashMap, + peers: HashMap, /// Prometheus metrics. metrics: Option, @@ -185,6 +225,7 @@ impl NotificationService for NotificationHandle { .get(&peer) // TODO(aaro): check what the current implementation does in case the peer doesn't exist .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? + .sink .send_sync_notification(notification); Ok(()) } @@ -201,6 +242,7 @@ impl NotificationService for NotificationHandle { .get(&peer) // TODO(aaro): check what the current implementation does in case the peer doesn't exist .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? + .sink .reserve_notification() .await .map_err(|_| error::Error::ConnectionClosed)? @@ -217,40 +259,60 @@ impl NotificationService for NotificationHandle { /// Get next event from the `Notifications` event stream. async fn next_event(&mut self) -> Option { - match self.rx.next().await? { - InnerNotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => - Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }), - InnerNotificationEvent::NotificationStreamOpened { - peer, - handshake, - role, - negotiated_fallback, - sink, - } => { - self.peers.insert(peer, sink); - Some(NotificationEvent::NotificationStreamOpened { + loop { + match self.rx.next().await? { + InnerNotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => + return Some(NotificationEvent::ValidateInboundSubstream { + peer, + handshake, + result_tx, + }), + InnerNotificationEvent::NotificationStreamOpened { peer, handshake, role, negotiated_fallback, - }) - }, - InnerNotificationEvent::NotificationStreamClosed { peer } => { - self.peers.remove(&peer); - Some(NotificationEvent::NotificationStreamClosed { peer }) - }, - InnerNotificationEvent::NotificationReceived { peer, notification } => - Some(NotificationEvent::NotificationReceived { peer, notification }), - InnerNotificationEvent::NotificationSinkReplaced { peer, sink } => { - self.peers.insert(peer, sink.clone()); - Some(NotificationEvent::NotificationSinkReplaced { peer, sink }) - }, + sink, + } => { + self.peers.insert( + peer, + PeerContext { sink: sink.clone(), shared_sink: Arc::new(Mutex::new(sink)) }, + ); + return Some(NotificationEvent::NotificationStreamOpened { + peer, + handshake, + role, + negotiated_fallback, + }) + }, + InnerNotificationEvent::NotificationStreamClosed { peer } => { + self.peers.remove(&peer); + return Some(NotificationEvent::NotificationStreamClosed { peer }) + }, + InnerNotificationEvent::NotificationReceived { peer, notification } => + return Some(NotificationEvent::NotificationReceived { peer, notification }), + InnerNotificationEvent::NotificationSinkReplaced { peer, sink } => { + println!("replace sink"); + + match self.peers.get_mut(&peer) { + None => log::error!( + "{}: notification sink replaced for {peer} but peer does not exist", + self.protocol + ), + Some(context) => { + println!("done"); + context.sink = sink.clone(); + *context.shared_sink.lock() = sink.clone(); + }, + } + }, + } } } // Clone [`NotificationService`] fn clone(&mut self) -> Result, ()> { - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + let mut subscribers = self.subscribers.lock(); let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); subscribers.push(event_tx); @@ -269,9 +331,12 @@ impl NotificationService for NotificationHandle { &self.protocol } - /// Get notification sink of the peer. - fn notification_sink(&self, peer: &PeerId) -> Option { - self.peers.get(peer).map(|sink| sink.clone()) + /// Get message sink of the peer. + fn message_sink(&self, peer: &PeerId) -> Option> { + match self.peers.get(peer) { + Some(context) => Some(Box::new(context.shared_sink.clone())), + None => None, + } } } @@ -350,7 +415,7 @@ impl ProtocolHandle { peer: PeerId, handshake: Vec, ) -> Result, ()> { - let subscribers = self.subscribers.lock().map_err(|_| ())?; + let subscribers = self.subscribers.lock(); log::trace!( target: LOG_TARGET, @@ -418,7 +483,7 @@ impl ProtocolHandle { ) -> Result<(), ()> { metrics::register_substream_opened(&self.metrics, &self.protocol); - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + let mut subscribers = self.subscribers.lock(); log::trace!(target: LOG_TARGET, "{}: substream opened for {peer:?}", self.protocol); subscribers.retain(|subscriber| { @@ -441,7 +506,7 @@ impl ProtocolHandle { pub fn report_substream_closed(&mut self, peer: PeerId) -> Result<(), ()> { metrics::register_substream_closed(&self.metrics, &self.protocol); - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + let mut subscribers = self.subscribers.lock(); log::trace!(target: LOG_TARGET, "{}: substream closed for {peer:?}", self.protocol); subscribers.retain(|subscriber| { @@ -462,7 +527,7 @@ impl ProtocolHandle { ) -> Result<(), ()> { metrics::register_notification_received(&self.metrics, &self.protocol, notification.len()); - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + let mut subscribers = self.subscribers.lock(); log::trace!(target: LOG_TARGET, "{}: notification received from {peer:?}", self.protocol); subscribers.retain(|subscriber| { @@ -483,7 +548,7 @@ impl ProtocolHandle { peer: PeerId, sink: NotificationsSink, ) -> Result<(), ()> { - let mut subscribers = self.subscribers.lock().map_err(|_| ())?; + let mut subscribers = self.subscribers.lock(); log::trace!( target: LOG_TARGET, diff --git a/client/network/src/protocol/notifications/service/tests.rs b/client/network/src/protocol/notifications/service/tests.rs index 1b7b2a4bb8922..c2df4d2224d4e 100644 --- a/client/network/src/protocol/notifications/service/tests.rs +++ b/client/network/src/protocol/notifications/service/tests.rs @@ -21,6 +21,8 @@ use crate::protocol::notifications::handler::{ NotificationsSinkMessage, ASYNC_NOTIFICATIONS_BUFFER_SIZE, }; +use std::future::Future; + #[tokio::test] async fn validate_and_accept_substream() { let (proto, mut notif) = notification_service("/proto/1".into(), None); @@ -620,12 +622,11 @@ async fn sending_notifications_using_notifications_sink_works() { } // get a copy of the notification sink and send a synchronous notification using. - let sink = notif.notification_sink(&peer_id).unwrap(); + let sink = notif.message_sink(&peer_id).unwrap(); sink.send_sync_notification(vec![1, 3, 3, 6]); // send an asynchronous notification using the acquired notifications sink. - let sender = sink.reserve_notification().await.unwrap(); - sender.send(vec![1, 3, 3, 7]).unwrap(); + let sender = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); assert_eq!( sync_rx.next().await, @@ -653,7 +654,7 @@ async fn sending_notifications_using_notifications_sink_works() { #[test] fn try_to_get_notifications_sink_for_non_existent_peer() { let (_proto, notif) = notification_service("/proto/1".into(), None); - assert!(notif.notification_sink(&PeerId::random()).is_none()); + assert!(notif.message_sink(&PeerId::random()).is_none()); } #[tokio::test] @@ -698,12 +699,11 @@ async fn notification_sink_replaced() { } // get a copy of the notification sink and send a synchronous notification using. - let sink = notif.notification_sink(&peer_id).unwrap(); + let sink = notif.message_sink(&peer_id).unwrap(); sink.send_sync_notification(vec![1, 3, 3, 6]); // send an asynchronous notification using the acquired notifications sink. - let sender = sink.reserve_notification().await.unwrap(); - sender.send(vec![1, 3, 3, 7]).unwrap(); + let sender = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); assert_eq!( sync_rx.next().await, @@ -733,16 +733,15 @@ async fn notification_sink_replaced() { let (new_sink, mut new_async_rx, mut new_sync_rx) = NotificationsSink::new(PeerId::random()); handle.report_notification_sink_replaced(peer_id, new_sink).unwrap(); - let new_received_sink = - if let Some(NotificationEvent::NotificationSinkReplaced { peer: _, sink }) = - notif.next_event().await - { - drop(sync_rx); - drop(async_rx); - sink - } else { - panic!("invalid event received"); - }; + // drop the old sinks and poll `notif` once to register the sink replacement + drop(sync_rx); + drop(async_rx); + + futures::future::poll_fn(|cx| { + let _ = std::pin::Pin::new(&mut notif.next_event()).poll(cx); + std::task::Poll::Ready(()) + }) + .await; // verify that using the `NotificationService` API automatically results in using the correct // sink @@ -758,21 +757,12 @@ async fn notification_sink_replaced() { Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }), ); - // try to use the old sink to send a synchronous notification but nothing is obviously received - // and no error is reported as defined in the documentation + // now send two notifications using the acquired message sink and verify that + // it also upated t sink.send_sync_notification(vec![1, 3, 3, 6]); - assert!(new_sync_rx.try_next().is_err()); - - // try to use the old sink to send an asynchronous notification but this time an error is - // returned as the connection has closed. - assert!(sink.reserve_notification().await.is_err()); - - // now send two notifications using the new sink - new_received_sink.send_sync_notification(vec![1, 3, 3, 6]); // send an asynchronous notification using the acquired notifications sink. - let sender = new_received_sink.reserve_notification().await.unwrap(); - sender.send(vec![1, 3, 3, 7]).unwrap(); + let sender = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); assert_eq!( new_sync_rx.next().await, diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 54bf8d460869f..62146cdf41def 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -27,7 +27,7 @@ use crate::{ request_responses::{IfDisconnected, RequestFailure}, service::signature::Signature, types::ProtocolName, - NotificationsSink, ReputationChange, + ReputationChange, }; use futures::{channel::oneshot, Stream}; @@ -644,6 +644,8 @@ pub enum NotificationEvent { /// Negotiated fallback. negotiated_fallback: Option, + // /// Message service associated with the peer. + // message_service: Box, }, - - /// Notification sink was replaced. - NotificationSinkReplaced { - /// Peer ID. - peer: PeerId, - - /// Notification sink. - sink: NotificationsSink, - }, } /// Notification service @@ -748,6 +741,28 @@ pub trait NotificationService: Debug + Send { /// Get protocol name of the `NotificationService`. fn protocol(&self) -> &ProtocolName; - /// Get notification sink of the peer. - fn notification_sink(&self, peer: &PeerId) -> Option; + /// Get message sink of the peer. + fn message_sink(&self, peer: &PeerId) -> Option>; +} + +/// Message sink for peers. +/// +/// If protocol cannot use [`NotificationService`] to send notifications to peer and requires, +/// e.g., notifications to be sent in another task. the protocol may acquire a [`MessageSink`] +/// object for each peer by calling [`NotificationService::notification_sink()`]. Calling this +/// function returns an object which allows the protocol to send notifications to the remote peer. +/// +/// Use of this API is discouraged as it's not as performant as sending notifications through +/// [`NotificationService`] due to synchronization required to keep the underlying notification +/// sink up to date with possible sink replacement events. +#[async_trait::async_trait] +pub trait MessageSink: Send { + /// Send synchronous `notification` to the peer associated with this [`MessageService`]. + fn send_sync_notification(&self, notification: Vec); + + /// Send an asynchronous `notification` to to the peer associated with this [`MessageService`], + /// allowing sender to exercise backpressure. + /// + /// Returns an error if the peer does not exist. + async fn send_async_notification(&self, notification: Vec) -> Result<(), error::Error>; } diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index c2762ef627d0d..43d112886eaf7 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -827,7 +827,6 @@ where self.process_block_announce_validation_result(res) } }, - NotificationEvent::NotificationSinkReplaced { .. } => {}, } } diff --git a/client/network/transactions/src/lib.rs b/client/network/transactions/src/lib.rs index ac132a2395e23..faeca6c3ec850 100644 --- a/client/network/transactions/src/lib.rs +++ b/client/network/transactions/src/lib.rs @@ -349,7 +349,6 @@ where warn!(target: "sub-libp2p", "Failed to decode transactions list"); } }, - NotificationEvent::NotificationSinkReplaced { .. } => {}, } } From 258fd5a3ac0fef8720cedc6dc714fec35c7a58c4 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Mon, 24 Jul 2023 17:36:08 +0300 Subject: [PATCH 27/43] Minor fixes --- .../src/protocol/notifications/service/mod.rs | 24 +++++++++-------- client/network/src/service/traits.rs | 26 ++++++++++++------- client/network/sync/src/engine.rs | 3 --- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index 5a43414118a70..04c70e1471e81 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -47,6 +47,9 @@ mod tests; /// Logging target for the file. const LOG_TARGET: &str = "notification-service"; +/// Default command queue size. +const COMMAND_QUEUE_SIZE: usize = 64; + /// Type representing subscribers of a notification protocol. type Subscribers = Arc>>>; @@ -156,7 +159,7 @@ pub enum NotificationCommand { /// if it wishes to send notifications through `NotificationsSink` directly. /// /// The distributable `NoticationsSink` is wrapped in an `Arc>` to allow -/// `NotificationsService` to swap underlying sink in case the sink is replaced. +/// `NotificationsService` to swap the underlying sink in case it's replaced. #[derive(Debug, Clone)] struct PeerContext { /// Sink for sending notificaitons. @@ -210,7 +213,7 @@ impl NotificationService for NotificationHandle { /// Instruct `Notifications` to close substream for `peer`. async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { - todo!("support for closing substreams not implemented yet"); + todo!("support for closing substreams not implemented yet, call `NetworkService::disconnect_peer()` instead"); } /// Send synchronous `notification` to `peer`. @@ -221,13 +224,13 @@ impl NotificationService for NotificationHandle { ) -> Result<(), error::Error> { log::trace!(target: LOG_TARGET, "{}: send sync notification to {peer:?}", self.protocol); - self.peers - .get(&peer) - // TODO(aaro): check what the current implementation does in case the peer doesn't exist - .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? - .sink - .send_sync_notification(notification); - Ok(()) + match self.peers.get(&peer) { + Some(info) => { + let _ = info.sink.send_sync_notification(notification); + Ok(()) + }, + None => Ok(()), + } } /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. @@ -240,7 +243,6 @@ impl NotificationService for NotificationHandle { self.peers .get(&peer) - // TODO(aaro): check what the current implementation does in case the peer doesn't exist .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? .sink .reserve_notification() @@ -581,7 +583,7 @@ pub fn notification_service( protocol: ProtocolName, metrics: Option, ) -> (ProtocolHandlePair, Box) { - let (cmd_tx, cmd_rx) = mpsc::channel(64); // TODO: zzz + let (cmd_tx, cmd_rx) = mpsc::channel(COMMAND_QUEUE_SIZE); let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); let subscribers = Arc::new(Mutex::new(vec![event_tx])); let metrics = metrics.map_or(None, |registry| metrics::register(®istry).ok()); diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 0a1804f18be00..0624d5a4ed533 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -666,8 +666,8 @@ pub enum NotificationEvent { /// Notification service /// -/// Defines traits that specify the behavior both protocol implementations and `Notifications` can -/// expect from each other. +/// Defines behaviors that both the protocol implementations and `Notifications` can expect from +/// each other. /// /// `Notifications` can send two different kinds of information to protocol: /// * substream-related information @@ -676,7 +676,7 @@ pub enum NotificationEvent { /// When an unvalidated, inbound substream is received by `Notifications`, it sends the inbound /// stream information (peer ID, handshake) to protocol for validation. Protocol must then verify /// that the handshake is valid (and in the future that it has a slot it can allocate for the peer) -/// and then report back `ValidationResult` which is either `Accept` or `Reject`. +/// and then report back the `ValidationResult` which is either `Accept` or `Reject`. /// /// After the validation result has been received by `Notifications`, it prepares the /// substream for communication by initializing the necessary sinks and emits @@ -687,13 +687,13 @@ pub enum NotificationEvent { /// * synchronous sending ([`ProtocolNotificationService::send_sync_notification()`]) /// * asynchronous sending ([`ProtocolNotificationService::send_async_notification()`]) /// -/// The former is used by protocols that don't have the implementation ready for exercising -/// backpressure and the latter for the protocols that can do it. +/// The former is used by protocols are not ready to exercise backpressure and the latter for the +/// protocols that can do it. /// /// Both local and remote peer can close the substream at any time. Local peer can do so by calling /// [`ProtocolNotificationService::close_substream()`] which instrucs `Notifications` to close the /// substream. Remote closing the substream is indicated to the local peer by receiving -/// `NotificationEvent::SubstreamClosed` event +/// [`NotificationEvent::SubstreamClosed`] event /// /// In case the protocol must update its handshake while it's operating (such as updating the best /// block information), it can do so by calling [`ProtocolNotificationService::set_handshake()`] @@ -709,9 +709,13 @@ pub trait NotificationService: Debug + Send { /// /// `dial_if_disconnected` informs `Notifications` whether to dial // the peer if there is currently no active connection to it. + // + // NOTE: not offered by the current implementation async fn open_substream(&mut self, peer: PeerId) -> Result<(), ()>; /// Instruct `Notifications` to close substream for `peer`. + // + // NOTE: not offered by the current implementation async fn close_substream(&mut self, peer: PeerId) -> Result<(), ()>; /// Send synchronous `notification` to `peer`. @@ -722,6 +726,8 @@ pub trait NotificationService: Debug + Send { ) -> Result<(), error::Error>; /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + /// + /// Returns an error if the peer doesn't exist. async fn send_async_notification( &self, peer: &PeerId, @@ -747,8 +753,8 @@ pub trait NotificationService: Debug + Send { /// Message sink for peers. /// -/// If protocol cannot use [`NotificationService`] to send notifications to peer and requires, -/// e.g., notifications to be sent in another task. the protocol may acquire a [`MessageSink`] +/// If protocol cannot use [`NotificationService`] to send notifications to peers and requires, +/// e.g., notifications to be sent in another task, the protocol may acquire a [`MessageSink`] /// object for each peer by calling [`NotificationService::notification_sink()`]. Calling this /// function returns an object which allows the protocol to send notifications to the remote peer. /// @@ -757,10 +763,10 @@ pub trait NotificationService: Debug + Send { /// sink up to date with possible sink replacement events. #[async_trait::async_trait] pub trait MessageSink: Send { - /// Send synchronous `notification` to the peer associated with this [`MessageService`]. + /// Send synchronous `notification` to the peer associated with this [`MessageSink`]. fn send_sync_notification(&self, notification: Vec); - /// Send an asynchronous `notification` to to the peer associated with this [`MessageService`], + /// Send an asynchronous `notification` to to the peer associated with this [`MessageSink`], /// allowing sender to exercise backpressure. /// /// Returns an error if the peer does not exist. diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index 77367f8f26c1c..69e07bbcd67ca 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -738,8 +738,6 @@ where match event { NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => { - // TODO(aaro): put peer on probation and wait to receive - // `NotificationStreamOpened` from them let validation_result = self .validate_handshake(&peer, handshake) .map_or(ValidationResult::Reject, |_| ValidationResult::Accept); @@ -840,7 +838,6 @@ where } self.chain_sync.peer_disconnected(&peer); - // TODO(aaro): remove this bookkeeping when `ProtocolController` is ready self.default_peers_set_no_slot_connected_peers.remove(&peer); self.event_streams .retain(|stream| stream.unbounded_send(SyncEvent::PeerDisconnected(peer)).is_ok()); From 1132424c1844c54ab83584664ca9921a397a6081 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 25 Jul 2023 11:22:53 +0300 Subject: [PATCH 28/43] Store peer role in `PeerStore` --- client/network/common/src/role.rs | 2 +- client/network/src/peer_store.rs | 40 +++++++++++++++++++++-- client/network/src/protocol_controller.rs | 3 ++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/client/network/common/src/role.rs b/client/network/common/src/role.rs index a6364daa69829..374a6f89e65c1 100644 --- a/client/network/common/src/role.rs +++ b/client/network/common/src/role.rs @@ -24,7 +24,7 @@ use codec::{self, Encode, EncodeLike, Input, Output}; /// > **Note**: This enum is different from the `Role` enum. The `Role` enum indicates what a /// > node says about itself, while `ObservedRole` is a `Role` merged with the /// > information known locally about that node. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum ObservedRole { /// Full node. Full, diff --git a/client/network/src/peer_store.rs b/client/network/src/peer_store.rs index 59886c335784b..7227c2114fe05 100644 --- a/client/network/src/peer_store.rs +++ b/client/network/src/peer_store.rs @@ -20,7 +20,7 @@ use libp2p::PeerId; use log::trace; use parking_lot::Mutex; use partial_sort::PartialSort; -use sc_network_common::types::ReputationChange; +use sc_network_common::{role::ObservedRole, types::ReputationChange}; use std::{ cmp::{Ord, Ordering, PartialOrd}, collections::{hash_map::Entry, HashMap, HashSet}, @@ -62,9 +62,15 @@ pub trait PeerStoreProvider: Debug + Send { /// Adjust peer reputation. fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange); + /// Set peer role. + fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole); + /// Get peer reputation. fn peer_reputation(&self, peer_id: &PeerId) -> i32; + /// Get peer role, if available. + fn peer_role(&self, peer_id: &PeerId) -> Option; + /// Get candidates with highest reputations for initiating outgoing connections. fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec; } @@ -91,10 +97,18 @@ impl PeerStoreProvider for PeerStoreHandle { self.inner.lock().report_peer(peer_id, change) } + fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole) { + self.inner.lock().set_peer_role(peer_id, role) + } + fn peer_reputation(&self, peer_id: &PeerId) -> i32 { self.inner.lock().peer_reputation(peer_id) } + fn peer_role(&self, peer_id: &PeerId) -> Option { + self.inner.lock().peer_role(peer_id) + } + fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { self.inner.lock().outgoing_candidates(count, ignored) } @@ -117,13 +131,19 @@ impl PeerStoreHandle { #[derive(Debug, Clone, Copy)] struct PeerInfo { + /// Reputation of the peer. reputation: i32, + + /// Instant when the peer was last updated. last_updated: Instant, + + /// Role of the peer, if known. + role: Option, } impl Default for PeerInfo { fn default() -> Self { - Self { reputation: 0, last_updated: Instant::now() } + Self { reputation: 0, last_updated: Instant::now(), role: None } } } @@ -237,10 +257,26 @@ impl PeerStoreInner { } } + fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole) { + log::trace!(target: LOG_TARGET, "Set {peer_id} role to {role:?}"); + + match self.peers.get_mut(peer_id) { + Some(info) => { + info.role = Some(role); + }, + None => + log::debug!(target: LOG_TARGET, "Failed to set role for {peer_id}, peer doesn't exist"), + } + } + fn peer_reputation(&self, peer_id: &PeerId) -> i32 { self.peers.get(peer_id).map_or(0, |info| info.reputation) } + fn peer_role(&self, peer_id: &PeerId) -> Option { + self.peers.get(peer_id).map_or(None, |info| info.role) + } + fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { let mut candidates = self .peers diff --git a/client/network/src/protocol_controller.rs b/client/network/src/protocol_controller.rs index 5b421e1485d69..9706f58075fb8 100644 --- a/client/network/src/protocol_controller.rs +++ b/client/network/src/protocol_controller.rs @@ -772,6 +772,7 @@ mod tests { ReputationChange, }; use libp2p::PeerId; + use sc_network_common::role::ObservedRole; use sc_utils::mpsc::{tracing_unbounded, TryRecvError}; use std::collections::HashSet; @@ -783,8 +784,10 @@ mod tests { fn is_banned(&self, peer_id: &PeerId) -> bool; fn register_protocol(&self, protocol_handle: ProtocolHandle); fn report_disconnect(&mut self, peer_id: PeerId); + fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole); fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange); fn peer_reputation(&self, peer_id: &PeerId) -> i32; + fn peer_role(&self, peer_id: &PeerId) -> Option; fn outgoing_candidates<'a>(&self, count: usize, ignored: HashSet<&'a PeerId>) -> Vec; } } From 4b02a3df6d7fd91353060a7eec390a9d4fe0da0d Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 25 Jul 2023 11:26:40 +0300 Subject: [PATCH 29/43] Don't return `Result` for `send_sync_notification()` The original API ignores the case where the peer doesn't exist --- .../src/protocol/notifications/behaviour.rs | 6 ---- .../src/protocol/notifications/service/mod.rs | 14 ++------- .../protocol/notifications/service/tests.rs | 29 ++++++++----------- client/network/src/service/traits.rs | 6 +--- 4 files changed, 16 insertions(+), 39 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 5620f7bc4a91f..5ded038cc7f40 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -512,7 +512,6 @@ impl Notifications { let event = NotificationsOut::CustomProtocolClosed { peer_id: *peer_id, set_id }; self.events.push_back(ToSwarm::GenerateEvent(event)); - // TODO(aaro): error handling? let _ = self.protocol_handles[usize::from(set_id)] .report_substream_closed(*peer_id); } @@ -830,7 +829,6 @@ impl Notifications { let event = NotificationsOut::CustomProtocolClosed { peer_id: entry.key().0, set_id }; self.events.push_back(ToSwarm::GenerateEvent(event)); - // TODO(aaro): error handling? let _ = self.protocol_handles[usize::from(set_id)] .report_substream_closed(entry.key().0); } @@ -1420,7 +1418,6 @@ impl NetworkBehaviour for Notifications { notifications_sink: replacement_sink.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); - // TODO(aaro): check error let _ = self.protocol_handles[usize::from(set_id)] .report_notification_sink_replaced( peer_id, @@ -1437,7 +1434,6 @@ impl NetworkBehaviour for Notifications { set_id, }; self.events.push_back(ToSwarm::GenerateEvent(event)); - // TODO(aaro): error handling? let _ = self.protocol_handles[usize::from(set_id)] .report_substream_closed(peer_id); } @@ -1859,7 +1855,6 @@ impl NetworkBehaviour for Notifications { trace!(target: "sub-libp2p", "External API <= Closed({}, {:?})", peer_id, set_id); let event = NotificationsOut::CustomProtocolClosed { peer_id, set_id }; self.events.push_back(ToSwarm::GenerateEvent(event)); - // TODO(aaro): error handling? let _ = self.protocol_handles[usize::from(set_id)] .report_substream_closed(peer_id); } @@ -2107,7 +2102,6 @@ impl NetworkBehaviour for Notifications { message: message.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); - // TODO(aaro): error handling let _ = self.protocol_handles[protocol_index] .report_notification_received(peer_id, message.to_vec()); } else { diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index 04c70e1471e81..6c920bc5a1c80 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -217,19 +217,11 @@ impl NotificationService for NotificationHandle { } /// Send synchronous `notification` to `peer`. - fn send_sync_notification( - &self, - peer: &PeerId, - notification: Vec, - ) -> Result<(), error::Error> { + fn send_sync_notification(&self, peer: &PeerId, notification: Vec) { log::trace!(target: LOG_TARGET, "{}: send sync notification to {peer:?}", self.protocol); - match self.peers.get(&peer) { - Some(info) => { - let _ = info.sink.send_sync_notification(notification); - Ok(()) - }, - None => Ok(()), + if let Some(info) = self.peers.get(&peer) { + let _ = info.sink.send_sync_notification(notification); } } diff --git a/client/network/src/protocol/notifications/service/tests.rs b/client/network/src/protocol/notifications/service/tests.rs index c2df4d2224d4e..5adffe8f2d13c 100644 --- a/client/network/src/protocol/notifications/service/tests.rs +++ b/client/network/src/protocol/notifications/service/tests.rs @@ -112,7 +112,7 @@ async fn send_sync_notification() { panic!("invalid event received"); } - notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]).unwrap(); + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); assert_eq!( sync_rx.next().await, Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }) @@ -174,13 +174,8 @@ async fn send_sync_notification_to_non_existent_peer() { let (_handle, _stream) = proto.split(); let peer = PeerId::random(); - if let Err(error::Error::PeerDoesntExist(peer_id)) = - notif.send_sync_notification(&peer, vec![1, 3, 3, 7]) - { - assert_eq!(peer, peer_id); - } else { - panic!("invalid error received from `send_sync_notification()`"); - } + // as per the original implementation, the call doesn't fail + notif.send_sync_notification(&peer, vec![1, 3, 3, 7]) } #[tokio::test] @@ -195,7 +190,7 @@ async fn send_async_notification_to_non_existent_peer() { { assert_eq!(peer, peer_id); } else { - panic!("invalid error received from `send_sync_notification()`"); + panic!("invalid error received from `send_async_notification()`"); } } @@ -360,7 +355,7 @@ async fn peer_disconnects_then_sync_notification_is_sent() { drop(sync_rx); // as per documentation, error is not reported but the notification is silently dropped - assert!(notif.send_sync_notification(&peer_id, vec![1, 3, 3, 7]).is_ok()); + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 7]); } #[tokio::test] @@ -560,7 +555,7 @@ async fn cloned_service_opening_substream_sending_and_receiving_notifications_wo for (i, notif) in vec![&mut notif1, &mut notif2, &mut notif3].iter().enumerate() { // send notification from each service and verify peer receives it - notif.send_sync_notification(&peer_id, vec![1, 3, 3, i as u8]).unwrap(); + notif.send_sync_notification(&peer_id, vec![1, 3, 3, i as u8]); assert_eq!( sync_rx.next().await, Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, i as u8] }) @@ -626,7 +621,7 @@ async fn sending_notifications_using_notifications_sink_works() { sink.send_sync_notification(vec![1, 3, 3, 6]); // send an asynchronous notification using the acquired notifications sink. - let sender = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); + let _ = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); assert_eq!( sync_rx.next().await, @@ -638,7 +633,7 @@ async fn sending_notifications_using_notifications_sink_works() { ); // send notifications using the stored notification sink as well. - notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]).unwrap(); + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); assert_eq!( @@ -703,7 +698,7 @@ async fn notification_sink_replaced() { sink.send_sync_notification(vec![1, 3, 3, 6]); // send an asynchronous notification using the acquired notifications sink. - let sender = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); + let _ = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); assert_eq!( sync_rx.next().await, @@ -715,7 +710,7 @@ async fn notification_sink_replaced() { ); // send notifications using the stored notification sink as well. - notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]).unwrap(); + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); assert_eq!( @@ -745,7 +740,7 @@ async fn notification_sink_replaced() { // verify that using the `NotificationService` API automatically results in using the correct // sink - notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]).unwrap(); + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); assert_eq!( @@ -762,7 +757,7 @@ async fn notification_sink_replaced() { sink.send_sync_notification(vec![1, 3, 3, 6]); // send an asynchronous notification using the acquired notifications sink. - let sender = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); + let _ = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); assert_eq!( new_sync_rx.next().await, diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 0624d5a4ed533..7fad8075989b2 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -719,11 +719,7 @@ pub trait NotificationService: Debug + Send { async fn close_substream(&mut self, peer: PeerId) -> Result<(), ()>; /// Send synchronous `notification` to `peer`. - fn send_sync_notification( - &self, - peer: &PeerId, - notification: Vec, - ) -> Result<(), error::Error>; + fn send_sync_notification(&self, peer: &PeerId, notification: Vec); /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. /// From c3ff587240f68667565e422d3ddd47dbed6d0d35 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 25 Jul 2023 13:29:02 +0300 Subject: [PATCH 30/43] Do not pass Prometheus registry to `notification_service()` --- client/network/src/config.rs | 2 +- client/network/src/protocol.rs | 3 ++ .../src/protocol/notifications/behaviour.rs | 8 +++-- .../src/protocol/notifications/service/mod.rs | 34 +++++++------------ .../protocol/notifications/service/tests.rs | 32 ++++++++--------- .../src/protocol/notifications/tests.rs | 3 +- client/network/src/service.rs | 1 + 7 files changed, 41 insertions(+), 42 deletions(-) diff --git a/client/network/src/config.rs b/client/network/src/config.rs index c806c38c2c560..822311400d903 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -500,7 +500,7 @@ impl NonDefaultSetConfig { set_config: SetConfig, ) -> (Self, Box) { let (protocol_handle_pair, notification_service) = - notification_service(protocol_name.clone(), None); // TODO(aaro): fix, needs `FullNetworkConfig` + notification_service(protocol_name.clone()); ( Self { protocol_name, diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 00637accfb6a9..bcd2edb53e2f3 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -36,6 +36,7 @@ use libp2p::{ }; use log::{debug, error, warn}; +use prometheus_endpoint::Registry; use sc_network_common::role::Roles; use sc_utils::mpsc::TracingUnboundedSender; use sp_runtime::traits::Block as BlockT; @@ -103,6 +104,7 @@ impl Protocol { pub fn new( roles: Roles, network_config: &config::NetworkConfiguration, + registry: &Option, notification_protocols: Vec, block_announces_protocol: config::NonDefaultSetConfig, _tx: TracingUnboundedSender>, @@ -167,6 +169,7 @@ impl Protocol { ( Notifications::new( peerset, + registry, // NOTE: Block announcement protocol is still very much hardcoded into // `Protocol`. This protocol must be the first notification protocol given to // `Notifications` diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 5ded038cc7f40..ff9f8bd80e6cf 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -41,6 +41,7 @@ use libp2p::{ }; use log::{debug, error, info, trace, warn}; use parking_lot::RwLock; +use prometheus_endpoint::Registry; use rand::distributions::{Distribution as _, Uniform}; use smallvec::SmallVec; use tokio::sync::oneshot::error::RecvError; @@ -395,6 +396,7 @@ impl Notifications { /// Creates a `CustomProtos`. pub fn new( peerset: crate::peerset::Peerset, + registry: &Option, notif_protocols: impl Iterator, ) -> Self { let (notif_protocols, protocol_handle_pairs): (Vec<_>, Vec<_>) = notif_protocols @@ -416,7 +418,8 @@ impl Notifications { .into_iter() .enumerate() .map(|(set_id, protocol_handle_pair)| { - let (protocol_handle, command_stream) = protocol_handle_pair.split(); + let (mut protocol_handle, command_stream) = protocol_handle_pair.split(); + protocol_handle.set_metrics(registry.clone()); (protocol_handle, (set_id, command_stream)) }) @@ -2312,7 +2315,7 @@ mod tests { Box, ) { let (protocol_handle_pair, notif_service) = - crate::protocol::notifications::service::notification_service("/proto/1".into(), None); + crate::protocol::notifications::service::notification_service("/proto/1".into()); let (peerset, peerset_handle) = { let mut sets = Vec::with_capacity(1); @@ -2330,6 +2333,7 @@ mod tests { ( Notifications::new( peerset, + &None, iter::once(( ProtocolConfig { name: "/foo".into(), diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index 6c920bc5a1c80..51aee5e0e2388 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -186,9 +186,6 @@ pub struct NotificationHandle { /// Connected peers. peers: HashMap, - - /// Prometheus metrics. - metrics: Option, } impl NotificationHandle { @@ -198,9 +195,8 @@ impl NotificationHandle { tx: mpsc::Sender, rx: TracingUnboundedReceiver, subscribers: Arc>>>, - metrics: Option, ) -> Self { - Self { protocol, tx, rx, subscribers, peers: HashMap::new(), metrics } + Self { protocol, tx, rx, subscribers, peers: HashMap::new() } } } @@ -316,7 +312,6 @@ impl NotificationService for NotificationHandle { rx: event_rx, peers: self.peers.clone(), subscribers: self.subscribers.clone(), - metrics: self.metrics.clone(), })) } @@ -345,9 +340,6 @@ pub struct ProtocolHandlePair { // Receiver for notification commands received from the protocol implementation. rx: mpsc::Receiver, - - /// Prometheus metrics. - metrics: Option, } impl ProtocolHandlePair { @@ -356,9 +348,8 @@ impl ProtocolHandlePair { protocol: ProtocolName, subscribers: Subscribers, rx: mpsc::Receiver, - metrics: Option, ) -> Self { - Self { protocol, subscribers, rx, metrics } + Self { protocol, subscribers, rx } } /// Consume `self` and split [`ProtocolHandlePair`] into a handle which allows it to send events @@ -367,7 +358,7 @@ impl ProtocolHandlePair { self, ) -> (ProtocolHandle, Box + Send + Unpin>) { ( - ProtocolHandle::new(self.protocol, self.subscribers, self.metrics), + ProtocolHandle::new(self.protocol, self.subscribers), Box::new(ReceiverStream::new(self.rx)), ) } @@ -391,12 +382,13 @@ pub struct ProtocolHandle { } impl ProtocolHandle { - fn new( - protocol: ProtocolName, - subscribers: Subscribers, - metrics: Option, - ) -> Self { - Self { protocol, subscribers, num_peers: 0usize, metrics } + fn new(protocol: ProtocolName, subscribers: Subscribers) -> Self { + Self { protocol, subscribers, num_peers: 0usize, metrics: None } + } + + /// Set metrics. + pub fn set_metrics(&mut self, metrics: Option) { + self.metrics = metrics.map_or(None, |registry| metrics::register(®istry).ok()); } /// Report to the protocol that a substream has been opened and it must be validated by the @@ -573,15 +565,13 @@ impl ProtocolHandle { /// Handle pair allows `Notifications` and the protocol to communicate with each other directly. pub fn notification_service( protocol: ProtocolName, - metrics: Option, ) -> (ProtocolHandlePair, Box) { let (cmd_tx, cmd_rx) = mpsc::channel(COMMAND_QUEUE_SIZE); let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); let subscribers = Arc::new(Mutex::new(vec![event_tx])); - let metrics = metrics.map_or(None, |registry| metrics::register(®istry).ok()); ( - ProtocolHandlePair::new(protocol.clone(), subscribers.clone(), cmd_rx, metrics.clone()), - Box::new(NotificationHandle::new(protocol.clone(), cmd_tx, event_rx, subscribers, metrics)), + ProtocolHandlePair::new(protocol.clone(), subscribers.clone(), cmd_rx), + Box::new(NotificationHandle::new(protocol.clone(), cmd_tx, event_rx, subscribers)), ) } diff --git a/client/network/src/protocol/notifications/service/tests.rs b/client/network/src/protocol/notifications/service/tests.rs index 5adffe8f2d13c..f8e52c98fe1d9 100644 --- a/client/network/src/protocol/notifications/service/tests.rs +++ b/client/network/src/protocol/notifications/service/tests.rs @@ -25,7 +25,7 @@ use std::future::Future; #[tokio::test] async fn validate_and_accept_substream() { - let (proto, mut notif) = notification_service("/proto/1".into(), None); + let (proto, mut notif) = notification_service("/proto/1".into()); let (handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -46,7 +46,7 @@ async fn validate_and_accept_substream() { #[tokio::test] async fn substream_opened() { - let (proto, mut notif) = notification_service("/proto/1".into(), None); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, _) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); @@ -73,7 +73,7 @@ async fn substream_opened() { #[tokio::test] async fn send_sync_notification() { - let (proto, mut notif) = notification_service("/proto/1".into(), None); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -121,7 +121,7 @@ async fn send_sync_notification() { #[tokio::test] async fn send_async_notification() { - let (proto, mut notif) = notification_service("/proto/1".into(), None); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -169,7 +169,7 @@ async fn send_async_notification() { #[tokio::test] async fn send_sync_notification_to_non_existent_peer() { - let (proto, notif) = notification_service("/proto/1".into(), None); + let (proto, notif) = notification_service("/proto/1".into()); let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); let (_handle, _stream) = proto.split(); let peer = PeerId::random(); @@ -180,7 +180,7 @@ async fn send_sync_notification_to_non_existent_peer() { #[tokio::test] async fn send_async_notification_to_non_existent_peer() { - let (proto, notif) = notification_service("/proto/1".into(), None); + let (proto, notif) = notification_service("/proto/1".into()); let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); let (_handle, _stream) = proto.split(); let peer = PeerId::random(); @@ -196,7 +196,7 @@ async fn send_async_notification_to_non_existent_peer() { #[tokio::test] async fn receive_notification() { - let (proto, mut notif) = notification_service("/proto/1".into(), None); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -250,7 +250,7 @@ async fn receive_notification() { #[tokio::test] async fn backpressure_works() { - let (proto, mut notif) = notification_service("/proto/1".into(), None); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -310,7 +310,7 @@ async fn backpressure_works() { #[tokio::test] async fn peer_disconnects_then_sync_notification_is_sent() { - let (proto, mut notif) = notification_service("/proto/1".into(), None); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, _, sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -360,7 +360,7 @@ async fn peer_disconnects_then_sync_notification_is_sent() { #[tokio::test] async fn peer_disconnects_then_async_notification_is_sent() { - let (proto, mut notif) = notification_service("/proto/1".into(), None); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, async_rx, _) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -415,7 +415,7 @@ async fn peer_disconnects_then_async_notification_is_sent() { #[tokio::test] async fn cloned_service_opening_substream_works() { - let (proto, mut notif1) = notification_service("/proto/1".into(), None); + let (proto, mut notif1) = notification_service("/proto/1".into()); let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); let (handle, _stream) = proto.split(); let mut notif2 = notif1.clone().unwrap(); @@ -455,7 +455,7 @@ async fn cloned_service_opening_substream_works() { #[tokio::test] async fn cloned_service_one_service_rejects_substream() { - let (proto, mut notif1) = notification_service("/proto/1".into(), None); + let (proto, mut notif1) = notification_service("/proto/1".into()); let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); let (handle, _stream) = proto.split(); let mut notif2 = notif1.clone().unwrap(); @@ -494,7 +494,7 @@ async fn cloned_service_one_service_rejects_substream() { #[tokio::test] async fn cloned_service_opening_substream_sending_and_receiving_notifications_work() { - let (proto, mut notif1) = notification_service("/proto/1".into(), None); + let (proto, mut notif1) = notification_service("/proto/1".into()); let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let mut notif2 = notif1.clone().unwrap(); @@ -577,7 +577,7 @@ async fn cloned_service_opening_substream_sending_and_receiving_notifications_wo #[tokio::test] async fn sending_notifications_using_notifications_sink_works() { - let (proto, mut notif) = notification_service("/proto/1".into(), None); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, mut async_rx, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); @@ -648,13 +648,13 @@ async fn sending_notifications_using_notifications_sink_works() { #[test] fn try_to_get_notifications_sink_for_non_existent_peer() { - let (_proto, notif) = notification_service("/proto/1".into(), None); + let (_proto, notif) = notification_service("/proto/1".into()); assert!(notif.message_sink(&PeerId::random()).is_none()); } #[tokio::test] async fn notification_sink_replaced() { - let (proto, mut notif) = notification_service("/proto/1".into(), None); + let (proto, mut notif) = notification_service("/proto/1".into()); let (sink, mut async_rx, mut sync_rx) = NotificationsSink::new(PeerId::random()); let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); diff --git a/client/network/src/protocol/notifications/tests.rs b/client/network/src/protocol/notifications/tests.rs index 70745b17bc1b5..76b303e9f3810 100644 --- a/client/network/src/protocol/notifications/tests.rs +++ b/client/network/src/protocol/notifications/tests.rs @@ -84,11 +84,12 @@ fn build_nodes() -> (Swarm, Swarm) { }], }); let (protocol_handle_pair, _notif_service) = - crate::protocol::notifications::service::notification_service("/foo".into(), None); + crate::protocol::notifications::service::notification_service("/foo".into()); let behaviour = CustomProtoWithAddr { inner: Notifications::new( peerset, + &None, iter::once(( ProtocolConfig { name: "/foo".into(), diff --git a/client/network/src/service.rs b/client/network/src/service.rs index a22a837e31e9c..0cf38c404c4db 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -265,6 +265,7 @@ where let (protocol, peerset_handle, mut known_addresses) = Protocol::new( From::from(¶ms.role), &network_config, + ¶ms.metrics_registry, notification_protocols, params.block_announce_config, params.tx, From b8e2fcc30150a29223d558f4aa057b82c0881d3b Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Thu, 3 Aug 2023 13:13:33 +0300 Subject: [PATCH 31/43] Fix metrics --- client/consensus/grandpa/src/communication/tests.rs | 6 +----- client/network-gossip/src/bridge.rs | 6 +----- client/network/src/protocol/notifications/behaviour.rs | 5 +++-- .../network/src/protocol/notifications/service/metrics.rs | 6 +++--- client/network/src/protocol/notifications/service/mod.rs | 7 +++---- 5 files changed, 11 insertions(+), 19 deletions(-) diff --git a/client/consensus/grandpa/src/communication/tests.rs b/client/consensus/grandpa/src/communication/tests.rs index 1f1f5ea17e958..0026ad6108a0b 100644 --- a/client/consensus/grandpa/src/communication/tests.rs +++ b/client/consensus/grandpa/src/communication/tests.rs @@ -226,11 +226,7 @@ impl NotificationService for TestNotificationService { } /// Send synchronous `notification` to `peer`. - fn send_sync_notification( - &self, - _peer: &PeerId, - _notification: Vec, - ) -> Result<(), sc_network::error::Error> { + fn send_sync_notification(&self, _peer: &PeerId, _notification: Vec) { // TODO: this needs to be implemented unimplemented!(); } diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 11b4effaeb3d8..163829662a616 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -513,11 +513,7 @@ mod tests { } /// Send synchronous `notification` to `peer`. - fn send_sync_notification( - &self, - _peer: &PeerId, - _notification: Vec, - ) -> Result<(), sc_network::error::Error> { + fn send_sync_notification(&self, _peer: &PeerId, _notification: Vec) { unimplemented!(); } diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index ff9f8bd80e6cf..069a0f9838cf5 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -21,7 +21,7 @@ use crate::{ peerset::DropReason, protocol::notifications::{ handler::{self, NotificationsSink, NotifsHandler, NotifsHandlerIn, NotifsHandlerOut}, - service::{NotificationCommand, ProtocolHandle}, + service::{metrics, NotificationCommand, ProtocolHandle}, }, service::traits::ValidationResult, types::ProtocolName, @@ -414,12 +414,13 @@ impl Notifications { .unzip(); assert!(!notif_protocols.is_empty()); + let metrics = registry.as_ref().and_then(|registry| metrics::register(®istry).ok()); let (protocol_handles, command_streams): (Vec<_>, Vec<_>) = protocol_handle_pairs .into_iter() .enumerate() .map(|(set_id, protocol_handle_pair)| { let (mut protocol_handle, command_stream) = protocol_handle_pair.split(); - protocol_handle.set_metrics(registry.clone()); + protocol_handle.set_metrics(metrics.clone()); (protocol_handle, (set_id, command_stream)) }) diff --git a/client/network/src/protocol/notifications/service/metrics.rs b/client/network/src/protocol/notifications/service/metrics.rs index dd5bcb230e7bf..9bc6cc0527d61 100644 --- a/client/network/src/protocol/notifications/service/metrics.rs +++ b/client/network/src/protocol/notifications/service/metrics.rs @@ -38,7 +38,7 @@ impl Metrics { HistogramVec::new( HistogramOpts { common_opts: Opts::new( - "substrate_sub_libp2p_notifications_sizes", + "substrate_sub_libp2p_notification_service_total_sizes", "Sizes of the notifications send to and received from all nodes", ), buckets: prometheus::exponential_buckets(64.0, 4.0, 8) @@ -51,7 +51,7 @@ impl Metrics { notifications_streams_closed_total: prometheus::register( CounterVec::new( Opts::new( - "substrate_sub_libp2p_notifications_streams_closed_total", + "substrate_sub_libp2p_notification_service_streams_closed_total", "Total number of notification substreams that have been closed", ), &["protocol"], @@ -61,7 +61,7 @@ impl Metrics { notifications_streams_opened_total: prometheus::register( CounterVec::new( Opts::new( - "substrate_sub_libp2p_notifications_streams_opened_total", + "substrate_sub_libp2p_notification_service_streams_opened_total", "Total number of notification substreams that have been opened", ), &["protocol"], diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index 51aee5e0e2388..8f2bc59b6a602 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -31,7 +31,6 @@ use futures::{ }; use libp2p::PeerId; use parking_lot::Mutex; -use prometheus_endpoint::Registry; use tokio::sync::{mpsc, oneshot}; use tokio_stream::wrappers::ReceiverStream; @@ -40,7 +39,7 @@ use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnbound use std::{collections::HashMap, fmt::Debug, sync::Arc}; -mod metrics; +pub(crate) mod metrics; #[cfg(test)] mod tests; @@ -387,8 +386,8 @@ impl ProtocolHandle { } /// Set metrics. - pub fn set_metrics(&mut self, metrics: Option) { - self.metrics = metrics.map_or(None, |registry| metrics::register(®istry).ok()); + pub fn set_metrics(&mut self, metrics: Option) { + self.metrics = metrics; } /// Report to the protocol that a substream has been opened and it must be validated by the From aad966a9d2106d4fd86f1d255b0b43cea124c29b Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Fri, 4 Aug 2023 14:20:32 +0300 Subject: [PATCH 32/43] Rework peer role detection Previously the peer role was decoded in `Protocol` but now that the protocol is directly communicating with `Notifications`, the handshake decoding is no longer done on behalf of the protocol. Introduce a new function `NetworkPeers::peer_role()` which allows the protocol to fetch the peer role. The function takes the received handshake (which should contain the role) and in case the handshake doesn't contain the protocol, it's fetched from `PeerStore`. If the peer role cannot be determined, `None` is returned, indicating to the protocol that it should discard the peer. --- Cargo.lock | 1 + .../grandpa/src/communication/tests.rs | 23 +++---- client/network-gossip/Cargo.toml | 1 + client/network-gossip/src/bridge.rs | 27 ++++++--- client/network-gossip/src/state_machine.rs | 4 ++ client/network/common/src/role.rs | 12 ++++ client/network/src/config.rs | 2 + client/network/src/lib.rs | 2 +- client/network/src/mock.rs | 2 +- client/network/src/protocol.rs | 4 +- .../src/protocol/notifications/behaviour.rs | 4 -- .../src/protocol/notifications/service/mod.rs | 13 +--- .../protocol/notifications/service/tests.rs | 60 ++++--------------- client/network/src/service.rs | 19 +++++- client/network/src/service/traits.rs | 15 ++++- client/network/statement/src/lib.rs | 7 ++- client/network/sync/src/engine.rs | 7 +++ client/network/sync/src/service/mock.rs | 2 + client/network/transactions/src/lib.rs | 7 ++- client/offchain/src/api.rs | 6 +- client/offchain/src/lib.rs | 8 ++- client/service/src/builder.rs | 3 +- 22 files changed, 134 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f305842150af..2c8f9addadedd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9270,6 +9270,7 @@ dependencies = [ "libp2p-identity", "log", "multiaddr", + "parity-scale-codec", "quickcheck", "sc-network", "sc-network-common", diff --git a/client/consensus/grandpa/src/communication/tests.rs b/client/consensus/grandpa/src/communication/tests.rs index 53b2e68e4e4b6..cd6d51d78d702 100644 --- a/client/consensus/grandpa/src/communication/tests.rs +++ b/client/consensus/grandpa/src/communication/tests.rs @@ -24,7 +24,7 @@ use super::{ }; use crate::{communication::grandpa_protocol_name, environment::SharedVoterSetState}; use futures::prelude::*; -use parity_scale_codec::Encode; +use parity_scale_codec::{DecodeAll, Encode}; use sc_network::{ config::{MultiaddrWithPeerId, Role}, event::Event as NetworkEvent, @@ -32,10 +32,10 @@ use sc_network::{ types::ProtocolName, Multiaddr, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, NetworkSyncForkRequest, NotificationSenderError, NotificationSenderT as NotificationSender, - NotificationsSink, PeerId, ReputationChange, + PeerId, ReputationChange, }; use sc_network_common::{ - role::ObservedRole, + role::{ObservedRole, Roles}, sync::{SyncEvent as SyncStreamEvent, SyncEventStream}, }; use sc_network_gossip::Validator; @@ -126,6 +126,12 @@ impl NetworkPeers for TestNetwork { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, handshake: Vec) -> Option { + Roles::decode_all(&mut &handshake[..]) + .ok() + .and_then(|role| Some(ObservedRole::from(role))) + } } impl NetworkEventStream for TestNetwork { @@ -264,7 +270,7 @@ impl NotificationService for TestNotificationService { unimplemented!(); } - fn message_sink(&self, peer: &PeerId) -> Option> { + fn message_sink(&self, _peer: &PeerId) -> Option> { unimplemented!(); } } @@ -466,9 +472,8 @@ fn good_commit_leads_to_relay() { let _ = tester.notification_tx.unbounded_send( NotificationEvent::NotificationStreamOpened { peer: sender_id, - role: ObservedRole::Full, negotiated_fallback: None, - handshake: vec![1, 3, 3, 7], + handshake: Roles::FULL.encode(), }, ); let _ = tester.notification_tx.unbounded_send( @@ -483,9 +488,8 @@ fn good_commit_leads_to_relay() { let _ = tester.notification_tx.unbounded_send( NotificationEvent::NotificationStreamOpened { peer: receiver_id, - role: ObservedRole::Full, negotiated_fallback: None, - handshake: vec![1, 3, 3, 7], + handshake: Roles::FULL.encode(), }, ); @@ -617,9 +621,8 @@ fn bad_commit_leads_to_report() { let _ = tester.notification_tx.unbounded_send( NotificationEvent::NotificationStreamOpened { peer: sender_id, - role: ObservedRole::Full, negotiated_fallback: None, - handshake: vec![1, 3, 3, 7], + handshake: Roles::FULL.encode(), }, ); let _ = tester.notification_tx.unbounded_send( diff --git a/client/network-gossip/Cargo.toml b/client/network-gossip/Cargo.toml index 2e2f5d67a0568..e69d8e56e99f5 100644 --- a/client/network-gossip/Cargo.toml +++ b/client/network-gossip/Cargo.toml @@ -32,3 +32,4 @@ async-trait = "0.1.57" tokio = "1.22.0" quickcheck = { version = "1.0.3", default-features = false } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +codec = { package = "parity-scale-codec", version = "3.6.1", features = ["derive"] } diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 09637b4bb1756..305bc7b2fc2e0 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -208,7 +208,15 @@ impl Future for GossipEngine { ); let _ = result_tx.send(ValidationResult::Accept); }, - NotificationEvent::NotificationStreamOpened { peer, role, .. } => { + NotificationEvent::NotificationStreamOpened { + peer, handshake, .. + } => { + log::debug!(target: "gossip", "handshake {handshake:?}"); + let Some(role) = this.network.peer_role(peer, handshake) else { + log::debug!(target: "gossip", "role for {peer} couldn't be determined"); + continue + }; + this.state_machine.new_peer(&mut *this.network, peer, role); }, NotificationEvent::NotificationStreamClosed { peer } => { @@ -326,6 +334,7 @@ impl futures::future::FusedFuture for GossipEngine { mod tests { use super::*; use crate::{ValidationResult, ValidatorContext}; + use codec::{DecodeAll, Encode}; use futures::{ channel::mpsc::{unbounded, UnboundedReceiver, UnboundedSender}, executor::{block_on, block_on_stream}, @@ -338,7 +347,7 @@ mod tests { service::traits::{MessageSink, NotificationEvent}, Event, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, NotificationSenderError, NotificationSenderT as NotificationSender, NotificationService, - NotificationsSink, + NotificationsSink, Roles, }; use sc_network_common::{role::ObservedRole, sync::SyncEventStream}; use sp_runtime::{ @@ -419,6 +428,12 @@ mod tests { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, handshake: Vec) -> Option { + Roles::decode_all(&mut &handshake[..]) + .ok() + .and_then(|role| Some(ObservedRole::from(role))) + } } impl NetworkEventStream for TestNetwork { @@ -548,7 +563,7 @@ mod tests { unimplemented!(); } - fn message_sink(&self, peer: &PeerId) -> Option> { + fn message_sink(&self, _peer: &PeerId) -> Option> { unimplemented!(); } } @@ -620,9 +635,8 @@ mod tests { // Register the remote peer. tx.send(NotificationEvent::NotificationStreamOpened { peer: remote_peer, - role: ObservedRole::Authority, negotiated_fallback: None, - handshake: vec![1, 3, 3, 7], + handshake: Roles::FULL.encode(), }) .await .unwrap(); @@ -783,9 +797,8 @@ mod tests { // Register the remote peer. tx.start_send(NotificationEvent::NotificationStreamOpened { peer: remote_peer, - role: ObservedRole::Authority, negotiated_fallback: None, - handshake: vec![1, 3, 3, 7], + handshake: Roles::FULL.encode(), }) .unwrap(); diff --git a/client/network-gossip/src/state_machine.rs b/client/network-gossip/src/state_machine.rs index 39c0a78e0d350..50ad28228a5bb 100644 --- a/client/network-gossip/src/state_machine.rs +++ b/client/network-gossip/src/state_machine.rs @@ -651,6 +651,10 @@ mod tests { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, _handshake: Vec) -> Option { + None + } } impl NetworkEventStream for NoOpNetwork { diff --git a/client/network/common/src/role.rs b/client/network/common/src/role.rs index 374a6f89e65c1..487ad7a5107fa 100644 --- a/client/network/common/src/role.rs +++ b/client/network/common/src/role.rs @@ -41,6 +41,18 @@ impl ObservedRole { } } +impl From for ObservedRole { + fn from(roles: Roles) -> Self { + if roles.is_authority() { + ObservedRole::Authority + } else if roles.is_full() { + ObservedRole::Full + } else { + ObservedRole::Light + } + } +} + /// Role of the local node. #[derive(Debug, Clone)] pub enum Role { diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 00bb031ca87b3..9302e665d7859 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -744,9 +744,11 @@ pub struct Params { /// Registry for recording prometheus metrics to. pub metrics_registry: Option, + // TODO(aaro): remove /// Block announce protocol configuration pub block_announce_config: NonDefaultSetConfig, + // TODO(aaro): remove /// TX channel for direct communication with `SyncingEngine` and `Protocol`. pub tx: TracingUnboundedSender>, } diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index 5cc97f0855261..41729f1f015cc 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -267,7 +267,7 @@ pub use event::{DhtEvent, Event, SyncEvent}; pub use libp2p::{multiaddr, Multiaddr, PeerId}; pub use request_responses::{Config, IfDisconnected, RequestFailure}; pub use sc_network_common::{ - role::ObservedRole, + role::{ObservedRole, Roles}, sync::{ warp::{WarpSyncPhase, WarpSyncProgress}, ExtendedPeerInfo, StateDownloadProgress, SyncEventStream, SyncState, SyncStatusProvider, diff --git a/client/network/src/mock.rs b/client/network/src/mock.rs index 5c82da8ba83dd..534b811897071 100644 --- a/client/network/src/mock.rs +++ b/client/network/src/mock.rs @@ -54,7 +54,7 @@ impl PeerStoreProvider for MockPeerStore { None } - fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole) { + fn set_peer_role(&mut self, _peer_id: &PeerId, _role: ObservedRole) { unimplemented!(); } diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index c04808f948a2e..6dd160b6d576d 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -100,14 +100,14 @@ impl Protocol { /// Create a new instance. pub fn new( roles: Roles, - network_config: &config::NetworkConfiguration, + _network_config: &config::NetworkConfiguration, registry: &Option, notification_protocols: Vec, block_announces_protocol: config::NonDefaultSetConfig, peer_store_handle: PeerStoreHandle, protocol_controller_handles: Vec, from_protocol_controllers: TracingUnboundedReceiver, - tx: TracingUnboundedSender>, + _tx: TracingUnboundedSender>, ) -> error::Result { let (behaviour, notification_protocols) = { let installed_protocols = iter::once(block_announces_protocol.protocol_name().clone()) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 4f02fe3007797..9b4b9b7d538e6 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -48,8 +48,6 @@ use smallvec::SmallVec; use tokio::sync::oneshot::error::RecvError; use tokio_stream::StreamMap; -use sc_network_common::role::ObservedRole; - use std::{ cmp, collections::{hash_map::Entry, VecDeque}, @@ -1945,8 +1943,6 @@ impl NetworkBehaviour for Notifications { .report_substream_opened( peer_id, received_handshake, - ObservedRole::Full, /* TODO(aaro): this needs to be - * queried from `Peerstore` */ negotiated_fallback, notifications_sink.clone(), ); diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index 8f2bc59b6a602..330ad53ccb142 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -34,7 +34,6 @@ use parking_lot::Mutex; use tokio::sync::{mpsc, oneshot}; use tokio_stream::wrappers::ReceiverStream; -use sc_network_common::role::ObservedRole; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use std::{collections::HashMap, fmt::Debug, sync::Arc}; @@ -102,9 +101,6 @@ enum InnerNotificationEvent { /// Received handshake. handshake: Vec, - /// Role of the peer. - role: ObservedRole, - /// Negotiated fallback. negotiated_fallback: Option, @@ -259,7 +255,6 @@ impl NotificationService for NotificationHandle { InnerNotificationEvent::NotificationStreamOpened { peer, handshake, - role, negotiated_fallback, sink, } => { @@ -270,7 +265,6 @@ impl NotificationService for NotificationHandle { return Some(NotificationEvent::NotificationStreamOpened { peer, handshake, - role, negotiated_fallback, }) }, @@ -281,15 +275,12 @@ impl NotificationService for NotificationHandle { InnerNotificationEvent::NotificationReceived { peer, notification } => return Some(NotificationEvent::NotificationReceived { peer, notification }), InnerNotificationEvent::NotificationSinkReplaced { peer, sink } => { - println!("replace sink"); - match self.peers.get_mut(&peer) { None => log::error!( "{}: notification sink replaced for {peer} but peer does not exist", self.protocol ), Some(context) => { - println!("done"); context.sink = sink.clone(); *context.shared_sink.lock() = sink.clone(); }, @@ -381,6 +372,7 @@ pub struct ProtocolHandle { } impl ProtocolHandle { + /// Create new [`ProtocolHandle`]. fn new(protocol: ProtocolName, subscribers: Subscribers) -> Self { Self { protocol, subscribers, num_peers: 0usize, metrics: None } } @@ -462,7 +454,6 @@ impl ProtocolHandle { &mut self, peer: PeerId, handshake: Vec, - role: ObservedRole, negotiated_fallback: Option, sink: NotificationsSink, ) -> Result<(), ()> { @@ -476,7 +467,6 @@ impl ProtocolHandle { .unbounded_send(InnerNotificationEvent::NotificationStreamOpened { peer, handshake: handshake.clone(), - role: role.clone(), negotiated_fallback: negotiated_fallback.clone(), sink: sink.clone(), }) @@ -554,6 +544,7 @@ impl ProtocolHandle { } /// Get the number of connected peers. + // TODO(aaro): remove if possible pub fn num_peers(&self) -> usize { self.num_peers } diff --git a/client/network/src/protocol/notifications/service/tests.rs b/client/network/src/protocol/notifications/service/tests.rs index f8e52c98fe1d9..36b3280b59d16 100644 --- a/client/network/src/protocol/notifications/service/tests.rs +++ b/client/network/src/protocol/notifications/service/tests.rs @@ -51,20 +51,16 @@ async fn substream_opened() { let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); + handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, - role, negotiated_fallback, handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); @@ -93,20 +89,16 @@ async fn send_sync_notification() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); + handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, - role, negotiated_fallback, handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); @@ -141,20 +133,16 @@ async fn send_async_notification() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); + handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, - role, negotiated_fallback, handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); @@ -216,20 +204,16 @@ async fn receive_notification() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); + handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, - role, negotiated_fallback, handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); @@ -270,20 +254,16 @@ async fn backpressure_works() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); + handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, - role, negotiated_fallback, handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); @@ -330,20 +310,16 @@ async fn peer_disconnects_then_sync_notification_is_sent() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); + handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, - role, negotiated_fallback, handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); @@ -380,20 +356,16 @@ async fn peer_disconnects_then_async_notification_is_sent() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); + handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, - role, negotiated_fallback, handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); @@ -519,21 +491,17 @@ async fn cloned_service_opening_substream_sending_and_receiving_notifications_wo assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that then notification stream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); + handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); for notif in vec![&mut notif1, &mut notif2, &mut notif3] { if let Some(NotificationEvent::NotificationStreamOpened { peer, - role, negotiated_fallback, handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); @@ -597,20 +565,16 @@ async fn sending_notifications_using_notifications_sink_works() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); + handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, - role, negotiated_fallback, handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); @@ -674,20 +638,16 @@ async fn notification_sink_replaced() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle - .report_substream_opened(peer_id, vec![1, 3, 3, 7], ObservedRole::Full, None, sink) - .unwrap(); + handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, - role, negotiated_fallback, handshake, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); - assert_eq!(role, ObservedRole::Full); assert_eq!(handshake, vec![1, 3, 3, 7]); } else { panic!("invalid event received"); diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 6d78aa1bc45ac..7ad7302d16c27 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -54,6 +54,7 @@ use crate::{ ReputationChange, }; +use codec::DecodeAll; use either::Either; use futures::{channel::oneshot, prelude::*}; use libp2p::{ @@ -72,7 +73,10 @@ use log::{debug, error, info, trace, warn}; use metrics::{Histogram, HistogramVec, MetricSources, Metrics}; use parking_lot::Mutex; -use sc_network_common::ExHashT; +use sc_network_common::{ + role::{ObservedRole, Roles}, + ExHashT, +}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_runtime::traits::Block as BlockT; @@ -130,6 +134,8 @@ pub struct NetworkService { protocol_handles: Vec, /// Shortcut to sync protocol handle (`protocol_handles[0]`). sync_protocol_handle: protocol_controller::ProtocolHandle, + /// Handle to `PeerStore`. + peer_store_handle: PeerStoreHandle, /// Marker to pin the `H` generic. Serves no purpose except to not break backwards /// compatibility. _marker: PhantomData, @@ -516,6 +522,7 @@ where notification_protocol_ids, protocol_handles, sync_protocol_handle, + peer_store_handle: params.peer_store.clone(), _marker: PhantomData, _block: Default::default(), }); @@ -979,6 +986,16 @@ where fn sync_num_connected(&self) -> usize { self.num_connected.load(Ordering::Relaxed) } + + fn peer_role(&self, peer_id: PeerId, handshake: Vec) -> Option { + match Roles::decode_all(&mut &handshake[..]) { + Ok(role) => Some(role.into()), + Err(_) => { + log::debug!(target: "sub-libp2p", "handshake doesn't contain peer role: {handshake:?}"); + self.peer_store_handle.peer_role(&peer_id) + }, + } + } } impl NetworkEventStream for NetworkService diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index e13912cd8d4c9..76b46c2a2203e 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -226,6 +226,14 @@ pub trait NetworkPeers { /// Returns the number of peers in the sync peer set we're connected to. fn sync_num_connected(&self) -> usize; + + /// Attempt to get peer role. + /// + /// Right now the peer role is decoded into the received handshake for all protocols + /// (`/block-announces/1` has other information as well). If the handshake cannot be + /// decoded into a role, the role queried from `PeerStore` and if the role is not stored + /// there either, `None` is returned and the peer should be discarded. + fn peer_role(&self, peer_id: PeerId, handshake: Vec) -> Option; } // Manual implementation to avoid extra boxing here @@ -301,6 +309,10 @@ where fn sync_num_connected(&self) -> usize { T::sync_num_connected(self) } + + fn peer_role(&self, peer_id: PeerId, handshake: Vec) -> Option { + T::peer_role(self, peer_id, handshake) + } } /// Provides access to network-level event stream. @@ -651,9 +663,6 @@ pub enum NotificationEvent { /// Received handshake. handshake: Vec, - /// Role of the peer. - role: ObservedRole, - /// Negotiated fallback. negotiated_fallback: Option, // /// Message service associated with the peer. diff --git a/client/network/statement/src/lib.rs b/client/network/statement/src/lib.rs index a66e5fa20c335..06e4993d52d05 100644 --- a/client/network/statement/src/lib.rs +++ b/client/network/statement/src/lib.rs @@ -313,7 +313,12 @@ where NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { let _ = result_tx.send(ValidationResult::Accept); }, - NotificationEvent::NotificationStreamOpened { peer, role, .. } => { + NotificationEvent::NotificationStreamOpened { peer, handshake, .. } => { + let Some(role) = self.network.peer_role(peer, handshake) else { + log::debug!(target: LOG_TARGET, "role for {peer} couldn't be determined"); + return; + }; + let _was_in = self.peers.insert( peer, Peer { diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index 69e07bbcd67ca..cec7e15b69786 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -37,6 +37,7 @@ use sc_client_api::{BlockBackend, HeaderBackend, ProofProvider}; use sc_consensus::import_queue::ImportQueueService; use sc_network::{ config::{FullNetworkConfiguration, NonDefaultSetConfig, ProtocolId}, + peer_store::{PeerStoreHandle, PeerStoreProvider}, service::traits::{NotificationEvent, ValidationResult}, types::ProtocolName, utils::LruHashSet, @@ -266,6 +267,9 @@ pub struct SyncingEngine { /// Stored as an `Option` so once the initial wait has passed, `SyncingEngine` /// can reset the peer timers and continue with the normal eviction process. syncing_started: Option, + + /// Handle to `PeerStore`. + peer_store_handle: PeerStoreHandle, } impl SyncingEngine @@ -294,6 +298,7 @@ where state_request_protocol_name: ProtocolName, warp_sync_protocol_name: Option, _rx: sc_utils::mpsc::TracingUnboundedReceiver>, + peer_store_handle: PeerStoreHandle, ) -> Result<(Self, SyncingService, NonDefaultSetConfig), ClientError> { let mode = net_config.network_config.sync_mode; let max_parallel_downloads = net_config.network_config.max_parallel_downloads; @@ -413,6 +418,7 @@ where notification_service, tick_timeout: Delay::new(TICK_TIMEOUT), syncing_started: None, + peer_store_handle, metrics: if let Some(r) = metrics_registry { match Metrics::register(r, is_major_syncing.clone()) { Ok(metrics) => Some(metrics), @@ -1026,6 +1032,7 @@ where log::debug!(target: LOG_TARGET, "connected {}", who); self.peers.insert(who, peer); + self.peer_store_handle.set_peer_role(&who, status.roles.into()); if no_slot_peer { self.default_peers_set_no_slot_connected_peers.insert(who); diff --git a/client/network/sync/src/service/mock.rs b/client/network/sync/src/service/mock.rs index 885eb1f8da593..47986a71d012a 100644 --- a/client/network/sync/src/service/mock.rs +++ b/client/network/sync/src/service/mock.rs @@ -27,6 +27,7 @@ use sc_network::{ NetworkNotification, NetworkPeers, NetworkRequest, NetworkSyncForkRequest, NotificationSenderError, NotificationSenderT, ReputationChange, }; +use sc_network_common::role::ObservedRole; use sp_runtime::traits::{Block as BlockT, NumberFor}; use std::collections::HashSet; @@ -105,6 +106,7 @@ mockall::mock! { peers: Vec ) -> Result<(), String>; fn sync_num_connected(&self) -> usize; + fn peer_role(&self, peer_id: PeerId, handshake: Vec) -> Option; } #[async_trait::async_trait] diff --git a/client/network/transactions/src/lib.rs b/client/network/transactions/src/lib.rs index 14b16b176ed54..b562f8ff8f340 100644 --- a/client/network/transactions/src/lib.rs +++ b/client/network/transactions/src/lib.rs @@ -324,7 +324,12 @@ where NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { let _ = result_tx.send(ValidationResult::Accept); }, - NotificationEvent::NotificationStreamOpened { peer, role, .. } => { + NotificationEvent::NotificationStreamOpened { peer, handshake, .. } => { + let Some(role) = self.network.peer_role(peer, handshake) else { + log::debug!(target: "sub-libp2p", "role for {peer} couldn't be determined"); + return; + }; + let _was_in = self.peers.insert( peer, Peer { diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs index c7df5784d329e..2901bab2f2677 100644 --- a/client/offchain/src/api.rs +++ b/client/offchain/src/api.rs @@ -223,7 +223,7 @@ mod tests { use sc_client_db::offchain::LocalStorage; use sc_network::{ config::MultiaddrWithPeerId, types::ProtocolName, NetworkPeers, NetworkStateInfo, - ReputationChange, + ObservedRole, ReputationChange, }; use sp_core::offchain::{storage::OffchainDb, DbExternalities, Externalities, StorageKind}; use std::time::SystemTime; @@ -294,6 +294,10 @@ mod tests { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, _handshake: Vec) -> Option { + None + } } impl NetworkStateInfo for TestNetwork { diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index a11ac7d86ecb8..704999aa334c7 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -330,7 +330,9 @@ mod tests { use libp2p::{Multiaddr, PeerId}; use sc_block_builder::BlockBuilderProvider as _; use sc_client_api::Backend as _; - use sc_network::{config::MultiaddrWithPeerId, types::ProtocolName, ReputationChange}; + use sc_network::{ + config::MultiaddrWithPeerId, types::ProtocolName, ObservedRole, ReputationChange, + }; use sc_transaction_pool::BasicPool; use sc_transaction_pool_api::{InPoolTransaction, TransactionPool}; use sp_consensus::BlockOrigin; @@ -422,6 +424,10 @@ mod tests { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, _handshake: Vec) -> Option { + None + } } #[test] diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 9a3dce2bfd51c..1c8e1607e16d8 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -889,7 +889,8 @@ where block_request_protocol_name, state_request_protocol_name, warp_request_protocol_name, - rx, + rx, // TODO(aaro): remove + peer_store_handle.clone(), )?; let sync_service_import_queue = sync_service.clone(); let sync_service = Arc::new(sync_service); From 295a6a7d970ce62996685275fb3027015df06a71 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Fri, 4 Aug 2023 15:55:30 +0300 Subject: [PATCH 33/43] Remove rejected peers from `ProtocolController` --- .../src/protocol/notifications/behaviour.rs | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 9b4b9b7d538e6..95c63fb52c93a 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -998,27 +998,34 @@ impl Notifications { } } - /// Function that is called when the peerset wants us to reject an incoming peer. + /// Function that is called when the protocol wants us to reject an incoming peer. fn protocol_report_reject(&mut self, index: IncomingIndex) { + if let Some((set_id, peer_id)) = self.peerset_report_reject(index) { + self.protocol_controller_handles[usize::from(set_id)].dropped(peer_id) + } + } + + /// Function that is called when the peerset wants us to reject an incoming peer. + fn peerset_report_reject(&mut self, index: IncomingIndex) -> Option<(SetId, PeerId)> { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) } else { error!(target: "sub-libp2p", "PSM => Reject({:?}): Invalid index", index); - return + return None }; if !incoming.alive { trace!(target: "sub-libp2p", "PSM => Reject({:?}, {}, {:?}): Obsolete incoming, \ ignoring", index, incoming.peer_id, incoming.set_id); - return + return None } let state = match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { Some(s) => s, None => { debug_assert!(false); - return + return None }, }; @@ -1031,7 +1038,7 @@ impl Notifications { "PSM => Reject({:?}, {}, {:?}): Ignoring obsolete incoming index, we are already awaiting {:?}.", index, incoming.peer_id, incoming.set_id, incoming_index ); - return + return None } else if index > incoming_index { error!( target: "sub-libp2p", @@ -1039,7 +1046,7 @@ impl Notifications { index, incoming.peer_id, incoming.set_id, incoming_index ); debug_assert!(false); - return + return None } trace!(target: "sub-libp2p", "PSM => Reject({:?}, {}, {:?}): Rejecting connections.", @@ -1063,10 +1070,15 @@ impl Notifications { } *state = PeerState::Disabled { connections, backoff_until }; + Some((incoming.set_id, incoming.peer_id)) + }, + peer => { + error!( + target: "sub-libp2p", + "State mismatch in libp2p: Expected alive incoming. Got {peer:?}.", + ); + None }, - peer => error!(target: "sub-libp2p", - "State mismatch in libp2p: Expected alive incoming. Got {:?}.", - peer), } } } @@ -2128,7 +2140,7 @@ impl NetworkBehaviour for Notifications { self.peerset_report_preaccept(index); }, Poll::Ready(Some(Message::Reject(index))) => { - self.protocol_report_reject(index); + let _ = self.peerset_report_reject(index); }, Poll::Ready(Some(Message::Connect { peer_id, set_id, .. })) => { self.peerset_report_connect(peer_id, set_id); @@ -2175,7 +2187,6 @@ impl NetworkBehaviour for Notifications { self.protocol_report_accept(index); }, Ok(ValidationResult::Reject) => { - // TODO(aaro): remove connection from peerset self.protocol_report_reject(index); }, Err(_) => { From fe8eeedfffaa22739861e18aa4eff9f595c2b771 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Mon, 7 Aug 2023 13:21:41 +0300 Subject: [PATCH 34/43] Fix BEEFY test --- client/consensus/beefy/src/tests.rs | 28 ++++++++++++++++++++++++---- client/network/sync/src/engine.rs | 2 +- client/network/test/src/lib.rs | 13 +++++++------ client/network/test/src/service.rs | 13 +++++++------ 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/client/consensus/beefy/src/tests.rs b/client/consensus/beefy/src/tests.rs index e85f86bea5281..be404862d4e0a 100644 --- a/client/consensus/beefy/src/tests.rs +++ b/client/consensus/beefy/src/tests.rs @@ -1022,9 +1022,7 @@ async fn should_initialize_voter_at_genesis() { assert_eq!(state, persisted_state); } -// TODO(aaro): fix #[tokio::test] -#[ignore] async fn should_initialize_voter_at_custom_genesis() { let keys = &[BeefyKeyring::Alice]; let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap(); @@ -1043,7 +1041,25 @@ async fn should_initialize_voter_at_custom_genesis() { net.peer(0).client().as_client().finalize_block(hashes[8], None).unwrap(); // load persistent state - nothing in DB, should init at genesis - let persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap(); + // + // NOTE: code from `voter_init_setup()` is moved here because the new network event system + // doesn't allow creating a new `GossipEngine` as the notification handle is consumed by the + // first `GossipEngine` + let known_peers = Arc::new(Mutex::new(KnownPeers::new())); + let (gossip_validator, _) = GossipValidator::new(known_peers); + let gossip_validator = Arc::new(gossip_validator); + let mut gossip_engine = sc_network_gossip::GossipEngine::new( + net.peer(0).network_service().clone(), + net.peer(0).sync_service().clone(), + net.peer(0).take_notification_service(&beefy_gossip_proto_name()).unwrap(), + "/beefy/whatever", + gossip_validator, + None, + ); + let (beefy_genesis, best_grandpa) = + wait_for_runtime_pallet(&api, &mut gossip_engine, &mut finality).await.unwrap(); + let persisted_state = + load_or_init_voter_state(&*backend, &api, beefy_genesis, best_grandpa, 1).unwrap(); // Test initialization at session boundary. // verify voter initialized with single session starting at block `custom_pallet_genesis` (7) @@ -1073,7 +1089,11 @@ async fn should_initialize_voter_at_custom_genesis() { net.peer(0).client().as_client().finalize_block(hashes[10], None).unwrap(); // load persistent state - state preset in DB, but with different pallet genesis - let new_persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap(); + // the network state persists and uses the old `GossipEngine` initialized for `peer(0)` + let (beefy_genesis, best_grandpa) = + wait_for_runtime_pallet(&api, &mut gossip_engine, &mut finality).await.unwrap(); + let new_persisted_state = + load_or_init_voter_state(&*backend, &api, beefy_genesis, best_grandpa, 1).unwrap(); // verify voter initialized with single session starting at block `new_pallet_genesis` (10) let sessions = new_persisted_state.voting_oracle().sessions(); diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index cec7e15b69786..e6b4e02976fab 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -816,7 +816,7 @@ where /// Returns a result if the handshake of this peer was indeed accepted. pub fn on_sync_peer_disconnected(&mut self, peer: PeerId) { let Some(info) = self.peers.remove(&peer) else { - log::error!(target: LOG_TARGET, "{peer} does not exist in `SyncingEngine`"); + log::debug!(target: LOG_TARGET, "{peer} does not exist in `SyncingEngine`"); return }; diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index a5811b226f1c7..373d81ee22724 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -895,6 +895,12 @@ where protocol_config }; + let peer_store = PeerStore::new( + network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(), + ); + let peer_store_handle = peer_store.handle(); + self.spawn_task(peer_store.run().boxed()); + let block_announce_validator = config .block_announce_validator .unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator)); @@ -918,6 +924,7 @@ where state_request_protocol_config.name.clone(), Some(warp_protocol_config.name.clone()), rx, + peer_store_handle.clone(), ) .unwrap(); let sync_service_import_queue = Box::new(sync_service.clone()); @@ -939,12 +946,6 @@ where full_net_config.add_notification_protocol(config); } - let peer_store = PeerStore::new( - network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(), - ); - let peer_store_handle = peer_store.handle(); - self.spawn_task(peer_store.run().boxed()); - let genesis_hash = client.hash(Zero::zero()).ok().flatten().expect("Genesis block exists; qed"); let network = NetworkWorker::new(sc_network::config::Params { diff --git a/client/network/test/src/service.rs b/client/network/test/src/service.rs index d712baaa03801..2ddc1154385b3 100644 --- a/client/network/test/src/service.rs +++ b/client/network/test/src/service.rs @@ -178,6 +178,12 @@ impl TestNetworkBuilder { protocol_config }; + let peer_store = PeerStore::new( + network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(), + ); + let peer_store_handle = peer_store.handle(); + tokio::spawn(peer_store.run().boxed()); + let (chain_sync_network_provider, chain_sync_network_handle) = self.chain_sync_network.unwrap_or(NetworkServiceProvider::new()); let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); @@ -196,6 +202,7 @@ impl TestNetworkBuilder { state_request_protocol_config.name.clone(), None, rx, + peer_store_handle.clone(), ) .unwrap(); let mut link = self.link.unwrap_or(Box::new(chain_sync_service.clone())); @@ -225,12 +232,6 @@ impl TestNetworkBuilder { full_net_config.add_request_response_protocol(config); } - let peer_store = PeerStore::new( - network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(), - ); - let peer_store_handle = peer_store.handle(); - tokio::spawn(peer_store.run().boxed()); - let genesis_hash = client.hash(Zero::zero()).ok().flatten().expect("Genesis block exists; qed"); let worker = NetworkWorker::< From ccb3beb21efd9377489a09ab713a0c8ef24ba0ad Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Mon, 7 Aug 2023 18:15:49 +0300 Subject: [PATCH 35/43] Remove dead code --- client/network/src/config.rs | 7 +------ client/network/src/lib.rs | 9 +++++---- client/network/src/protocol.rs | 4 +--- client/network/src/protocol/notifications/service/mod.rs | 1 - client/network/src/service.rs | 2 -- client/network/sync/src/engine.rs | 1 - client/service/src/builder.rs | 3 --- 7 files changed, 7 insertions(+), 20 deletions(-) diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 9302e665d7859..ccee7db14fb68 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -43,7 +43,6 @@ pub use sc_network_common::{ sync::{warp::WarpSyncProvider, SyncMode}, ExHashT, }; -use sc_utils::mpsc::TracingUnboundedSender; use sp_runtime::traits::Block as BlockT; use std::{ @@ -744,13 +743,9 @@ pub struct Params { /// Registry for recording prometheus metrics to. pub metrics_registry: Option, - // TODO(aaro): remove + // TODO(aaro): remove maybe? /// Block announce protocol configuration pub block_announce_config: NonDefaultSetConfig, - - // TODO(aaro): remove - /// TX channel for direct communication with `SyncingEngine` and `Protocol`. - pub tx: TracingUnboundedSender>, } /// Full network configuration. diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index 41729f1f015cc..b3648106734dd 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -277,10 +277,11 @@ pub use sc_network_common::{ pub use service::{ signature::Signature, traits::{ - KademliaKey, NetworkBlock, NetworkDHTProvider, NetworkEventStream, NetworkNotification, - NetworkPeers, NetworkRequest, NetworkSigner, NetworkStateInfo, NetworkStatus, - NetworkStatusProvider, NetworkSyncForkRequest, NotificationSender as NotificationSenderT, - NotificationSenderError, NotificationSenderReady, NotificationService, + KademliaKey, MessageSink, NetworkBlock, NetworkDHTProvider, NetworkEventStream, + NetworkNotification, NetworkPeers, NetworkRequest, NetworkSigner, NetworkStateInfo, + NetworkStatus, NetworkStatusProvider, NetworkSyncForkRequest, + NotificationSender as NotificationSenderT, NotificationSenderError, + NotificationSenderReady, NotificationService, }, DecodingError, Keypair, NetworkService, NetworkWorker, NotificationSender, NotificationsSink, OutboundFailure, PublicKey, diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 6dd160b6d576d..3f2a1d4b309b0 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -38,7 +38,7 @@ use log::{debug, error, warn}; use prometheus_endpoint::Registry; use sc_network_common::role::Roles; -use sc_utils::mpsc::{TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::TracingUnboundedReceiver; use sp_runtime::traits::Block as BlockT; use std::{ @@ -100,14 +100,12 @@ impl Protocol { /// Create a new instance. pub fn new( roles: Roles, - _network_config: &config::NetworkConfiguration, registry: &Option, notification_protocols: Vec, block_announces_protocol: config::NonDefaultSetConfig, peer_store_handle: PeerStoreHandle, protocol_controller_handles: Vec, from_protocol_controllers: TracingUnboundedReceiver, - _tx: TracingUnboundedSender>, ) -> error::Result { let (behaviour, notification_protocols) = { let installed_protocols = iter::once(block_announces_protocol.protocol_name().clone()) diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index 330ad53ccb142..628e68a2e842f 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -544,7 +544,6 @@ impl ProtocolHandle { } /// Get the number of connected peers. - // TODO(aaro): remove if possible pub fn num_peers(&self) -> usize { self.num_peers } diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 7ad7302d16c27..f4421e57d9f78 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -383,14 +383,12 @@ where let protocol = Protocol::new( From::from(¶ms.role), - &network_config, ¶ms.metrics_registry, notification_protocols, params.block_announce_config, params.peer_store.clone(), protocol_handles.clone(), from_protocol_controllers, - params.tx, )?; // Build the swarm. diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index e6b4e02976fab..d4523c5aa238f 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -297,7 +297,6 @@ where block_request_protocol_name: ProtocolName, state_request_protocol_name: ProtocolName, warp_sync_protocol_name: Option, - _rx: sc_utils::mpsc::TracingUnboundedReceiver>, peer_store_handle: PeerStoreHandle, ) -> Result<(Self, SyncingService, NonDefaultSetConfig), ClientError> { let mode = net_config.network_config.sync_mode; diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 1c8e1607e16d8..02c9c9b0cac5d 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -873,7 +873,6 @@ where let peer_store_handle = peer_store.handle(); spawn_handle.spawn("peer-store", Some("networking"), peer_store.run()); - let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); let (engine, sync_service, block_announce_config) = SyncingEngine::new( Roles::from(&config.role), @@ -889,7 +888,6 @@ where block_request_protocol_name, state_request_protocol_name, warp_request_protocol_name, - rx, // TODO(aaro): remove peer_store_handle.clone(), )?; let sync_service_import_queue = sync_service.clone(); @@ -911,7 +909,6 @@ where fork_id: config.chain_spec.fork_id().map(ToOwned::to_owned), metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()), block_announce_config, - tx, }; let has_bootnodes = !network_params.network_config.network_config.boot_nodes.is_empty(); From d38b53b96df0479c1c38859f92430668e92fbe42 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Tue, 8 Aug 2023 16:04:07 +0300 Subject: [PATCH 36/43] Start using `NotificationService` properly in `SyncingEngine` --- client/network/src/config.rs | 2 +- client/network/src/lib.rs | 4 +- client/network/src/protocol.rs | 3 +- client/network/src/protocol/message.rs | 1 + client/network/src/protocol/notifications.rs | 2 +- .../src/protocol/notifications/behaviour.rs | 5 +- .../src/protocol/notifications/handler.rs | 2 +- .../src/protocol/notifications/service/mod.rs | 14 +- .../protocol/notifications/service/tests.rs | 60 +- client/network/src/service.rs | 5 +- client/network/src/service/traits.rs | 24 +- client/network/sync/src/engine.rs | 550 ++++++++++++++---- client/network/test/src/lib.rs | 3 - client/network/test/src/service.rs | 3 - 14 files changed, 529 insertions(+), 149 deletions(-) diff --git a/client/network/src/config.rs b/client/network/src/config.rs index ccee7db14fb68..dfc6ebcd3f28b 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -540,7 +540,7 @@ impl NonDefaultSetConfig { } /// Take `ProtocolHandlePair` from `NonDefaultSetConfig` - pub(crate) fn take_protocol_handle(self) -> ProtocolHandlePair { + pub fn take_protocol_handle(self) -> ProtocolHandlePair { self.protocol_handle_pair } diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index b3648106734dd..2f297f45ccc6d 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -283,8 +283,8 @@ pub use service::{ NotificationSender as NotificationSenderT, NotificationSenderError, NotificationSenderReady, NotificationService, }, - DecodingError, Keypair, NetworkService, NetworkWorker, NotificationSender, NotificationsSink, - OutboundFailure, PublicKey, + DecodingError, Keypair, NetworkService, NetworkWorker, NotificationCommand, NotificationSender, + NotificationsSink, OutboundFailure, ProtocolHandle, PublicKey, }; pub use types::ProtocolName; diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 3f2a1d4b309b0..5c0acf3d4228d 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -52,7 +52,8 @@ use std::{ use notifications::{Notifications, NotificationsOut}; pub use notifications::{ - notification_service, NotificationsSink, NotifsHandlerError, ProtocolHandlePair, Ready, + notification_service, NotificationCommand, NotificationsSink, NotifsHandlerError, + ProtocolHandle, ProtocolHandlePair, Ready, }; mod notifications; diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index 66dca2975375f..247580083f99e 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -29,6 +29,7 @@ use sc_network_common::message::RequestId; use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; /// Type alias for using the message type using block type parameters. +#[allow(unused)] pub type Message = generic::Message< ::Header, ::Hash, diff --git a/client/network/src/protocol/notifications.rs b/client/network/src/protocol/notifications.rs index 1a53df2cdf781..b5b8b697456e8 100644 --- a/client/network/src/protocol/notifications.rs +++ b/client/network/src/protocol/notifications.rs @@ -22,7 +22,7 @@ pub use self::{ behaviour::{Notifications, NotificationsOut, ProtocolConfig}, handler::{NotificationsSink, NotifsHandlerError, Ready}, - service::{notification_service, ProtocolHandlePair}, + service::{notification_service, NotificationCommand, ProtocolHandle, ProtocolHandlePair}, }; mod behaviour; diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 95c63fb52c93a..c3f6f1092a655 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -23,7 +23,7 @@ use crate::{ service::{metrics, NotificationCommand, ProtocolHandle}, }, protocol_controller::{self, IncomingIndex, Message, SetId}, - service::traits::ValidationResult, + service::traits::{Direction, ValidationResult}, types::ProtocolName, }; @@ -1951,9 +1951,12 @@ impl NetworkBehaviour for Notifications { notifications_sink: notifications_sink.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); + let direction = + if inbound { Direction::Inbound } else { Direction::Outbound }; let _ = self.protocol_handles[protocol_index] .report_substream_opened( peer_id, + direction, received_handshake, negotiated_fallback, notifications_sink.clone(), diff --git a/client/network/src/protocol/notifications/handler.rs b/client/network/src/protocol/notifications/handler.rs index 031b38945b521..daaf714c696a3 100644 --- a/client/network/src/protocol/notifications/handler.rs +++ b/client/network/src/protocol/notifications/handler.rs @@ -335,7 +335,7 @@ pub struct NotificationsSink { inner: Arc, } -#[cfg(test)] +// #[cfg(test)] impl NotificationsSink { /// Create new pub fn new( diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index 628e68a2e842f..fdf76594dd1f6 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -21,7 +21,9 @@ use crate::{ error, protocol::notifications::handler::NotificationsSink, - service::traits::{MessageSink, NotificationEvent, NotificationService, ValidationResult}, + service::traits::{ + Direction, MessageSink, NotificationEvent, NotificationService, ValidationResult, + }, types::ProtocolName, }; @@ -98,6 +100,9 @@ enum InnerNotificationEvent { /// Peer ID. peer: PeerId, + /// Direction of the substream. + direction: Direction, + /// Received handshake. handshake: Vec, @@ -136,6 +141,7 @@ enum InnerNotificationEvent { /// Notification commands. /// /// Sent by the installed protocols to `Notifications` to open/close/modify substreams. +#[derive(Debug)] pub enum NotificationCommand { /// Instruct `Notifications` to open a substream to peer. OpenSubstream(PeerId), @@ -236,7 +242,7 @@ impl NotificationService for NotificationHandle { } /// Set handshake for the notification protocol replacing the old handshake. - async fn set_hanshake(&mut self, handshake: Vec) -> Result<(), ()> { + async fn set_handshake(&mut self, handshake: Vec) -> Result<(), ()> { log::trace!(target: LOG_TARGET, "{}: set handshake to {handshake:?}", self.protocol); self.tx.send(NotificationCommand::SetHandshake(handshake)).await.map_err(|_| ()) @@ -256,6 +262,7 @@ impl NotificationService for NotificationHandle { peer, handshake, negotiated_fallback, + direction, sink, } => { self.peers.insert( @@ -265,6 +272,7 @@ impl NotificationService for NotificationHandle { return Some(NotificationEvent::NotificationStreamOpened { peer, handshake, + direction, negotiated_fallback, }) }, @@ -453,6 +461,7 @@ impl ProtocolHandle { pub fn report_substream_opened( &mut self, peer: PeerId, + direction: Direction, handshake: Vec, negotiated_fallback: Option, sink: NotificationsSink, @@ -466,6 +475,7 @@ impl ProtocolHandle { subscriber .unbounded_send(InnerNotificationEvent::NotificationStreamOpened { peer, + direction, handshake: handshake.clone(), negotiated_fallback: negotiated_fallback.clone(), sink: sink.clone(), diff --git a/client/network/src/protocol/notifications/service/tests.rs b/client/network/src/protocol/notifications/service/tests.rs index 36b3280b59d16..f750cc4584f25 100644 --- a/client/network/src/protocol/notifications/service/tests.rs +++ b/client/network/src/protocol/notifications/service/tests.rs @@ -51,17 +51,21 @@ async fn substream_opened() { let (mut handle, _stream) = proto.split(); let peer_id = PeerId::random(); - handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, negotiated_fallback, handshake, + direction, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); } else { panic!("invalid event received"); } @@ -89,17 +93,21 @@ async fn send_sync_notification() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, negotiated_fallback, handshake, + direction, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); } else { panic!("invalid event received"); } @@ -133,17 +141,21 @@ async fn send_async_notification() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, negotiated_fallback, handshake, + direction, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); } else { panic!("invalid event received"); } @@ -204,17 +216,21 @@ async fn receive_notification() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, negotiated_fallback, handshake, + direction, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); } else { panic!("invalid event received"); } @@ -254,17 +270,21 @@ async fn backpressure_works() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, negotiated_fallback, handshake, + direction, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); } else { panic!("invalid event received"); } @@ -310,17 +330,21 @@ async fn peer_disconnects_then_sync_notification_is_sent() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, negotiated_fallback, handshake, + direction, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); } else { panic!("invalid event received"); } @@ -356,17 +380,21 @@ async fn peer_disconnects_then_async_notification_is_sent() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, negotiated_fallback, handshake, + direction, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); } else { panic!("invalid event received"); } @@ -491,18 +519,22 @@ async fn cloned_service_opening_substream_sending_and_receiving_notifications_wo assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that then notification stream has been opened - handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); for notif in vec![&mut notif1, &mut notif2, &mut notif3] { if let Some(NotificationEvent::NotificationStreamOpened { peer, negotiated_fallback, handshake, + direction, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); } else { panic!("invalid event received"); } @@ -565,17 +597,21 @@ async fn sending_notifications_using_notifications_sink_works() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, negotiated_fallback, handshake, + direction, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); } else { panic!("invalid event received"); } @@ -638,17 +674,21 @@ async fn notification_sink_replaced() { assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); // report that a substream has been opened - handle.report_substream_opened(peer_id, vec![1, 3, 3, 7], None, sink).unwrap(); + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); if let Some(NotificationEvent::NotificationStreamOpened { peer, negotiated_fallback, handshake, + direction, }) = notif.next_event().await { assert_eq!(peer_id, peer); assert_eq!(negotiated_fallback, None); assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); } else { panic!("invalid event received"); } diff --git a/client/network/src/service.rs b/client/network/src/service.rs index f4421e57d9f78..d24e3057b705b 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -96,7 +96,10 @@ use std::{ pub use behaviour::{InboundFailure, OutboundFailure, ResponseFailure}; pub use libp2p::identity::{DecodingError, Keypair, PublicKey}; -pub use protocol::NotificationsSink; +pub use protocol::{NotificationCommand, NotificationsSink, ProtocolHandle}; + +// #[cfg(test)] +// pub use crate::protocol:: mod metrics; mod out_events; diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 76b46c2a2203e..3620089216802 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -639,6 +639,23 @@ pub enum ValidationResult { Reject, } +/// Substream direction. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum Direction { + /// Substream opened by the remote node. + Inbound, + + /// Substream opened by the local node. + Outbound, +} + +impl Direction { + /// Is the direction inbound. + pub fn is_inbound(&self) -> bool { + std::matches!(self, Direction::Inbound) + } +} + /// Events received by the protocol from `Notifications`. #[derive(Debug)] pub enum NotificationEvent { @@ -660,13 +677,14 @@ pub enum NotificationEvent { /// Peer ID. peer: PeerId, + /// Is the substream inbound or outbound. + direction: Direction, + /// Received handshake. handshake: Vec, /// Negotiated fallback. negotiated_fallback: Option, - // /// Message service associated with the peer. - // message_service: Box Result<(), error::Error>; /// Set handshake for the notification protocol replacing the old handshake. - async fn set_hanshake(&mut self, handshake: Vec) -> Result<(), ()>; + async fn set_handshake(&mut self, handshake: Vec) -> Result<(), ()>; /// Get next event from the `Notifications` event stream. async fn next_event(&mut self) -> Option; diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index d4523c5aa238f..55c9433504947 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -54,7 +54,7 @@ use sc_network_common::{ use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_blockchain::HeaderMetadata; use sp_consensus::block_validation::BlockAnnounceValidator; -use sp_runtime::traits::{Block as BlockT, Header, NumberFor, Zero}; +use sp_runtime::traits::{Block as BlockT, Header, Zero}; use std::{ collections::{HashMap, HashSet}, @@ -587,18 +587,6 @@ where } } - /// Inform sync about new best imported block. - pub fn new_best_block_imported(&mut self, hash: B::Hash, number: NumberFor) { - log::debug!(target: "sync", "New best block imported {:?}/#{}", hash, number); - - self.chain_sync.update_chain_info(&hash, number); - self.network_service.set_notification_handshake( - self.block_announce_protocol_name.clone(), - BlockAnnouncesHandshake::::build(self.roles, number, hash, self.genesis_hash) - .encode(), - ) - } - pub async fn run(mut self) { self.syncing_started = Some(Instant::now()); @@ -698,8 +686,24 @@ where } }, ToServiceCommand::AnnounceBlock(hash, data) => self.announce_block(hash, data), - ToServiceCommand::NewBestBlockImported(hash, number) => - self.new_best_block_imported(hash, number), + ToServiceCommand::NewBestBlockImported(hash, number) => { + log::debug!(target: "sync", "New best block imported {:?}/#{}", hash, number); + + self.chain_sync.update_chain_info(&hash, number); + while let Poll::Pending = self + .notification_service + .set_handshake( + BlockAnnouncesHandshake::::build( + self.roles, + number, + hash, + self.genesis_hash, + ) + .encode(), + ) + .poll_unpin(cx) + {} + }, ToServiceCommand::Status(tx) => { let mut status = self.chain_sync.status(); status.num_connected_peers = self.peers.len() as u32; @@ -744,29 +748,51 @@ where match event { NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => { let validation_result = self - .validate_handshake(&peer, handshake) + .validate_connection(&peer, handshake, true) .map_or(ValidationResult::Reject, |_| ValidationResult::Accept); let _ = result_tx.send(validation_result); }, - NotificationEvent::NotificationStreamOpened { peer, handshake, .. } => { + NotificationEvent::NotificationStreamOpened { + peer, handshake, direction, .. + } => { log::debug!( target: LOG_TARGET, - "substream opened for {peer}, handshake {handshake:?}" + "Substream opened for {peer}, handshake {handshake:?}" ); - match self.validate_handshake(&peer, handshake) { - Ok(handshake) => - // TODO: fix inbound - if self.on_sync_peer_connected(peer, &handshake, false).is_err() { - log::debug!(target: LOG_TARGET, "failed to register peer"); + match self.validate_connection(&peer, handshake, direction.is_inbound()) { + Ok(handshake) => { + if self + .on_sync_peer_connected(peer, &handshake, direction.is_inbound()) + .is_err() + { + log::debug!(target: LOG_TARGET, "Failed to register peer {peer}"); self.network_service.disconnect_peer( peer, self.block_announce_protocol_name.clone(), ); - }, - Err(err) => { - log::debug!(target: LOG_TARGET, "failed to decode handshake: {err:?}"); + } + }, + // TODO: `wrong_genesis` is a really ugly hack to work around the issue that + // that `PeerStore` thinks the peer is connected when in fact it can be + // still be under validation. If the peer has different genesis than the + // local node the validation fails but the peer cannot be reported in + // `validate_connection()` as that is also called by + // `ValiateInboundSubstream` which means that the peer is still being + // validated and banning the peer when handling that event would + // result in peer getting dropped twice. + // + // The proper way to fix this is to integrate `ProtocolController` more + // tightly with `NotificationService` or add an additional API call for + // banning pre-accepted peers (which is not desirable) + Err(wrong_genesis) => { + log::debug!(target: LOG_TARGET, "`SyncingEngine` rejected {peer}"); + + if wrong_genesis { + self.peer_store_handle.report_peer(*peer_id, rep::GENESIS_MISMATCH); + } + self.network_service .disconnect_peer(peer, self.block_announce_protocol_name.clone()); }, @@ -848,42 +874,32 @@ where .retain(|stream| stream.unbounded_send(SyncEvent::PeerDisconnected(peer)).is_ok()); } + /// Validate received handshake. fn validate_handshake( &mut self, - peer: &PeerId, + peer_id: &PeerId, handshake: Vec, - ) -> Result, ()> { - log::trace!(target: LOG_TARGET, "New peer {peer} {handshake:?}"); + ) -> Result, bool> { + log::trace!(target: LOG_TARGET, "Validate handshake for {peer_id}"); let handshake = as DecodeAll>::decode_all(&mut &handshake[..]) .map_err(|error| { - log::debug!(target: LOG_TARGET, "failed to decode handshake for {peer}: {error:?}"); + log::debug!(target: LOG_TARGET, "Failed to decode handshake for {peer_id}: {error:?}"); + false })?; - if self.peers.contains_key(&peer) { - log::error!( - target: LOG_TARGET, - "Called on_sync_peer_connected with already connected peer {peer}", - ); - debug_assert!(false); - return Err(()) - } - if handshake.genesis_hash != self.genesis_hash { - // TODO(aaro): report peer but verify that `PeerStore` doesn't try to disconnect it - // self.network_service.report_peer(peer, rep::GENESIS_MISMATCH); - - if self.important_peers.contains(&peer) { + if self.important_peers.contains(&peer_id) { log::error!( target: LOG_TARGET, - "Reserved peer id `{peer}` is on a different chain (our genesis: {} theirs: {})", + "Reserved peer id `{peer_id}` is on a different chain (our genesis: {} theirs: {})", self.genesis_hash, handshake.genesis_hash, ); - } else if self.boot_node_ids.contains(&peer) { + } else if self.boot_node_ids.contains(&peer_id) { log::error!( target: LOG_TARGET, - "Bootnode with peer id `{peer}` is on a different chain (our genesis: {} theirs: {})", + "Bootnode with peer id `{peer_id}` is on a different chain (our genesis: {} theirs: {})", self.genesis_hash, handshake.genesis_hash, ); @@ -896,10 +912,33 @@ where ); } - return Err(()) + return Err(true) } - let no_slot_peer = self.default_peers_set_no_slot_peers.contains(&peer); + Ok(handshake) + } + + /// Validate inbound connection. + fn validate_connection( + &mut self, + peer_id: &PeerId, + handshake: Vec, + inbound: bool, + ) -> Result, bool> { + log::trace!(target: LOG_TARGET, "New peer {peer_id} {handshake:?}"); + + let handshake = self.validate_handshake(peer_id, handshake)?; + + if self.peers.contains_key(&peer_id) { + log::error!( + target: LOG_TARGET, + "Called `validate_connection()` with already connected peer {peer_id}", + ); + debug_assert!(false); + return Err(false) + } + + let no_slot_peer = self.default_peers_set_no_slot_peers.contains(&peer_id); let this_peer_reserved_slot: usize = if no_slot_peer { 1 } else { 0 }; if handshake.roles.is_full() && @@ -908,16 +947,25 @@ where self.default_peers_set_no_slot_connected_peers.len() + this_peer_reserved_slot { - log::debug!(target: LOG_TARGET, "Too many full nodes, rejecting {peer}"); - return Err(()) + log::debug!(target: LOG_TARGET, "Too many full nodes, rejecting {peer_id}"); + return Err(false) + } + + // make sure to accept no more than `--in-peers` many full nodes + if !no_slot_peer && + handshake.roles.is_full() && + inbound && self.num_in_peers == self.max_in_peers + { + log::debug!(target: LOG_TARGET, "All inbound slots have been consumed, rejecting {peer_id}"); + return Err(false) } if handshake.roles.is_light() && (self.peers.len() - self.chain_sync.num_peers()) >= self.default_peers_set_num_light { // Make sure that not all slots are occupied by light clients. - log::debug!(target: LOG_TARGET, "Too many light nodes, rejecting {peer}"); - return Err(()) + log::debug!(target: LOG_TARGET, "Too many light nodes, rejecting {peer_id}"); + return Err(false) } Ok(handshake) @@ -936,72 +984,6 @@ where ) -> Result<(), ()> { log::trace!(target: "sync", "New peer {} {:?}", who, status); - if self.peers.contains_key(&who) { - log::error!(target: "sync", "Called on_sync_peer_connected with already connected peer {}", who); - debug_assert!(false); - return Err(()) - } - - if status.genesis_hash != self.genesis_hash { - self.network_service.report_peer(who, rep::GENESIS_MISMATCH); - - if self.important_peers.contains(&who) { - log::error!( - target: "sync", - "Reserved peer id `{}` is on a different chain (our genesis: {} theirs: {})", - who, - self.genesis_hash, - status.genesis_hash, - ); - } else if self.boot_node_ids.contains(&who) { - log::error!( - target: "sync", - "Bootnode with peer id `{}` is on a different chain (our genesis: {} theirs: {})", - who, - self.genesis_hash, - status.genesis_hash, - ); - } else { - log::debug!( - target: "sync", - "Peer is on different chain (our genesis: {} theirs: {})", - self.genesis_hash, status.genesis_hash - ); - } - - return Err(()) - } - - let no_slot_peer = self.default_peers_set_no_slot_peers.contains(&who); - let this_peer_reserved_slot: usize = if no_slot_peer { 1 } else { 0 }; - - // make sure to accept no more than `--in-peers` many full nodes - if !no_slot_peer && - status.roles.is_full() && - inbound && self.num_in_peers == self.max_in_peers - { - log::debug!(target: "sync", "All inbound slots have been consumed, rejecting {who}"); - return Err(()) - } - - if status.roles.is_full() && - self.chain_sync.num_peers() >= - self.default_peers_set_num_full + - self.default_peers_set_no_slot_connected_peers.len() + - this_peer_reserved_slot - { - log::debug!(target: "sync", "Too many full nodes, rejecting {}", who); - return Err(()) - } - - if status.roles.is_light() && - (self.peers.len() - self.chain_sync.num_peers()) >= self.default_peers_set_num_light - { - // Make sure that not all slots are occupied by light clients. - log::debug!(target: "sync", "Too many light nodes, rejecting {}", who); - return Err(()) - } - let peer = Peer { info: ExtendedPeerInfo { roles: status.roles, @@ -1028,12 +1010,12 @@ where None }; - log::debug!(target: LOG_TARGET, "connected {}", who); + log::debug!(target: LOG_TARGET, "Peer connected {who}"); self.peers.insert(who, peer); self.peer_store_handle.set_peer_role(&who, status.roles.into()); - if no_slot_peer { + if self.default_peers_set_no_slot_peers.contains(&who) { self.default_peers_set_no_slot_connected_peers.insert(who); } else if inbound && status.roles.is_full() { self.num_in_peers += 1; @@ -1049,3 +1031,331 @@ where Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::service::network::NetworkServiceProvider; + use sc_block_builder::BlockBuilderProvider; + use sc_network::{ + config::NetworkConfiguration, peer_store::PeerStore, service::traits::Direction, + NetworkBlock, NotificationCommand, NotificationsSink, + }; + use sp_consensus::block_validation::DefaultBlockAnnounceValidator; + use substrate_test_runtime_client::{ + runtime::Block, DefaultTestClientBuilderExt, TestClient, TestClientBuilder, + TestClientBuilderExt, + }; + + fn make_syncing_engine() -> ( + Arc, + SyncingEngine, + SyncingService, + NonDefaultSetConfig, + ) { + let mut net_config = NetworkConfiguration::new_local(); + + // 8 outbound full nodes, 32 inbound full nodes and 100 inbound light nodes + net_config.default_peers_set_num_full = 40; + net_config.default_peers_set.in_peers = 132; + net_config.default_peers_set.out_peers = 8; + + let full_config = FullNetworkConfiguration::new(&net_config); + let client = Arc::new(TestClientBuilder::new().build()); + let block_announce_validator = Box::new(DefaultBlockAnnounceValidator); + let import_queue = Box::new(sc_consensus::import_queue::mock::MockImportQueueHandle::new()); + let (_chain_sync_network_provider, chain_sync_network_handle) = + NetworkServiceProvider::new(); + let peer_store = PeerStore::new(vec![]); + let handle = peer_store.handle(); + + let (engine, service, config) = SyncingEngine::new( + Roles::FULL, + client.clone(), + None, + &full_config, + ProtocolId::from("test-protocol-name"), + &None, + block_announce_validator, + None, + chain_sync_network_handle, + import_queue, + ProtocolName::from("/block-request/1"), + ProtocolName::from("/state-request/1"), + None, + handle, + ) + .unwrap(); + + (client, engine, service, config) + } + + #[tokio::test] + async fn reject_invalid_handshake() { + let (_client, engine, _service, config) = make_syncing_engine(); + let (handle, _) = config.take_protocol_handle().split(); + + tokio::spawn(engine.run()); + + let rx = handle.report_incoming_substream(PeerId::random(), vec![1, 2, 3, 4]).unwrap(); + + assert_eq!(rx.await, Ok(ValidationResult::Reject)); + } + + #[tokio::test] + async fn accept_valid_peer() { + let (client, engine, _service, config) = make_syncing_engine(); + let (handle, _) = config.take_protocol_handle().split(); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + let handshake = BlockAnnouncesHandshake::::build( + engine.roles, + *block.header().number(), + block.hash(), + engine.genesis_hash, + ) + .encode(); + + tokio::spawn(engine.run()); + + let rx = handle.report_incoming_substream(PeerId::random(), handshake).unwrap(); + + assert_eq!(rx.await, Ok(ValidationResult::Accept)); + } + + #[tokio::test] + async fn valid_peer_included_into_peers() { + let (client, engine, service, config) = make_syncing_engine(); + let (mut handle, _) = config.take_protocol_handle().split(); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + let handshake = BlockAnnouncesHandshake::::build( + engine.roles, + *block.header().number(), + block.hash(), + engine.genesis_hash, + ) + .encode(); + + tokio::spawn(engine.run()); + + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, handshake.clone()).unwrap(); + assert_eq!(rx.await, Ok(ValidationResult::Accept)); + let (sink, _, _) = NotificationsSink::new(peer_id); + + handle + .report_substream_opened(peer_id, Direction::Inbound, handshake, None, sink) + .unwrap(); + + if let Err(_) = tokio::time::timeout(Duration::from_secs(10), async move { + loop { + if service.num_sync_peers().await.unwrap() == 1 { + break + } + } + }) + .await + { + panic!("failed to add sync peer"); + } + } + + #[tokio::test] + async fn syncing_engine_full_of_inbound_peers() { + let (client, engine, service, config) = make_syncing_engine(); + let (mut handle, _) = config.take_protocol_handle().split(); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + let handshake = BlockAnnouncesHandshake::::build( + engine.roles, + *block.header().number(), + block.hash(), + engine.genesis_hash, + ) + .encode(); + + tokio::spawn(engine.run()); + + // add maximum number of allowed inbound peers to `SyncingEngine` + let num_in_peers = 32; + for i in 0..num_in_peers { + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, handshake.clone()).unwrap(); + assert_eq!(rx.await, Ok(ValidationResult::Accept)); + let (sink, _, _) = NotificationsSink::new(peer_id); + + handle + .report_substream_opened(peer_id, Direction::Inbound, handshake.clone(), None, sink) + .unwrap(); + let sync_service = service.clone(); + + if let Err(_) = tokio::time::timeout(Duration::from_secs(10), async move { + loop { + if sync_service.num_sync_peers().await.unwrap() == i + 1 { + break + } + } + }) + .await + { + panic!("failed to add sync peer"); + } + } + + // try to add one more peer and verify the peer is rejected during handshake validation + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, handshake.clone()).unwrap(); + assert_eq!(rx.await, Ok(ValidationResult::Reject)); + } + + #[tokio::test] + async fn syncing_engine_full_of_inbound_peers_outbound_accepted() { + let (client, engine, service, config) = make_syncing_engine(); + let (mut handle, _) = config.take_protocol_handle().split(); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + let handshake = BlockAnnouncesHandshake::::build( + engine.roles, + *block.header().number(), + block.hash(), + engine.genesis_hash, + ) + .encode(); + + tokio::spawn(engine.run()); + + // add maximum number of allowed inbound peers to `SyncingEngine` + let num_in_peers = 32; + for i in 0..num_in_peers { + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, handshake.clone()).unwrap(); + assert_eq!(rx.await, Ok(ValidationResult::Accept)); + let (sink, _, _) = NotificationsSink::new(peer_id); + + handle + .report_substream_opened(peer_id, Direction::Inbound, handshake.clone(), None, sink) + .unwrap(); + let sync_service = service.clone(); + + if let Err(_) = tokio::time::timeout(Duration::from_secs(10), async move { + loop { + if sync_service.num_sync_peers().await.unwrap() == i + 1 { + break + } + } + }) + .await + { + panic!("failed to add sync peer"); + } + } + + // try to add one more peer, this time outbound and verify it's accepted + let peer_id = PeerId::random(); + let (sink, _, _) = NotificationsSink::new(peer_id); + handle + .report_substream_opened(peer_id, Direction::Outbound, handshake.clone(), None, sink) + .unwrap(); + let sync_service = service.clone(); + + if let Err(_) = tokio::time::timeout(Duration::from_secs(10), async move { + loop { + if sync_service.num_sync_peers().await.unwrap() == num_in_peers + 1 { + break + } + } + }) + .await + { + panic!("failed to add sync peer"); + } + } + + #[tokio::test] + async fn syncing_engine_full_of_inbound_full_peers_but_not_light_peers() { + let (client, engine, service, config) = make_syncing_engine(); + let (mut handle, _) = config.take_protocol_handle().split(); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + let genesis = engine.genesis_hash.clone(); + let handshake = BlockAnnouncesHandshake::::build( + engine.roles, + *block.header().number(), + block.hash(), + engine.genesis_hash, + ) + .encode(); + + tokio::spawn(engine.run()); + + // add maximum number of allowed inbound peers to `SyncingEngine` + let num_in_peers = 32; + for i in 0..num_in_peers { + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, handshake.clone()).unwrap(); + assert_eq!(rx.await, Ok(ValidationResult::Accept)); + let (sink, _, _) = NotificationsSink::new(peer_id); + + handle + .report_substream_opened(peer_id, Direction::Inbound, handshake.clone(), None, sink) + .unwrap(); + let sync_service = service.clone(); + + if let Err(_) = tokio::time::timeout(Duration::from_secs(10), async move { + loop { + if sync_service.num_sync_peers().await.unwrap() == i + 1 { + break + } + } + }) + .await + { + panic!("failed to add sync peer"); + } + } + + // try to add one more peer, this time outbound and verify it's accepted + let handshake = BlockAnnouncesHandshake::::build( + Roles::LIGHT, + *block.header().number(), + block.hash(), + genesis, + ) + .encode(); + let peer_id = PeerId::random(); + let (sink, _, _) = NotificationsSink::new(peer_id); + handle + .report_substream_opened(peer_id, Direction::Inbound, handshake.clone(), None, sink) + .unwrap(); + let sync_service = service.clone(); + + if let Err(_) = tokio::time::timeout(Duration::from_secs(10), async move { + loop { + if sync_service.status().await.unwrap().num_connected_peers == num_in_peers + 1 { + break + } + } + }) + .await + { + panic!("failed to add sync peer"); + } + } + + #[tokio::test] + async fn best_block_import_updates_handshake() { + let (client, engine, service, config) = make_syncing_engine(); + let (_handle, mut commands) = config.take_protocol_handle().split(); + + tokio::spawn(engine.run()); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + service.new_best_block_imported(block.hash(), *block.header().number()); + + assert!(std::matches!( + commands.next().await.unwrap(), + NotificationCommand::SetHandshake(_) + )); + } +} diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 373d81ee22724..0119cb1a25b82 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -907,7 +907,6 @@ where let (chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); - let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (engine, sync_service, block_announce_config) = sc_network_sync::engine::SyncingEngine::new( Roles::from(if config.is_authority { &Role::Authority } else { &Role::Full }), @@ -923,7 +922,6 @@ where block_request_protocol_config.name.clone(), state_request_protocol_config.name.clone(), Some(warp_protocol_config.name.clone()), - rx, peer_store_handle.clone(), ) .unwrap(); @@ -960,7 +958,6 @@ where fork_id, metrics_registry: None, block_announce_config, - tx, }) .unwrap(); diff --git a/client/network/test/src/service.rs b/client/network/test/src/service.rs index 2ddc1154385b3..b634c7d9e97d7 100644 --- a/client/network/test/src/service.rs +++ b/client/network/test/src/service.rs @@ -186,7 +186,6 @@ impl TestNetworkBuilder { let (chain_sync_network_provider, chain_sync_network_handle) = self.chain_sync_network.unwrap_or(NetworkServiceProvider::new()); - let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (engine, chain_sync_service, block_announce_config) = SyncingEngine::new( Roles::from(&config::Role::Full), client.clone(), @@ -201,7 +200,6 @@ impl TestNetworkBuilder { block_request_protocol_config.name.clone(), state_request_protocol_config.name.clone(), None, - rx, peer_store_handle.clone(), ) .unwrap(); @@ -249,7 +247,6 @@ impl TestNetworkBuilder { protocol_id, fork_id, metrics_registry: None, - tx, }) .unwrap(); From a84000f7ed99d357af72c87dc198daa3e6e85b6b Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Wed, 9 Aug 2023 10:32:26 +0300 Subject: [PATCH 37/43] Minor code cleanups --- .../grandpa/src/communication/tests.rs | 8 +- client/network-gossip/src/bridge.rs | 21 +--- client/network/src/config.rs | 1 - .../src/protocol/notifications/behaviour.rs | 104 +++++++++--------- .../src/protocol/notifications/handler.rs | 64 +++++++++-- .../protocol/notifications/service/metrics.rs | 28 +++-- .../src/protocol/notifications/tests.rs | 18 ++- client/network/sync/src/engine.rs | 2 +- client/network/transactions/src/lib.rs | 1 - 9 files changed, 153 insertions(+), 94 deletions(-) diff --git a/client/consensus/grandpa/src/communication/tests.rs b/client/consensus/grandpa/src/communication/tests.rs index cd6d51d78d702..3fbc954d89cf6 100644 --- a/client/consensus/grandpa/src/communication/tests.rs +++ b/client/consensus/grandpa/src/communication/tests.rs @@ -28,7 +28,7 @@ use parity_scale_codec::{DecodeAll, Encode}; use sc_network::{ config::{MultiaddrWithPeerId, Role}, event::Event as NetworkEvent, - service::traits::{MessageSink, NotificationEvent, NotificationService}, + service::traits::{Direction, MessageSink, NotificationEvent, NotificationService}, types::ProtocolName, Multiaddr, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, NetworkSyncForkRequest, NotificationSenderError, NotificationSenderT as NotificationSender, @@ -239,7 +239,6 @@ impl NotificationService for TestNotificationService { /// Send synchronous `notification` to `peer`. fn send_sync_notification(&self, _peer: &PeerId, _notification: Vec) { - // TODO: this needs to be implemented unimplemented!(); } @@ -253,7 +252,7 @@ impl NotificationService for TestNotificationService { } /// Set handshake for the notification protocol replacing the old handshake. - async fn set_hanshake(&mut self, _handshake: Vec) -> Result<(), ()> { + async fn set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { unimplemented!(); } @@ -472,6 +471,7 @@ fn good_commit_leads_to_relay() { let _ = tester.notification_tx.unbounded_send( NotificationEvent::NotificationStreamOpened { peer: sender_id, + direction: Direction::Inbound, negotiated_fallback: None, handshake: Roles::FULL.encode(), }, @@ -488,6 +488,7 @@ fn good_commit_leads_to_relay() { let _ = tester.notification_tx.unbounded_send( NotificationEvent::NotificationStreamOpened { peer: receiver_id, + direction: Direction::Inbound, negotiated_fallback: None, handshake: Roles::FULL.encode(), }, @@ -621,6 +622,7 @@ fn bad_commit_leads_to_report() { let _ = tester.notification_tx.unbounded_send( NotificationEvent::NotificationStreamOpened { peer: sender_id, + direction: Direction::Inbound, negotiated_fallback: None, handshake: Roles::FULL.encode(), }, diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 305bc7b2fc2e0..a6c5662d1d806 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -202,16 +202,11 @@ impl Future for GossipEngine { handshake, result_tx, } => { - log::debug!( - target: "gossip", - "accepting inbound substream from {peer}, handshake {handshake:?}" - ); let _ = result_tx.send(ValidationResult::Accept); }, NotificationEvent::NotificationStreamOpened { peer, handshake, .. } => { - log::debug!(target: "gossip", "handshake {handshake:?}"); let Some(role) = this.network.peer_role(peer, handshake) else { log::debug!(target: "gossip", "role for {peer} couldn't be determined"); continue @@ -344,7 +339,7 @@ mod tests { use quickcheck::{Arbitrary, Gen, QuickCheck}; use sc_network::{ config::MultiaddrWithPeerId, - service::traits::{MessageSink, NotificationEvent}, + service::traits::{Direction, MessageSink, NotificationEvent}, Event, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, NotificationSenderError, NotificationSenderT as NotificationSender, NotificationService, NotificationsSink, Roles, @@ -515,28 +510,20 @@ mod tests { rx: UnboundedReceiver, } - // TODO: provide implementation #[async_trait::async_trait] impl sc_network::service::traits::NotificationService for TestNotificationService { - /// Instruct `Notifications` to open a new substream for `peer`. - /// - /// `dial_if_disconnected` informs `Notifications` whether to dial - // the peer if there is currently no active connection to it. async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { unimplemented!(); } - /// Instruct `Notifications` to close substream for `peer`. async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { unimplemented!(); } - /// Send synchronous `notification` to `peer`. fn send_sync_notification(&self, _peer: &PeerId, _notification: Vec) { unimplemented!(); } - /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. async fn send_async_notification( &self, _peer: &PeerId, @@ -545,12 +532,10 @@ mod tests { unimplemented!(); } - /// Set handshake for the notification protocol replacing the old handshake. - async fn set_hanshake(&mut self, _handshake: Vec) -> Result<(), ()> { + async fn set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { unimplemented!(); } - /// Get next event from the `Notifications` event stream. async fn next_event(&mut self) -> Option { self.rx.next().await } @@ -635,6 +620,7 @@ mod tests { // Register the remote peer. tx.send(NotificationEvent::NotificationStreamOpened { peer: remote_peer, + direction: Direction::Inbound, negotiated_fallback: None, handshake: Roles::FULL.encode(), }) @@ -797,6 +783,7 @@ mod tests { // Register the remote peer. tx.start_send(NotificationEvent::NotificationStreamOpened { peer: remote_peer, + direction: Direction::Inbound, negotiated_fallback: None, handshake: Roles::FULL.encode(), }) diff --git a/client/network/src/config.rs b/client/network/src/config.rs index dfc6ebcd3f28b..877b2dc0784da 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -743,7 +743,6 @@ pub struct Params { /// Registry for recording prometheus metrics to. pub metrics_registry: Option, - // TODO(aaro): remove maybe? /// Block announce protocol configuration pub block_announce_config: NonDefaultSetConfig, } diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index c3f6f1092a655..b4166eea342c9 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -167,6 +167,9 @@ pub struct Notifications { // returned by the completed future against the `crate::peerset::IncomingIndex` stored in // `PeerState::Incoming` to check whether the completed future is stale or not. pending_inbound_validations: FuturesUnordered, + + /// Metrics for notifications. + metrics: Option, } /// Configuration for a notifications protocol. @@ -441,6 +444,7 @@ impl Notifications { next_incoming_index: IncomingIndex(0), events: VecDeque::new(), pending_inbound_validations: FuturesUnordered::new(), + metrics, } } @@ -886,27 +890,28 @@ impl Notifications { /// Substream has been accepted by the `ProtocolController` and must now be sent /// to the protocol for validation. fn peerset_report_preaccept(&mut self, index: IncomingIndex) { - if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { - trace!( - target: LOG_TARGET, - "PSM => Preaccept({:?}): Sent to protocol for validation", - index - ); - let incoming = &self.incoming[pos]; - - match self.protocol_handles[usize::from(incoming.set_id)] - .report_incoming_substream(incoming.peer_id, incoming.handshake.clone()) - { - Ok(rx) => self - .pending_inbound_validations - .push(Box::pin(async move { (rx.await, index) })), - Err(err) => { - error!(target: LOG_TARGET, "protocol has exited: {err:?}"); - debug_assert!(false); - }, - } - } else { + let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) else { error!(target: LOG_TARGET, "PSM => Preaccept({:?}): Invalid index", index); + return; + }; + + trace!( + target: LOG_TARGET, + "PSM => Preaccept({:?}): Sent to protocol for validation", + index + ); + let incoming = &self.incoming[pos]; + + match self.protocol_handles[usize::from(incoming.set_id)] + .report_incoming_substream(incoming.peer_id, incoming.handshake.clone()) + { + Ok(rx) => self + .pending_inbound_validations + .push(Box::pin(async move { (rx.await, index) })), + Err(err) => { + error!(target: LOG_TARGET, "protocol has exited: {err:?}"); + debug_assert!(false); + }, } } @@ -1074,7 +1079,7 @@ impl Notifications { }, peer => { error!( - target: "sub-libp2p", + target: LOG_TARGET, "State mismatch in libp2p: Expected alive incoming. Got {peer:?}.", ); None @@ -1120,6 +1125,7 @@ impl NetworkBehaviour for Notifications { send_back_addr: remote_addr.clone(), }, self.notif_protocols.clone(), + self.metrics.clone(), )) } @@ -1134,6 +1140,7 @@ impl NetworkBehaviour for Notifications { peer, ConnectedPoint::Dialer { address: addr.clone(), role_override }, self.notif_protocols.clone(), + self.metrics.clone(), )) } @@ -2175,7 +2182,7 @@ impl NetworkBehaviour for Notifications { }, }, Poll::Ready(None) => { - error!(target: "sub-libp2p", "Protocol command streams have been shut down"); + error!(target: LOG_TARGET, "Protocol command streams have been shut down"); break }, Poll::Pending => break, @@ -2193,7 +2200,7 @@ impl NetworkBehaviour for Notifications { self.protocol_report_reject(index); }, Err(_) => { - error!(target: "sub-libp2p", "Protocol has shut down"); + error!(target: LOG_TARGET, "Protocol has shut down"); break }, } @@ -2303,7 +2310,6 @@ mod tests { } } - // <<<<<<< HEAD fn development_notifs( ) -> (Notifications, ProtocolController, Box) { let (protocol_handle_pair, notif_service) = @@ -2558,7 +2564,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2758,7 +2764,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2911,7 +2917,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2955,7 +2961,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3025,7 +3031,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3173,7 +3179,7 @@ mod tests { peer_id: peer, connection_id: conn1, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3299,7 +3305,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3373,7 +3379,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3436,7 +3442,7 @@ mod tests { peer_id: peer, connection_id: conn1, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected.clone(), vec![]), + handler: NotifsHandler::new(peer, connected.clone(), vec![], None), remaining_established: 0usize, }, )); @@ -3450,7 +3456,7 @@ mod tests { peer_id: peer, connection_id: conn2, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3504,7 +3510,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3562,7 +3568,7 @@ mod tests { peer_id: peer, connection_id: conn2, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3623,7 +3629,7 @@ mod tests { peer_id: peer, connection_id: conn1, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3692,7 +3698,7 @@ mod tests { peer_id: peer, connection_id: conn1, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3733,7 +3739,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3856,7 +3862,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4170,7 +4176,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4379,7 +4385,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(0), endpoint: &endpoint.clone(), - handler: NotifsHandler::new(peer, endpoint, vec![]), + handler: NotifsHandler::new(peer, endpoint, vec![], None), remaining_established: 0usize, }, )); @@ -4539,7 +4545,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(1337), endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4574,7 +4580,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(1337), endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4625,7 +4631,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(1337), endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4672,7 +4678,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4722,7 +4728,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(1337), endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4765,7 +4771,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected.clone(), vec![]), + handler: NotifsHandler::new(peer, connected.clone(), vec![], None), remaining_established: 0usize, }, )); @@ -4776,7 +4782,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); diff --git a/client/network/src/protocol/notifications/handler.rs b/client/network/src/protocol/notifications/handler.rs index daaf714c696a3..ea20abebccd37 100644 --- a/client/network/src/protocol/notifications/handler.rs +++ b/client/network/src/protocol/notifications/handler.rs @@ -58,9 +58,12 @@ //! [`NotifsHandlerIn::Open`] has gotten an answer. use crate::{ - protocol::notifications::upgrade::{ - NotificationsIn, NotificationsInSubstream, NotificationsOut, NotificationsOutSubstream, - UpgradeCollec, + protocol::notifications::{ + service::metrics, + upgrade::{ + NotificationsIn, NotificationsInSubstream, NotificationsOut, NotificationsOutSubstream, + UpgradeCollec, + }, }, types::ProtocolName, }; @@ -106,6 +109,28 @@ const OPEN_TIMEOUT: Duration = Duration::from_secs(10); /// open substreams. const INITIAL_KEEPALIVE_TIME: Duration = Duration::from_secs(5); +/// Metrics-related information. +#[derive(Debug)] +pub struct MetricsInfo { + /// Protocol name. + protocol: ProtocolName, + + /// Metrics. + metrics: Option, +} + +impl MetricsInfo { + /// Create new [`MetricsInfo`]. + pub fn new(protocol: ProtocolName, metrics: Option) -> Self { + Self { protocol, metrics } + } + + /// Create new empty [`MetricsInfo`]. + pub fn new_empty() -> Self { + Self { protocol: ProtocolName::from(""), metrics: None } + } +} + /// The actual handler once the connection has been established. /// /// See the documentation at the module level for more information. @@ -126,11 +151,24 @@ pub struct NotifsHandler { events_queue: VecDeque< ConnectionHandlerEvent, >, + + /// Metrics-related information. + metrics_info: Vec>, } impl NotifsHandler { /// Creates new [`NotifsHandler`]. - pub fn new(peer_id: PeerId, endpoint: ConnectedPoint, protocols: Vec) -> Self { + pub fn new( + peer_id: PeerId, + endpoint: ConnectedPoint, + protocols: Vec, + metrics: Option, + ) -> Self { + let metrics_info = protocols + .iter() + .map(|config| Arc::new(MetricsInfo::new(config.name.clone(), metrics.clone()))) + .collect(); + Self { protocols: protocols .into_iter() @@ -148,6 +186,7 @@ impl NotifsHandler { endpoint, when_connection_open: Instant::now(), events_queue: VecDeque::with_capacity(16), + metrics_info, } } } @@ -335,7 +374,7 @@ pub struct NotificationsSink { inner: Arc, } -// #[cfg(test)] +// NOTE: only used for testing but must be `pub` as other crates in `client/network` use this. impl NotificationsSink { /// Create new pub fn new( @@ -350,6 +389,7 @@ impl NotificationsSink { peer_id, async_channel: FuturesMutex::new(async_tx), sync_channel: Mutex::new(Some(sync_tx)), + metrics_info: Arc::new(MetricsInfo::new_empty()), }), }, async_rx, @@ -371,6 +411,8 @@ struct NotificationsSinkInner { /// back-pressure cannot be properly exerted. /// It will be removed in a future version. sync_channel: Mutex>>, + /// Metrics-related information. + metrics_info: Arc, } /// Message emitted through the [`NotificationsSink`] and processed by the background task @@ -404,8 +446,14 @@ impl NotificationsSink { let mut lock = self.inner.sync_channel.lock(); if let Some(tx) = lock.as_mut() { - let result = - tx.try_send(NotificationsSinkMessage::Notification { message: message.into() }); + let message = message.into(); + metrics::register_notification_sent( + &self.inner.metrics_info.metrics, + &self.inner.metrics_info.protocol, + message.len(), + ); + + let result = tx.try_send(NotificationsSinkMessage::Notification { message }); if result.is_err() { // Cloning the `mpsc::Sender` guarantees the allocation of an extra spot in the @@ -558,6 +606,7 @@ impl ConnectionHandler for NotifsHandler { peer_id: self.peer_id, async_channel: FuturesMutex::new(async_tx), sync_channel: Mutex::new(Some(sync_tx)), + metrics_info: self.metrics_info[protocol_index].clone(), }), }; @@ -909,6 +958,7 @@ pub mod tests { peer_id: peer, async_channel: FuturesMutex::new(async_tx), sync_channel: Mutex::new(Some(sync_tx)), + metrics_info: Arc::new(MetricsInfo::new_empty()), }), }; let (in_substream, out_substream) = MockSubstream::new(); diff --git a/client/network/src/protocol/notifications/service/metrics.rs b/client/network/src/protocol/notifications/service/metrics.rs index 9bc6cc0527d61..c0206508c1823 100644 --- a/client/network/src/protocol/notifications/service/metrics.rs +++ b/client/network/src/protocol/notifications/service/metrics.rs @@ -26,9 +26,14 @@ use prometheus_endpoint::{ /// Notification metrics. #[derive(Debug, Clone)] pub struct Metrics { - pub notifications_sizes: HistogramVec, - pub notifications_streams_closed_total: CounterVec, + // Total number of opened substreams. pub notifications_streams_opened_total: CounterVec, + + /// Total number of closed substreams. + pub notifications_streams_closed_total: CounterVec, + + /// In/outbound notification sizez. + pub notifications_sizes: HistogramVec, } impl Metrics { @@ -84,7 +89,7 @@ pub fn register_substream_opened(metrics: &Option, protocol: &ProtocolN } } -/// Register opened substream to Prometheus. +/// Register closed substream to Prometheus. pub fn register_substream_closed(metrics: &Option, protocol: &ProtocolName) { if let Some(metrics) = metrics { metrics @@ -94,16 +99,17 @@ pub fn register_substream_closed(metrics: &Option, protocol: &ProtocolN } } -/// Register opened substream to Prometheus. -pub fn _register_notification_sent( - _metrics: &Option, - _protocol: &ProtocolName, - _size: usize, -) { - todo!(); +/// Register sent notification to Prometheus. +pub fn register_notification_sent(metrics: &Option, protocol: &ProtocolName, size: usize) { + if let Some(metrics) = metrics { + metrics + .notifications_sizes + .with_label_values(&["out", protocol]) + .observe(size as f64); + } } -/// Register opened substream to Prometheus. +/// Register received notification to Prometheus. pub fn register_notification_received( metrics: &Option, protocol: &ProtocolName, diff --git a/client/network/src/protocol/notifications/tests.rs b/client/network/src/protocol/notifications/tests.rs index 0cc46a6e4413f..da8ddc811312b 100644 --- a/client/network/src/protocol/notifications/tests.rs +++ b/client/network/src/protocol/notifications/tests.rs @@ -22,6 +22,8 @@ use crate::{ peer_store::PeerStore, protocol::notifications::{Notifications, NotificationsOut, ProtocolConfig}, protocol_controller::{ProtoSetConfig, ProtocolController, SetId}, + service::traits::{NotificationEvent, ValidationResult}, + NotificationService, }; use futures::{future::BoxFuture, prelude::*}; @@ -70,7 +72,7 @@ fn build_nodes() -> (Swarm, Swarm) { .timeout(Duration::from_secs(20)) .boxed(); - let (protocol_handle_pair, _notif_service) = + let (protocol_handle_pair, mut notif_service) = crate::protocol::notifications::service::notification_service("/foo".into()); let peer_store = PeerStore::new(if index == 0 { keypairs.iter().skip(1).map(|keypair| keypair.public().to_peer_id()).collect() @@ -123,7 +125,17 @@ fn build_nodes() -> (Swarm, Swarm) { .collect(), }; - let runtime = tokio::runtime::Runtime::new().unwrap(); + let mut runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.spawn(async move { + loop { + if let NotificationEvent::ValidateInboundSubstream { result_tx, .. } = + notif_service.next_event().await.unwrap() + { + result_tx.send(ValidationResult::Accept).unwrap(); + } + } + }); + let mut swarm = SwarmBuilder::with_executor( transport, behaviour, @@ -253,7 +265,6 @@ impl NetworkBehaviour for CustomProtoWithAddr { } #[test] -#[ignore] fn reconnect_after_disconnect() { // We connect two nodes together, then force a disconnect (through the API of the `Service`), // check that the disconnect worked, and finally check whether they successfully reconnect. @@ -368,7 +379,6 @@ fn reconnect_after_disconnect() { } }; - // TODO: rewrite these using `NotificationService` match event { SwarmEvent::Behaviour(NotificationsOut::CustomProtocolOpen { .. }) | SwarmEvent::Behaviour(NotificationsOut::CustomProtocolClosed { .. }) => panic!(), diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index 55c9433504947..1add96f17d49e 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -790,7 +790,7 @@ where log::debug!(target: LOG_TARGET, "`SyncingEngine` rejected {peer}"); if wrong_genesis { - self.peer_store_handle.report_peer(*peer_id, rep::GENESIS_MISMATCH); + self.peer_store_handle.report_peer(peer, rep::GENESIS_MISMATCH); } self.network_service diff --git a/client/network/transactions/src/lib.rs b/client/network/transactions/src/lib.rs index b562f8ff8f340..93d9312c79cf4 100644 --- a/client/network/transactions/src/lib.rs +++ b/client/network/transactions/src/lib.rs @@ -473,7 +473,6 @@ where propagated_to.entry(hash).or_default().push(who.to_base58()); } trace!(target: "sync", "Sending {} transactions to {}", to_send.len(), who); - // TODO: don't ignore error let _ = self.notification_service.send_sync_notification(who, to_send.encode()); } } From 92da6465208fb030d45c146b61b6077705c5062b Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Wed, 9 Aug 2023 11:41:38 +0300 Subject: [PATCH 38/43] Fix warnings --- client/network-gossip/src/bridge.rs | 8 ++------ client/network/src/protocol/notifications/tests.rs | 3 +-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index a6c5662d1d806..3f46cf593a614 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -197,11 +197,7 @@ impl Future for GossipEngine { match next_notification { Poll::Ready(Some(event)) => match event { - NotificationEvent::ValidateInboundSubstream { - peer, - handshake, - result_tx, - } => { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { let _ = result_tx.send(ValidationResult::Accept); }, NotificationEvent::NotificationStreamOpened { @@ -342,7 +338,7 @@ mod tests { service::traits::{Direction, MessageSink, NotificationEvent}, Event, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, NotificationSenderError, NotificationSenderT as NotificationSender, NotificationService, - NotificationsSink, Roles, + Roles, }; use sc_network_common::{role::ObservedRole, sync::SyncEventStream}; use sp_runtime::{ diff --git a/client/network/src/protocol/notifications/tests.rs b/client/network/src/protocol/notifications/tests.rs index da8ddc811312b..cf33545bad09d 100644 --- a/client/network/src/protocol/notifications/tests.rs +++ b/client/network/src/protocol/notifications/tests.rs @@ -23,7 +23,6 @@ use crate::{ protocol::notifications::{Notifications, NotificationsOut, ProtocolConfig}, protocol_controller::{ProtoSetConfig, ProtocolController, SetId}, service::traits::{NotificationEvent, ValidationResult}, - NotificationService, }; use futures::{future::BoxFuture, prelude::*}; @@ -125,7 +124,7 @@ fn build_nodes() -> (Swarm, Swarm) { .collect(), }; - let mut runtime = tokio::runtime::Runtime::new().unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); runtime.spawn(async move { loop { if let NotificationEvent::ValidateInboundSubstream { result_tx, .. } = From 18d948e6df451a3a90e7814d4864772336f2b7ee Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Thu, 10 Aug 2023 11:04:26 +0300 Subject: [PATCH 39/43] Fix documentation --- client/network/src/config.rs | 2 +- .../src/protocol/notifications/service/mod.rs | 4 ++-- client/network/src/service/traits.rs | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 877b2dc0784da..f9cb7f3dd7cfc 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -482,7 +482,7 @@ pub struct NonDefaultSetConfig { /// Notification handle. /// /// Notification handle is created during `NonDefaultSetConfig` creation and its other half, - /// Box` is given to the protocol created the config and + /// `Box` is given to the protocol created the config and /// `ProtocolHandle` is given to `Notifications` when it initializes itself. This handle allows /// `Notifications ` to communicate with the protocol directly without relaying events through /// `sc-network.` diff --git a/client/network/src/protocol/notifications/service/mod.rs b/client/network/src/protocol/notifications/service/mod.rs index fdf76594dd1f6..05aa8fc41615c 100644 --- a/client/network/src/protocol/notifications/service/mod.rs +++ b/client/network/src/protocol/notifications/service/mod.rs @@ -60,12 +60,12 @@ type NotificationSink = Arc>; #[async_trait::async_trait] impl MessageSink for NotificationSink { - /// Send synchronous `notification` to the peer associated with this [`MessageService`]. + /// Send synchronous `notification` to the peer associated with this [`MessageSink`]. fn send_sync_notification(&self, notification: Vec) { self.lock().send_sync_notification(notification); } - /// Send an asynchronous `notification` to to the peer associated with this [`MessageService`], + /// Send an asynchronous `notification` to to the peer associated with this [`MessageSink`], /// allowing sender to exercise backpressure. /// /// Returns an error if the peer does not exist. diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 3620089216802..273e8b33e4e5a 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -723,19 +723,19 @@ pub enum NotificationEvent { /// notifications. /// /// Two different flavors of sending options are provided: -/// * synchronous sending ([`ProtocolNotificationService::send_sync_notification()`]) -/// * asynchronous sending ([`ProtocolNotificationService::send_async_notification()`]) +/// * synchronous sending ([`NotificationService::send_sync_notification()`]) +/// * asynchronous sending ([`NotificationService::send_async_notification()`]) /// /// The former is used by protocols are not ready to exercise backpressure and the latter for the /// protocols that can do it. /// /// Both local and remote peer can close the substream at any time. Local peer can do so by calling -/// [`ProtocolNotificationService::close_substream()`] which instrucs `Notifications` to close the +/// [`NotificationService::close_substream()`] which instrucs `Notifications` to close the /// substream. Remote closing the substream is indicated to the local peer by receiving -/// [`NotificationEvent::SubstreamClosed`] event +/// [`NotificationEvent::NotificationStreamClosed`] event /// /// In case the protocol must update its handshake while it's operating (such as updating the best -/// block information), it can do so by calling [`ProtocolNotificationService::set_handshake()`] +/// block information), it can do so by calling [`NotificationService::set_handshake()`] /// which instructs `Notifications` to update the handshake it stored during protocol /// initialization. /// @@ -790,7 +790,7 @@ pub trait NotificationService: Debug + Send { /// /// If protocol cannot use [`NotificationService`] to send notifications to peers and requires, /// e.g., notifications to be sent in another task, the protocol may acquire a [`MessageSink`] -/// object for each peer by calling [`NotificationService::notification_sink()`]. Calling this +/// object for each peer by calling [`NotificationService::message_sink()`]. Calling this /// function returns an object which allows the protocol to send notifications to the remote peer. /// /// Use of this API is discouraged as it's not as performant as sending notifications through From b7c3b6805d5a75076d29aedafebfdd951bcbd91e Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Thu, 10 Aug 2023 11:12:05 +0300 Subject: [PATCH 40/43] Apply review comments --- client/network/sync/src/engine.rs | 49 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index 1add96f17d49e..3eb317b34e168 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -38,7 +38,7 @@ use sc_consensus::import_queue::ImportQueueService; use sc_network::{ config::{FullNetworkConfiguration, NonDefaultSetConfig, ProtocolId}, peer_store::{PeerStoreHandle, PeerStoreProvider}, - service::traits::{NotificationEvent, ValidationResult}, + service::traits::{Direction, NotificationEvent, ValidationResult}, types::ProtocolName, utils::LruHashSet, NotificationService, ReputationChange, @@ -690,6 +690,7 @@ where log::debug!(target: "sync", "New best block imported {:?}/#{}", hash, number); self.chain_sync.update_chain_info(&hash, number); + // TODO: remove once `SyncingEngine` is refactored while let Poll::Pending = self .notification_service .set_handshake( @@ -748,7 +749,7 @@ where match event { NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => { let validation_result = self - .validate_connection(&peer, handshake, true) + .validate_connection(&peer, handshake, Direction::Inbound) .map_or(ValidationResult::Reject, |_| ValidationResult::Accept); let _ = result_tx.send(validation_result); @@ -761,12 +762,9 @@ where "Substream opened for {peer}, handshake {handshake:?}" ); - match self.validate_connection(&peer, handshake, direction.is_inbound()) { + match self.validate_connection(&peer, handshake, direction) { Ok(handshake) => { - if self - .on_sync_peer_connected(peer, &handshake, direction.is_inbound()) - .is_err() - { + if self.on_sync_peer_connected(peer, &handshake, direction).is_err() { log::debug!(target: LOG_TARGET, "Failed to register peer {peer}"); self.network_service.disconnect_peer( peer, @@ -774,18 +772,6 @@ where ); } }, - // TODO: `wrong_genesis` is a really ugly hack to work around the issue that - // that `PeerStore` thinks the peer is connected when in fact it can be - // still be under validation. If the peer has different genesis than the - // local node the validation fails but the peer cannot be reported in - // `validate_connection()` as that is also called by - // `ValiateInboundSubstream` which means that the peer is still being - // validated and banning the peer when handling that event would - // result in peer getting dropped twice. - // - // The proper way to fix this is to integrate `ProtocolController` more - // tightly with `NotificationService` or add an additional API call for - // banning pre-accepted peers (which is not desirable) Err(wrong_genesis) => { log::debug!(target: LOG_TARGET, "`SyncingEngine` rejected {peer}"); @@ -918,12 +904,24 @@ where Ok(handshake) } - /// Validate inbound connection. + /// Validate connection. + // NOTE Returning `Err(bool)` is a really ugly hack to work around the issue that + // that `PeerStore` thinks the peer is connected when in fact it can be + // still be under validation. If the peer has different genesis than the + // local node the validation fails but the peer cannot be reported in + // `validate_connection()` as that is also called by + // `ValiateInboundSubstream` which means that the peer is still being + // validated and banning the peer when handling that event would + // result in peer getting dropped twice. + // + // The proper way to fix this is to integrate `ProtocolController` more + // tightly with `NotificationService` or add an additional API call for + // banning pre-accepted peers (which is not desirable) fn validate_connection( &mut self, peer_id: &PeerId, handshake: Vec, - inbound: bool, + direction: Direction, ) -> Result, bool> { log::trace!(target: LOG_TARGET, "New peer {peer_id} {handshake:?}"); @@ -954,7 +952,8 @@ where // make sure to accept no more than `--in-peers` many full nodes if !no_slot_peer && handshake.roles.is_full() && - inbound && self.num_in_peers == self.max_in_peers + direction.is_inbound() && + self.num_in_peers == self.max_in_peers { log::debug!(target: LOG_TARGET, "All inbound slots have been consumed, rejecting {peer_id}"); return Err(false) @@ -980,7 +979,7 @@ where &mut self, who: PeerId, status: &BlockAnnouncesHandshake, - inbound: bool, + direction: Direction, ) -> Result<(), ()> { log::trace!(target: "sync", "New peer {} {:?}", who, status); @@ -995,7 +994,7 @@ where ), last_notification_sent: Instant::now(), last_notification_received: Instant::now(), - inbound, + inbound: direction.is_inbound(), }; let req = if peer.info.roles.is_full() { @@ -1017,7 +1016,7 @@ where if self.default_peers_set_no_slot_peers.contains(&who) { self.default_peers_set_no_slot_connected_peers.insert(who); - } else if inbound && status.roles.is_full() { + } else if direction.is_inbound() && status.roles.is_full() { self.num_in_peers += 1; } From 10bdc287d016551552a118749e856cb485531531 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Thu, 10 Aug 2023 17:06:28 +0300 Subject: [PATCH 41/43] Fix documentation --- client/network/statement/src/lib.rs | 4 ++-- client/network/transactions/src/lib.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/network/statement/src/lib.rs b/client/network/statement/src/lib.rs index 06e4993d52d05..b2094249b65e0 100644 --- a/client/network/statement/src/lib.rs +++ b/client/network/statement/src/lib.rs @@ -21,8 +21,8 @@ //! Usage: //! //! - Use [`StatementHandlerPrototype::new`] to create a prototype. -//! - Pass the return value of [`StatementHandlerPrototype::set_config`] to the network -//! configuration as an extra peers set. +//! - Pass the `NonDefaultSetConfig` returned from [`StatementHandlerPrototype::new`] to the network +//! configuration as an extra peers set. //! - Use [`StatementHandlerPrototype::build`] then [`StatementHandler::run`] to obtain a //! `Future` that processes statements. diff --git a/client/network/transactions/src/lib.rs b/client/network/transactions/src/lib.rs index 93d9312c79cf4..e9c4383ba1326 100644 --- a/client/network/transactions/src/lib.rs +++ b/client/network/transactions/src/lib.rs @@ -21,8 +21,8 @@ //! Usage: //! //! - Use [`TransactionsHandlerPrototype::new`] to create a prototype. -//! - Pass the return value of [`TransactionsHandlerPrototype::set_config`] to the network -//! configuration as an extra peers set. +//! - Pass the `NonDefaultSetConfig` returned from [`TransactionsHandlerPrototype::new`] to the +//! network configuration as an extra peers set. //! - Use [`TransactionsHandlerPrototype::build`] then [`TransactionsHandler::run`] to obtain a //! `Future` that processes transactions. From e5e6af0a725c9eab665582a7219d5650a2858651 Mon Sep 17 00:00:00 2001 From: Aaro Altonen <48052676+altonen@users.noreply.github.com> Date: Fri, 11 Aug 2023 11:14:53 +0300 Subject: [PATCH 42/43] Apply suggestions from code review Co-authored-by: Dmitry Markin --- client/network-gossip/src/bridge.rs | 6 +++--- client/network/src/protocol.rs | 12 ------------ .../src/protocol/notifications/service/metrics.rs | 2 +- .../src/protocol/notifications/service/tests.rs | 2 +- client/network/src/service.rs | 3 --- client/network/src/service/traits.rs | 6 +++--- client/network/sync/src/engine.rs | 6 +++--- client/network/test/src/service.rs | 1 - 8 files changed, 11 insertions(+), 27 deletions(-) diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 3f46cf593a614..0433da966d9f3 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -188,14 +188,14 @@ impl Future for GossipEngine { 'outer: loop { match &mut this.forwarding_state { ForwardingState::Idle => { - let next_notification = this.notification_service.next_event().poll_unpin(cx); + let next_notification_event = this.notification_service.next_event().poll_unpin(cx); let sync_event_stream = this.sync_event_stream.poll_next_unpin(cx); - if next_notification.is_pending() && sync_event_stream.is_pending() { + if next_notification_event.is_pending() && sync_event_stream.is_pending() { break } - match next_notification { + match next_notification_event { Poll::Ready(Some(event)) => match event { NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { let _ = result_tx.send(ValidationResult::Accept); diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 5c0acf3d4228d..0f34a60f03d81 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -388,10 +388,6 @@ impl NetworkBehaviour for Protocol { if self.bad_handshake_substreams.contains(&(peer_id, set_id)) { CustomMessageOutcome::None } else if set_id == HARDCODED_PEERSETS_SYNC { - // let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationSinkReplaced { - // remote: peer_id, - // sink: notifications_sink, - // }); CustomMessageOutcome::None } else { CustomMessageOutcome::NotificationStreamReplaced { @@ -408,10 +404,6 @@ impl NetworkBehaviour for Protocol { // substream, and consequently shouldn't receive a closing event either. CustomMessageOutcome::None } else if set_id == HARDCODED_PEERSETS_SYNC { - // let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationStreamClosed { - // remote: peer_id, - // }); - // self.peers.remove(&peer_id); CustomMessageOutcome::None } else { CustomMessageOutcome::NotificationStreamClosed { @@ -424,10 +416,6 @@ impl NetworkBehaviour for Protocol { if self.bad_handshake_substreams.contains(&(peer_id, set_id)) { CustomMessageOutcome::None } else if set_id == HARDCODED_PEERSETS_SYNC { - // let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationsReceived { - // remote: peer_id, - // messages: vec![message.freeze()], - // }); CustomMessageOutcome::None } else { let protocol_name = self.notification_protocols[usize::from(set_id)].clone(); diff --git a/client/network/src/protocol/notifications/service/metrics.rs b/client/network/src/protocol/notifications/service/metrics.rs index c0206508c1823..65ccedac07621 100644 --- a/client/network/src/protocol/notifications/service/metrics.rs +++ b/client/network/src/protocol/notifications/service/metrics.rs @@ -32,7 +32,7 @@ pub struct Metrics { /// Total number of closed substreams. pub notifications_streams_closed_total: CounterVec, - /// In/outbound notification sizez. + /// In/outbound notification sizes. pub notifications_sizes: HistogramVec, } diff --git a/client/network/src/protocol/notifications/service/tests.rs b/client/network/src/protocol/notifications/service/tests.rs index f750cc4584f25..72e97b79b50f3 100644 --- a/client/network/src/protocol/notifications/service/tests.rs +++ b/client/network/src/protocol/notifications/service/tests.rs @@ -753,7 +753,7 @@ async fn notification_sink_replaced() { ); // now send two notifications using the acquired message sink and verify that - // it also upated t + // it's also updated sink.send_sync_notification(vec![1, 3, 3, 6]); // send an asynchronous notification using the acquired notifications sink. diff --git a/client/network/src/service.rs b/client/network/src/service.rs index d24e3057b705b..eebf8d18149a4 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -98,9 +98,6 @@ pub use behaviour::{InboundFailure, OutboundFailure, ResponseFailure}; pub use libp2p::identity::{DecodingError, Keypair, PublicKey}; pub use protocol::{NotificationCommand, NotificationsSink, ProtocolHandle}; -// #[cfg(test)] -// pub use crate::protocol:: - mod metrics; mod out_events; diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 273e8b33e4e5a..a62283155166c 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -229,7 +229,7 @@ pub trait NetworkPeers { /// Attempt to get peer role. /// - /// Right now the peer role is decoded into the received handshake for all protocols + /// Right now the peer role is decoded from the received handshake for all protocols /// (`/block-announces/1` has other information as well). If the handshake cannot be /// decoded into a role, the role queried from `PeerStore` and if the role is not stored /// there either, `None` is returned and the peer should be discarded. @@ -726,13 +726,13 @@ pub enum NotificationEvent { /// * synchronous sending ([`NotificationService::send_sync_notification()`]) /// * asynchronous sending ([`NotificationService::send_async_notification()`]) /// -/// The former is used by protocols are not ready to exercise backpressure and the latter for the +/// The former is used by the protocols not ready to exercise backpressure and the latter by the /// protocols that can do it. /// /// Both local and remote peer can close the substream at any time. Local peer can do so by calling /// [`NotificationService::close_substream()`] which instrucs `Notifications` to close the /// substream. Remote closing the substream is indicated to the local peer by receiving -/// [`NotificationEvent::NotificationStreamClosed`] event +/// [`NotificationEvent::NotificationStreamClosed`] event. /// /// In case the protocol must update its handshake while it's operating (such as updating the best /// block information), it can do so by calling [`NotificationService::set_handshake()`] diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index 3eb317b34e168..e29bf287136da 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -905,8 +905,8 @@ where } /// Validate connection. - // NOTE Returning `Err(bool)` is a really ugly hack to work around the issue that - // that `PeerStore` thinks the peer is connected when in fact it can be + // NOTE Returning `Err(bool)` is a really ugly hack to work around the issue + // that `ProtocolController` thinks the peer is connected when in fact it can // still be under validation. If the peer has different genesis than the // local node the validation fails but the peer cannot be reported in // `validate_connection()` as that is also called by @@ -1176,7 +1176,7 @@ mod tests { tokio::spawn(engine.run()); - // add maximum number of allowed inbound peers to `SyncingEngine` + // add maximum number of allowed full inbound peers to `SyncingEngine` let num_in_peers = 32; for i in 0..num_in_peers { let peer_id = PeerId::random(); diff --git a/client/network/test/src/service.rs b/client/network/test/src/service.rs index b634c7d9e97d7..b438aa8fffd8b 100644 --- a/client/network/test/src/service.rs +++ b/client/network/test/src/service.rs @@ -504,7 +504,6 @@ async fn notifications_back_pressure() { let receiver = tokio::spawn(async move { let mut received_notifications = 0; - // let mut sync_protocol_name = None; while received_notifications < TOTAL_NOTIFS { match handle2.next_event().await.unwrap() { From 25c12b23fa779837e74ad619f2aa8411a3accd3e Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Fri, 11 Aug 2023 12:17:07 +0300 Subject: [PATCH 43/43] Apply review comments --- client/network-gossip/src/bridge.rs | 3 ++- client/network/src/peer_store.rs | 11 ++++++----- client/network/src/protocol.rs | 5 ++--- .../network/src/protocol/notifications/behaviour.rs | 13 ++++++++++--- client/network/sync/src/engine.rs | 8 +++++--- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 0433da966d9f3..4319debb8a4ec 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -188,7 +188,8 @@ impl Future for GossipEngine { 'outer: loop { match &mut this.forwarding_state { ForwardingState::Idle => { - let next_notification_event = this.notification_service.next_event().poll_unpin(cx); + let next_notification_event = + this.notification_service.next_event().poll_unpin(cx); let sync_event_stream = this.sync_event_stream.poll_next_unpin(cx); if next_notification_event.is_pending() && sync_event_stream.is_pending() { diff --git a/client/network/src/peer_store.rs b/client/network/src/peer_store.rs index 90bb9bb115fc6..cb793e9c1e6a4 100644 --- a/client/network/src/peer_store.rs +++ b/client/network/src/peer_store.rs @@ -265,12 +265,13 @@ impl PeerStoreInner { fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole) { log::trace!(target: LOG_TARGET, "Set {peer_id} role to {role:?}"); - match self.peers.get_mut(peer_id) { - Some(info) => { - info.role = Some(role); + match self.peers.entry(*peer_id) { + Entry::Occupied(mut entry) => { + entry.get_mut().role = Some(role); + }, + Entry::Vacant(entry) => { + entry.insert(PeerInfo { role: Some(role), ..Default::default() }); }, - None => - log::debug!(target: LOG_TARGET, "Failed to set role for {peer_id}, peer doesn't exist"), } } diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 0f34a60f03d81..20b774976540a 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -384,7 +384,7 @@ impl NetworkBehaviour for Protocol { CustomMessageOutcome::None } }, - NotificationsOut::CustomProtocolReplaced { peer_id, notifications_sink, set_id } => { + NotificationsOut::CustomProtocolReplaced { peer_id, notifications_sink, set_id } => if self.bad_handshake_substreams.contains(&(peer_id, set_id)) { CustomMessageOutcome::None } else if set_id == HARDCODED_PEERSETS_SYNC { @@ -395,8 +395,7 @@ impl NetworkBehaviour for Protocol { protocol: self.notification_protocols[usize::from(set_id)].clone(), notifications_sink, } - } - }, + }, NotificationsOut::CustomProtocolClosed { peer_id, set_id } => { if self.bad_handshake_substreams.remove(&(peer_id, set_id)) { // The substream that has just been closed had been opened with a bad diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index b4166eea342c9..408b61f4b352f 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -1003,15 +1003,20 @@ impl Notifications { } } + /// Function that is called when `ProtocolController` wants us to reject an incoming peer. + fn peerset_report_reject(&mut self, index: IncomingIndex) { + let _ = self.report_reject(index); + } + /// Function that is called when the protocol wants us to reject an incoming peer. fn protocol_report_reject(&mut self, index: IncomingIndex) { - if let Some((set_id, peer_id)) = self.peerset_report_reject(index) { + if let Some((set_id, peer_id)) = self.report_reject(index) { self.protocol_controller_handles[usize::from(set_id)].dropped(peer_id) } } /// Function that is called when the peerset wants us to reject an incoming peer. - fn peerset_report_reject(&mut self, index: IncomingIndex) -> Option<(SetId, PeerId)> { + fn report_reject(&mut self, index: IncomingIndex) -> Option<(SetId, PeerId)> { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) @@ -1846,9 +1851,11 @@ impl NetworkBehaviour for Notifications { let event = NotificationsOut::CustomProtocolReplaced { peer_id, set_id, - notifications_sink: replacement_sink, + notifications_sink: replacement_sink.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); + let _ = self.protocol_handles[usize::from(set_id)] + .report_notification_sink_replaced(peer_id, replacement_sink); } *entry.into_mut() = PeerState::Enabled { connections }; diff --git a/client/network/sync/src/engine.rs b/client/network/sync/src/engine.rs index e29bf287136da..c3c36805d4550 100644 --- a/client/network/sync/src/engine.rs +++ b/client/network/sync/src/engine.rs @@ -789,7 +789,7 @@ where }, NotificationEvent::NotificationReceived { peer, notification } => { if !self.peers.contains_key(&peer) { - log::trace!( + log::error!( target: LOG_TARGET, "received notification from {peer} who had been earlier refused by `SyncingEngine`", ); @@ -855,7 +855,6 @@ where } self.chain_sync.peer_disconnected(&peer); - self.default_peers_set_no_slot_connected_peers.remove(&peer); self.event_streams .retain(|stream| stream.unbounded_send(SyncEvent::PeerDisconnected(peer)).is_ok()); } @@ -959,10 +958,13 @@ where return Err(false) } + // make sure that all slots are not occupied by light peers + // + // `ChainSync` only accepts full peers whereas `SyncingEngine` accepts both full and light + // peers. Verify that there is a slot in `SyncingEngine` for the inbound light peer if handshake.roles.is_light() && (self.peers.len() - self.chain_sync.num_peers()) >= self.default_peers_set_num_light { - // Make sure that not all slots are occupied by light clients. log::debug!(target: LOG_TARGET, "Too many light nodes, rejecting {peer_id}"); return Err(false) }