From 78d3406a7f77751706cf025519dffa02dd60c42e Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Tue, 10 Sep 2019 16:24:01 +0300 Subject: [PATCH 01/14] quick and dirty concept in progress --- src/either.rs | 2 +- src/net.rs | 4 +- src/net/connection.rs | 28 ++++++- src/net/events.rs | 40 ++++++++++ src/net/managers.rs | 146 ++++++++++++++++++++++++++++++++++ src/net/socket.rs | 131 +++++++++++++++--------------- src/net/virtual_connection.rs | 9 ++- 7 files changed, 286 insertions(+), 74 deletions(-) create mode 100644 src/net/managers.rs diff --git a/src/either.rs b/src/either.rs index 5b00ec96..d430cc15 100644 --- a/src/either.rs +++ b/src/either.rs @@ -1,4 +1,4 @@ -pub(crate) enum Either { +pub enum Either { Left(L), Right(R), } diff --git a/src/net.rs b/src/net.rs index ede86d33..48a4d018 100644 --- a/src/net.rs +++ b/src/net.rs @@ -1,6 +1,7 @@ //! This module provides the logic between the low-level abstract types and the types that the user will be interacting with. //! You can think of the socket, connection management, congestion control. +mod managers; mod connection; mod events; mod link_conditioner; @@ -10,7 +11,8 @@ mod virtual_connection; pub mod constants; -pub use self::events::SocketEvent; +pub use self::managers::{ConnectionManager, SocketManager}; +pub use self::events::{SocketEvent, ConnectionSendEvent, ConnectionReceiveEvent }; pub use self::link_conditioner::LinkConditioner; pub use self::quality::{NetworkQuality, RttMeasurer}; pub use self::socket::Socket; diff --git a/src/net/connection.rs b/src/net/connection.rs index 2ee26360..81d8e1b0 100644 --- a/src/net/connection.rs +++ b/src/net/connection.rs @@ -1,11 +1,11 @@ -pub use crate::net::{NetworkQuality, RttMeasurer, VirtualConnection}; +pub use crate::net::{NetworkQuality, RttMeasurer, VirtualConnection, ConnectionManager}; use crate::config::Config; use crate::either::Either::{self, Left, Right}; use std::{ collections::HashMap, net::SocketAddr, - time::{Duration, Instant}, + time::{Duration, Instant}, }; /// Maintains a registry of active "connections". Essentially, when we receive a packet on the @@ -29,12 +29,14 @@ impl ActiveConnections { address: SocketAddr, config: &Config, time: Instant, + state_manager: Box, ) -> &mut VirtualConnection { self.connections .entry(address) - .or_insert_with(|| VirtualConnection::new(address, config, time)) + .or_insert_with(|| VirtualConnection::new(address, config, time, state_manager)) } + /// Try to get or create a [VirtualConnection] by address. If the connection does not exist, it will be /// created and returned, but not inserted into the table of active connections. pub(crate) fn get_or_create_connection( @@ -42,14 +44,19 @@ impl ActiveConnections { address: SocketAddr, config: &Config, time: Instant, + state_manager: Box, ) -> Either<&mut VirtualConnection, VirtualConnection> { if let Some(connection) = self.connections.get_mut(&address) { Left(connection) } else { - Right(VirtualConnection::new(address, config, time)) + Right(VirtualConnection::new(address, config, time, state_manager)) } } + pub fn try_get(&mut self, address: &SocketAddr) -> Option<&mut VirtualConnection> { + self.connections.get_mut(address) + } + /// Removes the connection from `ActiveConnections` by socket address. pub fn remove_connection( &mut self, @@ -102,12 +109,25 @@ impl ActiveConnections { #[cfg(test)] mod tests { + use super::{ActiveConnections, Config}; use std::{ sync::Arc, time::{Duration, Instant}, }; + use super::managers::ConnectionManager; + + + #[derive(Debug)] + struct DummyConnManager { + } + + impl ConnectionManager for DummyConnManager { + + } + + const ADDRESS: &str = "127.0.0.1:12345"; #[test] diff --git a/src/net/events.rs b/src/net/events.rs index 3bd9de54..c05bae56 100644 --- a/src/net/events.rs +++ b/src/net/events.rs @@ -1,4 +1,5 @@ use crate::packet::Packet; +use crate::error::ErrorKind; use std::net::SocketAddr; /// Events that can occur in `laminar` and that will be pushed through the `event_receiver` returned by `Socket::bind`. @@ -13,3 +14,42 @@ pub enum SocketEvent { /// You can control the timeout in the config. Timeout(SocketAddr), } + +#[derive(Debug)] +pub enum DisconnectReason { + // TODO here can be all sorts of reasons + Timeout, + TooManyPacketsInFlight, + TooManyPacketErrors, + ClosedByClient, + ClosedByHost, +} + +#[derive(Debug)] +pub enum DestroyReason { + // because disconnected + Disconnected(DisconnectReason), + // because wasnt able to connect + HandshakeFailed(ErrorKind) +} + +#[derive(Debug)] +pub enum ConnectionReceiveEvent { + /// When the connection is actually added to active connections list. + Created, + /// When connection manager changes to connected state + Connected, + /// Actual received packet, this should only occure after Connected state + Packet(Packet), + /// When connection manager changes to disconnected state + Disconnected(DisconnectReason), + /// When connection is actually removed from connections list. + Destroyed(DestroyReason) +} + +#[derive(Debug)] +pub enum ConnectionSendEvent { + Connect, + Packet(Packet), + Disconnect, +} \ No newline at end of file diff --git a/src/net/managers.rs b/src/net/managers.rs new file mode 100644 index 00000000..dee6521d --- /dev/null +++ b/src/net/managers.rs @@ -0,0 +1,146 @@ +pub use crate::either::Either; +use crate::net::events::ConnectionSendEvent; +use crate::packet::Packet; +use std::fmt::Debug; +use std::net::SocketAddr; + +#[derive(Debug)] +pub enum ConnectionManagerState { + Connecting, + Connected, + Disconnected, +} + +pub enum PacketsToSend<'a> { + None, + Same(&'a Packet), + New(Vec), +} + +pub struct ProcessPacketResult<'a> { + pub packets: PacketsToSend<'a>, + pub state: ConnectionManagerState, +} + +impl<'a> ProcessPacketResult<'a> { + pub fn new(packets: PacketsToSend<'a>, state: ConnectionManagerState) -> Self { + Self { + packets, + state, + } + } +} + +/// This is higher level abstraction allows to implement encrypt, custom authentication schemes, etc. +/// It can initiate handshaking process +/// Client and host messages will only passthrough when connection is in Connected state +pub trait ConnectionManager: Debug { + /// Pre-process incoming raw data, can be useful data needs to be decrypted before actually parsing packet + fn preprocess_incoming<'s, 'a>( + &'s self, + data: &'a [u8], + ) -> Result>, String>; + /// Post-process outgoing data, can be useful to encrypt data before sending + fn postprocess_outgoing<'s, 'a>(&'s self, data: &'a [u8]) -> Either<&'a [u8], Vec>; + /// Process incoming packet: + /// * can generate new packets for sending + /// * always returns current state + fn process_incoming<'s, 'a>( + &'s mut self, + packet: &'a Packet, + ) -> Result, String>; + /// Process incoming packet: + /// * can generate new packets for sending + /// * always returns current state + fn process_outgoing<'s, 'a>( + &'s mut self, + event: &'a ConnectionSendEvent, + ) -> ProcessPacketResult<'a>; +} + +/// Tracks all sorts of global statistics, and decided whether to create `ConnectionManager` for new connections or not. +pub trait SocketManager: Debug { + // answers very important question, + // can we accept/create new connection, it also accepts raw bytes of the first packet, so it can do additional decision based on this + fn accept_new_connection( + &mut self, + addr: &SocketAddr, + bytes: &[u8], + ) -> Option>; + + // all sorts of statistics might be useful here to help deciding whether new connection can be created or not + fn track_connection_error(&mut self, addr: &SocketAddr, error: String) {} + fn track_sent_bytes(&mut self, addr: &SocketAddr, bytes: usize); + fn track_received_bytes(&mut self, addr: &SocketAddr, bytes: usize); + fn track_ignored_bytes(&mut self, addr: &SocketAddr, bytes: usize); + fn track_connection_destroyed(&mut self, addr: &SocketAddr); +} + +/// dumbies implementation that is always in connected state and simply resends all packets +#[derive(Debug)] +struct DumbConnectionManager; + +impl ConnectionManager for DumbConnectionManager { + // simply return same data + fn preprocess_incoming<'s, 'a>( + &'s self, + data: &'a [u8], + ) -> Result>, String> { + Ok(Either::Left(data)) + } + + // simply return same data + fn postprocess_outgoing<'s, 'a>(&'s self, data: &'a [u8]) -> Either<&'a [u8], Vec> { + Either::Left(data) + } + + // this forwards all packets and always stays in connected state + fn process_incoming<'s, 'a>( + &'s mut self, + packet: &'a Packet, + ) -> Result, String> { + Ok(ProcessPacketResult::new( + PacketsToSend::None, + ConnectionManagerState::Connected, + )) + } + // ignore connect and disconnect events, and simply forward the data + fn process_outgoing<'s, 'a>( + &'s mut self, + event: &'a ConnectionSendEvent, + ) -> ProcessPacketResult<'a> { + match &event { + ConnectionSendEvent::Packet(data) => ProcessPacketResult::new( + PacketsToSend::Same(data), + ConnectionManagerState::Connected, + ), + _ => ProcessPacketResult::new(PacketsToSend::None, ConnectionManagerState::Connected), + } + } +} + +/// simples implementation of socket manager +/// it only does one thing, creates new connection only when he "properly" greets :) +#[derive(Debug)] +struct DumbSocketManager; + +impl SocketManager for DumbSocketManager { + fn accept_new_connection( + &mut self, + addr: &SocketAddr, + bytes: &[u8], + ) -> Option> { + if bytes == "Yo, man, its me!".as_bytes() { + Some(Box::new(DumbConnectionManager)) + } else { + println!("Ignore this address, until he properly greets;)"); + None + } + } + + fn track_connection_error(&mut self, addr: &SocketAddr, error: String) {} + fn track_sent_bytes(&mut self, addr: &SocketAddr, bytes: usize) {} + fn track_received_bytes(&mut self, addr: &SocketAddr, bytes: usize) {} + fn track_ignored_bytes(&mut self, addr: &SocketAddr, bytes: usize) {} + fn track_connection_destroyed(&mut self, addr: &SocketAddr) {} +} diff --git a/src/net/socket.rs b/src/net/socket.rs index f1a7ce96..2b88c53a 100644 --- a/src/net/socket.rs +++ b/src/net/socket.rs @@ -1,8 +1,8 @@ -use crate::either::Either::{Left, Right}; use crate::{ config::Config, error::{ErrorKind, Result}, net::{connection::ActiveConnections, events::SocketEvent, link_conditioner::LinkConditioner}, + net::managers::{SocketManager}, packet::{DeliveryGuarantee, Outgoing, Packet}, }; use crossbeam_channel::{self, unbounded, Receiver, SendError, Sender, TryRecvError}; @@ -27,6 +27,7 @@ pub struct Socket { receiver: Receiver, sender: Sender, + manager: Box, } enum UdpSocketState { @@ -38,21 +39,21 @@ impl Socket { /// Binds to the socket and then sets up `ActiveConnections` to manage the "connections". /// Because UDP connections are not persistent, we can only infer the status of the remote /// endpoint by looking to see if they are still sending packets or not - pub fn bind(addresses: A) -> Result { - Socket::bind_with_config(addresses, Config::default()) + pub fn bind(addresses: A, manager: Box) -> Result { + Socket::bind_with_config(addresses, Config::default(), manager) } /// Bind to any local port on the system, if available - pub fn bind_any() -> Result { - Self::bind_any_with_config(Config::default()) + pub fn bind_any(manager: Box) -> Result { + Self::bind_any_with_config(Config::default(), manager) } /// Bind to any local port on the system, if available, with a given config - pub fn bind_any_with_config(config: Config) -> Result { + pub fn bind_any_with_config(config: Config, manager: Box) -> Result { let loopback = Ipv4Addr::new(127, 0, 0, 1); let address = SocketAddrV4::new(loopback, 0); let socket = UdpSocket::bind(address)?; - Self::bind_internal(socket, config) + Self::bind_internal(socket, config, manager) } /// Binds to the socket and then sets up `ActiveConnections` to manage the "connections". @@ -60,16 +61,16 @@ impl Socket { /// endpoint by looking to see if they are still sending packets or not /// /// This function allows you to configure laminar with the passed configuration. - pub fn bind_with_config(addresses: A, config: Config) -> Result { + pub fn bind_with_config(addresses: A, config: Config, manager: Box) -> Result { let socket = UdpSocket::bind(addresses)?; - Self::bind_internal(socket, config) + Self::bind_internal(socket, config, manager) } - fn bind_internal(socket: UdpSocket, config: Config) -> Result { + fn bind_internal(socket: UdpSocket, config: Config, manager: Box) -> Result { socket.set_nonblocking(!config.blocking_mode)?; let (event_sender, event_receiver) = unbounded(); let (packet_sender, packet_receiver) = unbounded(); - Ok(Socket { + Ok(Socket { recv_buffer: vec![0; config.receive_buffer_max_size], socket, config, @@ -80,6 +81,8 @@ impl Socket { sender: packet_sender, receiver: event_receiver, + + manager, }) } @@ -242,48 +245,46 @@ impl Socket { } // Serializes and sends a `Packet` on the socket. On success, returns the number of bytes written. - fn send_to(&mut self, packet: Packet, time: Instant) -> Result { - let connection = - self.connections - .get_or_insert_connection(packet.addr(), &self.config, time); - - let dropped = connection.gather_dropped_packets(); - let mut processed_packets: Vec = dropped - .iter() - .flat_map(|waiting_packet| { - connection.process_outgoing( - &waiting_packet.payload, - // Because a delivery guarantee is only sent with reliable packets - DeliveryGuarantee::Reliable, - // This is stored with the dropped packet because they could be mixed - waiting_packet.ordering_guarantee, - waiting_packet.item_identifier, - time, - ) - }) - .collect(); - - let processed_packet = connection.process_outgoing( - packet.payload(), - packet.delivery_guarantee(), - packet.order_guarantee(), - None, - time, - )?; - - processed_packets.push(processed_packet); - + fn send_to(&mut self, packet: Packet, time: Instant) -> Result { let mut bytes_sent = 0; + let address = &packet.addr(); + if let Some(connection) = self.connections.try_get(address) { + let dropped = connection.gather_dropped_packets(); + let mut processed_packets: Vec = dropped + .iter() + .flat_map(|waiting_packet| { + connection.process_outgoing( + &waiting_packet.payload, + // Because a delivery guarantee is only sent with reliable packets + DeliveryGuarantee::Reliable, + // This is stored with the dropped packet because they could be mixed + waiting_packet.ordering_guarantee, + waiting_packet.item_identifier, + time, + ) + }) + .collect(); + + let processed_packet = connection.process_outgoing( + packet.payload(), + packet.delivery_guarantee(), + packet.order_guarantee(), + None, + time, + )?; - for processed_packet in processed_packets { - if self.should_send_packet() { - match processed_packet { - Outgoing::Packet(outgoing) => { - bytes_sent += self.send_packet(&packet.addr(), &outgoing.contents())?; - } - Outgoing::Fragments(packets) => { - for outgoing in packets { - bytes_sent += self.send_packet(&packet.addr(), &outgoing.contents())?; + processed_packets.push(processed_packet); + + for processed_packet in processed_packets { + if self.should_send_packet() { + match processed_packet { + Outgoing::Packet(outgoing) => { + bytes_sent += self.send_packet(address, &outgoing.contents())?; + } + Outgoing::Fragments(packets) => { + for outgoing in packets { + bytes_sent += self.send_packet(address, &outgoing.contents())?; + } } } } @@ -301,21 +302,21 @@ impl Socket { } let received_payload = &self.recv_buffer[..recv_len]; - if !self.connections.exists(&address) { - self.event_sender.send(SocketEvent::Connect(address))?; - } - - let connection = - self.connections - .get_or_create_connection(address, &self.config, time); + let connection = if let Some(conn) = self.connections.try_get(&address) { + Some(conn) + } else { + if let Some(manager) = self.manager.accept_new_connection(&address, received_payload) { + // TODO emit create event + self.event_sender.send(SocketEvent::Connect(address))?; + Some(self.connections.get_or_insert_connection(address, &self.config, time, manager)) + } else { + None + } + }; - match connection { - Left(existing) => { - existing.process_incoming(received_payload, &self.event_sender, time)?; - } - Right(mut anonymous) => { - anonymous.process_incoming(received_payload, &self.event_sender, time)?; - } + if let Some(conn) = connection + { + conn.process_incoming(received_payload, &self.event_sender, time)?; } } Err(e) => { diff --git a/src/net/virtual_connection.rs b/src/net/virtual_connection.rs index a56f5b21..60466e93 100644 --- a/src/net/virtual_connection.rs +++ b/src/net/virtual_connection.rs @@ -7,12 +7,13 @@ use crate::{ }, net::constants::{ ACKED_PACKET_HEADER, DEFAULT_ORDERING_STREAM, DEFAULT_SEQUENCING_STREAM, - STANDARD_HEADER_SIZE, + STANDARD_HEADER_SIZE, }, + net::managers::ConnectionManager, packet::{ DeliveryGuarantee, OrderingGuarantee, Outgoing, OutgoingPacket, OutgoingPacketBuilder, Packet, PacketReader, PacketType, SequenceNumber, - }, + }, SocketEvent, }; @@ -38,11 +39,12 @@ pub struct VirtualConnection { config: Config, fragmentation: Fragmentation, + state_manager: Box, } impl VirtualConnection { /// Creates and returns a new Connection that wraps the provided socket address - pub fn new(addr: SocketAddr, config: &Config, time: Instant) -> VirtualConnection { + pub fn new(addr: SocketAddr, config: &Config, time: Instant, state_manager: Box) -> VirtualConnection { VirtualConnection { last_heard: time, last_sent: time, @@ -53,6 +55,7 @@ impl VirtualConnection { congestion_handler: CongestionHandler::new(config), fragmentation: Fragmentation::new(config), config: config.to_owned(), + state_manager, } } From 57079793ab871a06a3eaa30b5de4dc6bc1819f3c Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Wed, 11 Sep 2019 12:25:44 +0300 Subject: [PATCH 02/14] WIP exploring ConnectionManager design --- src/error.rs | 14 +++++ src/lib.rs | 1 + src/net.rs | 3 +- src/net/connection.rs | 2 +- src/net/managers.rs | 54 +++++++++++++---- src/net/virtual_connection.rs | 111 ++++++++++++++++++---------------- 6 files changed, 120 insertions(+), 65 deletions(-) diff --git a/src/error.rs b/src/error.rs index c6203219..d5d3d1d5 100644 --- a/src/error.rs +++ b/src/error.rs @@ -7,6 +7,7 @@ use std::{ fmt::{self, Display, Formatter}, io, result, }; +use crate::net::managers::ConnectionManagerError; /// Wrapped result type for Laminar errors. pub type Result = result::Result; @@ -30,6 +31,8 @@ pub enum ErrorKind { SendError(SendError), /// Expected header but could not be read from buffer. CouldNotReadHeader(String), + /// Errors that is returned from ConnectionManager either preprocessing data or processing packet + ConnectionError(ConnectionManagerError), } impl Display for ErrorKind { @@ -67,6 +70,11 @@ impl Display for ErrorKind { "Expected {} header but could not be read from buffer.", header ), + ErrorKind::ConnectionError(err) => write!( + fmt, + "Something when wrong in ConnectionManager. Reason: {:?}.", + err + ) } } } @@ -179,6 +187,12 @@ impl From> for ErrorKind { } } +impl From for ErrorKind { + fn from(inner: ConnectionManagerError) -> Self { + ErrorKind::ConnectionError(inner) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/lib.rs b/src/lib.rs index 1f830b77..0d0bfe66 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,3 +41,4 @@ pub use self::config::Config; pub use self::error::{ErrorKind, Result}; pub use self::net::{LinkConditioner, Socket, SocketEvent}; pub use self::packet::{DeliveryGuarantee, OrderingGuarantee, Packet}; +pub use self::net::managers::{ConnectionManager, ConnectionManagerError}; diff --git a/src/net.rs b/src/net.rs index 48a4d018..2c229348 100644 --- a/src/net.rs +++ b/src/net.rs @@ -1,7 +1,6 @@ //! This module provides the logic between the low-level abstract types and the types that the user will be interacting with. //! You can think of the socket, connection management, congestion control. -mod managers; mod connection; mod events; mod link_conditioner; @@ -10,8 +9,8 @@ mod socket; mod virtual_connection; pub mod constants; +pub mod managers; -pub use self::managers::{ConnectionManager, SocketManager}; pub use self::events::{SocketEvent, ConnectionSendEvent, ConnectionReceiveEvent }; pub use self::link_conditioner::LinkConditioner; pub use self::quality::{NetworkQuality, RttMeasurer}; diff --git a/src/net/connection.rs b/src/net/connection.rs index 81d8e1b0..15eb1169 100644 --- a/src/net/connection.rs +++ b/src/net/connection.rs @@ -1,4 +1,4 @@ -pub use crate::net::{NetworkQuality, RttMeasurer, VirtualConnection, ConnectionManager}; +pub use crate::net::{NetworkQuality, RttMeasurer, VirtualConnection, managers::ConnectionManager}; use crate::config::Config; use crate::either::Either::{self, Left, Right}; diff --git a/src/net/managers.rs b/src/net/managers.rs index dee6521d..ebf2c07b 100644 --- a/src/net/managers.rs +++ b/src/net/managers.rs @@ -4,13 +4,36 @@ use crate::packet::Packet; use std::fmt::Debug; use std::net::SocketAddr; -#[derive(Debug)] -pub enum ConnectionManagerState { +/// At any given moment, any connection can be only in these states. +/// These states are only managed through `ConnectionManager`, and define behaviour for sending and receiving packets. +/// Only these state transition is allowed: +/// | Old | New | +/// | ---------- | ---------- | +/// | Connecting | Connected | +/// | Connecting | Disconnected | +/// | Connected | Disconnected | +/// | Disconnected | Connecting | +/// If these rules are not satisfied, panic! will be called. +/// Each state specifies what can and cannot be done: +/// * Connecting - This is initial state when socket is created, at this moment no `events` can be sent or received, +/// in this state `ConnectionManager` is able to receive and sent `packets` to properly initiate connection. +/// * Connected - Only in this state all events will be sent or received between peers. +/// * Disconnected - in this state `ConnectionManager` is not able to send or receive any packets. +/// It can only process incoming events and decide if it can reset connection and change to Connecting state, +/// otherwise connection will be closed when all packets-in-flight finishes sending or after connection timeout. +#[derive(Debug, PartialEq)] +pub enum ConnectionState { Connecting, Connected, Disconnected, } +impl Default for ConnectionState { + fn default() -> Self { + Self::Connecting + } +} + pub enum PacketsToSend<'a> { None, Same(&'a Packet), @@ -19,11 +42,11 @@ pub enum PacketsToSend<'a> { pub struct ProcessPacketResult<'a> { pub packets: PacketsToSend<'a>, - pub state: ConnectionManagerState, + pub state: ConnectionState, } impl<'a> ProcessPacketResult<'a> { - pub fn new(packets: PacketsToSend<'a>, state: ConnectionManagerState) -> Self { + pub fn new(packets: PacketsToSend<'a>, state: ConnectionState) -> Self { Self { packets, state, @@ -31,6 +54,12 @@ impl<'a> ProcessPacketResult<'a> { } } +#[derive(Debug)] +pub enum ConnectionManagerError { + UnableToPreprocess(String), + HandshakeError(String) +} + /// This is higher level abstraction allows to implement encrypt, custom authentication schemes, etc. /// It can initiate handshaking process /// Client and host messages will only passthrough when connection is in Connected state @@ -39,7 +68,7 @@ pub trait ConnectionManager: Debug { fn preprocess_incoming<'s, 'a>( &'s self, data: &'a [u8], - ) -> Result>, String>; + ) -> Result>, ConnectionManagerError>; /// Post-process outgoing data, can be useful to encrypt data before sending fn postprocess_outgoing<'s, 'a>(&'s self, data: &'a [u8]) -> Either<&'a [u8], Vec>; /// Process incoming packet: @@ -48,7 +77,7 @@ pub trait ConnectionManager: Debug { fn process_incoming<'s, 'a>( &'s mut self, packet: &'a Packet, - ) -> Result, String>; + ) -> Result, ConnectionManagerError>; /// Process incoming packet: /// * can generate new packets for sending /// * always returns current state @@ -76,6 +105,9 @@ pub trait SocketManager: Debug { fn track_connection_destroyed(&mut self, addr: &SocketAddr); } + + + /// dumbies implementation that is always in connected state and simply resends all packets #[derive(Debug)] struct DumbConnectionManager; @@ -85,7 +117,7 @@ impl ConnectionManager for DumbConnectionManager { fn preprocess_incoming<'s, 'a>( &'s self, data: &'a [u8], - ) -> Result>, String> { + ) -> Result>, ConnectionManagerError> { Ok(Either::Left(data)) } @@ -98,10 +130,10 @@ impl ConnectionManager for DumbConnectionManager { fn process_incoming<'s, 'a>( &'s mut self, packet: &'a Packet, - ) -> Result, String> { + ) -> Result, ConnectionManagerError> { Ok(ProcessPacketResult::new( PacketsToSend::None, - ConnectionManagerState::Connected, + ConnectionState::Connected, )) } // ignore connect and disconnect events, and simply forward the data @@ -112,9 +144,9 @@ impl ConnectionManager for DumbConnectionManager { match &event { ConnectionSendEvent::Packet(data) => ProcessPacketResult::new( PacketsToSend::Same(data), - ConnectionManagerState::Connected, + ConnectionState::Connected, ), - _ => ProcessPacketResult::new(PacketsToSend::None, ConnectionManagerState::Connected), + _ => ProcessPacketResult::new(PacketsToSend::None, ConnectionState::Connected), } } } diff --git a/src/net/virtual_connection.rs b/src/net/virtual_connection.rs index 60466e93..9fb92ea0 100644 --- a/src/net/virtual_connection.rs +++ b/src/net/virtual_connection.rs @@ -1,6 +1,7 @@ use crate::{ config::Config, error::{ErrorKind, PacketErrorKind, Result}, + either::Either, infrastructure::{ arranging::{Arranging, ArrangingSystem, OrderingSystem, SequencingSystem}, AcknowledgmentHandler, CongestionHandler, Fragmentation, SentPacket, @@ -9,7 +10,7 @@ use crate::{ ACKED_PACKET_HEADER, DEFAULT_ORDERING_STREAM, DEFAULT_SEQUENCING_STREAM, STANDARD_HEADER_SIZE, }, - net::managers::ConnectionManager, + net::managers::{ConnectionManager, ConnectionState}, packet::{ DeliveryGuarantee, OrderingGuarantee, Outgoing, OutgoingPacket, OutgoingPacketBuilder, Packet, PacketReader, PacketType, SequenceNumber, @@ -40,6 +41,7 @@ pub struct VirtualConnection { config: Config, fragmentation: Fragmentation, state_manager: Box, + current_state: ConnectionState, } impl VirtualConnection { @@ -56,6 +58,7 @@ impl VirtualConnection { fragmentation: Fragmentation::new(config), config: config.to_owned(), state_manager, + current_state: ConnectionState::Connecting } } @@ -235,12 +238,24 @@ impl VirtualConnection { } } - /// This processes the incoming data and returns a packet if the data is complete. + pub fn process_incoming( &mut self, received_data: &[u8], sender: &Sender, time: Instant, + ) -> crate::Result<()> { + match self.state_manager.preprocess_incoming(received_data)? { + Either::Left(bytes) => self.process_incoming_impl(bytes, sender, time), + Either::Right(bytes) => self.process_incoming_impl(bytes.as_slice(), sender, time) + } + } + /// This processes the incoming data and returns a packet if the data is complete. + fn process_incoming_impl( + &mut self, + received_data: &[u8], + sender: &Sender, + time: Instant, ) -> crate::Result<()> { self.last_heard = time; @@ -271,25 +286,20 @@ impl VirtualConnection { .get_or_create_stream(arranging_header.stream_id()); if let Some(packet) = stream.arrange(arranging_header.arranging_id(), payload) { - Self::queue_packet( + self.process_packet( sender, - packet, - self.remote_address, + packet, header.delivery_guarantee(), - OrderingGuarantee::Sequenced(Some(arranging_header.stream_id())), - )?; + OrderingGuarantee::Sequenced(Some(arranging_header.stream_id())))?; } return Ok(()); } - - Self::queue_packet( + self.process_packet( sender, - packet_reader.read_payload(), - self.remote_address, + packet_reader.read_payload(), header.delivery_guarantee(), - header.ordering_guarantee(), - )?; + header.ordering_guarantee())?; } DeliveryGuarantee::Reliable => { if header.is_fragment() { @@ -301,10 +311,9 @@ impl VirtualConnection { .handle_fragment(fragment_header, &payload) { Ok(Some(payload)) => { - Self::queue_packet( + self.process_packet( sender, payload.into_boxed_slice(), - self.remote_address, header.delivery_guarantee(), OrderingGuarantee::None, )?; @@ -340,10 +349,9 @@ impl VirtualConnection { if let Some(packet) = stream.arrange(arranging_header.arranging_id(), payload) { - Self::queue_packet( + self.process_packet( sender, packet, - self.remote_address, header.delivery_guarantee(), OrderingGuarantee::Sequenced(Some(arranging_header.stream_id())), )?; @@ -358,35 +366,24 @@ impl VirtualConnection { let stream = self .ordering_system .get_or_create_stream(arranging_header.stream_id()); - - if let Some(packet) = - stream.arrange(arranging_header.arranging_id(), payload) - { - Self::queue_packet( + let arranged_packet = stream.arrange(arranging_header.arranging_id(), payload); + let packets = arranged_packet + .into_iter() + .chain(stream.iter_mut()) + .collect::>(); + for packet in packets { + self.process_packet( sender, - packet, - self.remote_address, - header.delivery_guarantee(), - OrderingGuarantee::Ordered(Some(arranging_header.stream_id())), - )?; - - while let Some(packet) = stream.iter_mut().next() { - Self::queue_packet( - sender, - packet, - self.remote_address, - header.delivery_guarantee(), - OrderingGuarantee::Ordered(Some(arranging_header.stream_id())), - )?; - } + packet, + header.delivery_guarantee(), + OrderingGuarantee::Ordered(Some(arranging_header.stream_id())))?; } } else { let payload = packet_reader.read_payload(); - Self::queue_packet( + self.process_packet( sender, payload, - self.remote_address, header.delivery_guarantee(), header.ordering_guarantee(), )?; @@ -406,19 +403,31 @@ impl VirtualConnection { Ok(()) } - fn queue_packet( - tx: &Sender, - payload: Box<[u8]>, - remote_addr: SocketAddr, - delivery: DeliveryGuarantee, - ordering: OrderingGuarantee, + fn process_packet( + &mut self, + sender: &Sender, + payload: Box<[u8]>, + delivery: DeliveryGuarantee, + ordering: OrderingGuarantee ) -> Result<()> { - tx.send(SocketEvent::Packet(Packet::new( - remote_addr, - payload, - delivery, - ordering, - )))?; + let packet = Packet::new(self.remote_address, payload,delivery, ordering); + let result = self.state_manager.process_incoming(&packet)?; + + if self.current_state == result.state { + if self.current_state == ConnectionState::Connected { + sender.send(SocketEvent::Packet(packet))?; + } + } else { + match (&self.current_state, &result.state) { + (ConnectionState::Connecting, ConnectionState::Connected) => "", + (ConnectionState::Connecting, ConnectionState::Disconnected) => "", + (ConnectionState::Connected, ConnectionState::Disconnected) => "", + (ConnectionState::Disconnected, ConnectionState::Connecting) => "", + _ => panic!("Invalid state transition {:?} -> {:?}", self.current_state, result.state) + }; + self.current_state = result.state; + } + // todo implement ability to return generated packets Ok(()) } From b348f59bedc8dffb41673067c21ca14602fcb3cd Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Thu, 12 Sep 2019 07:38:09 +0300 Subject: [PATCH 03/14] WIP refining ConnectionManager design --- src/lib.rs | 2 +- src/net.rs | 2 +- src/net/connection.rs | 37 ++++++++- src/net/events.rs | 47 +++++++----- src/net/managers.rs | 108 ++++++++++----------------- src/net/socket.rs | 13 +--- src/net/virtual_connection.rs | 103 +++++++++++++++++++++---- src/packet/enums.rs | 3 + src/packet/header/standard_header.rs | 5 ++ 9 files changed, 203 insertions(+), 117 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0d0bfe66..5bfc19bc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,6 +39,6 @@ pub use self::throughput::ThroughputMonitoring; pub use self::config::Config; pub use self::error::{ErrorKind, Result}; -pub use self::net::{LinkConditioner, Socket, SocketEvent}; +pub use self::net::{LinkConditioner, Socket, SocketEvent, DisconnectReason}; pub use self::packet::{DeliveryGuarantee, OrderingGuarantee, Packet}; pub use self::net::managers::{ConnectionManager, ConnectionManagerError}; diff --git a/src/net.rs b/src/net.rs index 2c229348..a96d2bd0 100644 --- a/src/net.rs +++ b/src/net.rs @@ -11,7 +11,7 @@ mod virtual_connection; pub mod constants; pub mod managers; -pub use self::events::{SocketEvent, ConnectionSendEvent, ConnectionReceiveEvent }; +pub use self::events::{SocketEvent, DisconnectReason, DestroyReason, ConnectionSendEvent, ConnectionReceiveEvent }; pub use self::link_conditioner::LinkConditioner; pub use self::quality::{NetworkQuality, RttMeasurer}; pub use self::socket::Socket; diff --git a/src/net/connection.rs b/src/net/connection.rs index 15eb1169..c231bda2 100644 --- a/src/net/connection.rs +++ b/src/net/connection.rs @@ -1,7 +1,13 @@ pub use crate::net::{NetworkQuality, RttMeasurer, VirtualConnection, managers::ConnectionManager}; +use crate::{ + config::Config, + either::Either::{self, Left, Right}, + net::events::{SocketEvent, DisconnectReason, DestroyReason}, + net::managers::{ConnectionState, SocketManager}, + packet::Packet, +}; +use crossbeam_channel::{self, Sender, SendError}; -use crate::config::Config; -use crate::either::Either::{self, Left, Right}; use std::{ collections::HashMap, net::SocketAddr, @@ -61,8 +67,20 @@ impl ActiveConnections { pub fn remove_connection( &mut self, address: &SocketAddr, - ) -> Option<(SocketAddr, VirtualConnection)> { - self.connections.remove_entry(address) + sender: &Sender, + manager: &mut dyn SocketManager, + reason: DestroyReason + ) -> Result> { + if let Some((_, conn)) = self.connections.remove_entry(address) { + manager.track_connection_destroyed(address); + if conn.get_current_state() == ConnectionState::Connected { + sender.send(SocketEvent::Disconnected(DisconnectReason::UnrecoverableError(reason.clone())))?; + } + sender.send(SocketEvent::Destroyed(reason))?; + Ok(true) + } else { + Ok(false) + } } /// Check for and return `VirtualConnection`s which have been idling longer than `max_idle_time`. @@ -95,6 +113,17 @@ impl ActiveConnections { .map(|(_, connection)| connection) } + pub fn connection_manager_generated_packets( + &mut self + ) -> Vec { + // TODO implement + vec!{} + // self.connections + // .iter_mut() + // .map(|(_, connection)| connection.get_connection_manager_packets()) + // .flatten() + } + /// Returns true if the given connection exists. pub fn exists(&self, address: &SocketAddr) -> bool { self.connections.contains_key(&address) diff --git a/src/net/events.rs b/src/net/events.rs index c05bae56..982f0eaf 100644 --- a/src/net/events.rs +++ b/src/net/events.rs @@ -1,36 +1,43 @@ use crate::packet::Packet; -use crate::error::ErrorKind; +use crate::net::managers::ConnectionManagerError; use std::net::SocketAddr; /// Events that can occur in `laminar` and that will be pushed through the `event_receiver` returned by `Socket::bind`. -#[derive(Debug, PartialEq)] -pub enum SocketEvent { - /// A packet was received from a client. - Packet(Packet), - /// A new client connected. - /// Clients are uniquely identified by the ip:port combination at this layer. - Connect(SocketAddr), - /// The client has been idling for a configurable amount of time. - /// You can control the timeout in the config. - Timeout(SocketAddr), -} +// #[derive(Debug, PartialEq)] +// pub enum SocketEvent { +// /// A packet was received from a client. +// Packet(Packet), +// /// A new client connected. +// /// Clients are uniquely identified by the ip:port combination at this layer. +// Connect(SocketAddr), +// /// The client has been idling for a configurable amount of time. +// /// You can control the timeout in the config. +// Timeout(SocketAddr), +// } -#[derive(Debug)] -pub enum DisconnectReason { - // TODO here can be all sorts of reasons +#[derive(Debug, PartialEq, Clone)] +pub enum DestroyReason { + // because wasnt able to connect + HandshakeFailed(ConnectionManagerError), Timeout, TooManyPacketsInFlight, TooManyPacketErrors, +} + +#[derive(Debug, PartialEq)] +pub enum DisconnectReason { ClosedByClient, ClosedByHost, + UnrecoverableError(DestroyReason) } -#[derive(Debug)] -pub enum DestroyReason { - // because disconnected +#[derive(Debug, PartialEq)] +pub enum SocketEvent { + Created(SocketAddr), + Connected, + Packet(Packet), Disconnected(DisconnectReason), - // because wasnt able to connect - HandshakeFailed(ErrorKind) + Destroyed(DestroyReason), } #[derive(Debug)] diff --git a/src/net/managers.rs b/src/net/managers.rs index ebf2c07b..f49f290a 100644 --- a/src/net/managers.rs +++ b/src/net/managers.rs @@ -11,17 +11,17 @@ use std::net::SocketAddr; /// | ---------- | ---------- | /// | Connecting | Connected | /// | Connecting | Disconnected | -/// | Connected | Disconnected | +/// | Connected | Disconnected | /// | Disconnected | Connecting | /// If these rules are not satisfied, panic! will be called. /// Each state specifies what can and cannot be done: -/// * Connecting - This is initial state when socket is created, at this moment no `events` can be sent or received, +/// * Connecting - This is initial state when socket is created, at this moment no `events` can be sent or received, /// in this state `ConnectionManager` is able to receive and sent `packets` to properly initiate connection. /// * Connected - Only in this state all events will be sent or received between peers. -/// * Disconnected - in this state `ConnectionManager` is not able to send or receive any packets. -/// It can only process incoming events and decide if it can reset connection and change to Connecting state, +/// * Disconnected - in this state `ConnectionManager` is not able to send or receive any packets. +/// It can only process incoming events and decide if it can reset connection and change to Connecting state, /// otherwise connection will be closed when all packets-in-flight finishes sending or after connection timeout. -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] pub enum ConnectionState { Connecting, Connected, @@ -34,35 +34,18 @@ impl Default for ConnectionState { } } -pub enum PacketsToSend<'a> { - None, - Same(&'a Packet), - New(Vec), -} - -pub struct ProcessPacketResult<'a> { - pub packets: PacketsToSend<'a>, - pub state: ConnectionState, -} - -impl<'a> ProcessPacketResult<'a> { - pub fn new(packets: PacketsToSend<'a>, state: ConnectionState) -> Self { - Self { - packets, - state, - } - } -} - -#[derive(Debug)] -pub enum ConnectionManagerError { - UnableToPreprocess(String), - HandshakeError(String) -} - -/// This is higher level abstraction allows to implement encrypt, custom authentication schemes, etc. -/// It can initiate handshaking process -/// Client and host messages will only passthrough when connection is in Connected state +/// Generic error type, that is used by ConnectionManager implementation +#[derive(Debug, PartialEq, Clone)] +pub struct ConnectionManagerError(String); + +/// It abstracts pure UDP packets, and allows to implement Connected/Disconnected states. +/// This table summary shows where exactly ConnectionManager sits in layer hierarchy. +/// | Abstraction layer | Capabilities | +/// |-------------------|-----------------------------------------------------------------------------------------------------------------------------------| +/// | Application | Can receive these events: Created->Connected->Packet->Disconnected->Destroyed, And send these events: Connect->Packet->Disconnect | +/// | ConnectionManager | Receive raw Packet events, and manage connection state, can also generate Packets to initiate connection | +/// | Laminar | Handles raw UDP bytes and provides reliability, ordering, fragmentation, etc.. capabilities | +/// | ConnectionManager | May change raw incoming and outgoing bytes | pub trait ConnectionManager: Debug { /// Pre-process incoming raw data, can be useful data needs to be decrypted before actually parsing packet fn preprocess_incoming<'s, 'a>( @@ -74,17 +57,19 @@ pub trait ConnectionManager: Debug { /// Process incoming packet: /// * can generate new packets for sending /// * always returns current state - fn process_incoming<'s, 'a>( - &'s mut self, - packet: &'a Packet, - ) -> Result, ConnectionManagerError>; - /// Process incoming packet: + fn process_incoming( + &mut self, + packet: &Packet, + createPacket: &mut dyn FnMut(Packet) -> Result<(), String> + ) -> Result; + /// Process outgoing packet: /// * can generate new packets for sending /// * always returns current state - fn process_outgoing<'s, 'a>( - &'s mut self, - event: &'a ConnectionSendEvent, - ) -> ProcessPacketResult<'a>; + fn process_outgoing( + &mut self, + event: &ConnectionSendEvent, + createPacket: &mut dyn FnMut(Packet) -> Result<(), String> + ) -> ConnectionState; } /// Tracks all sorts of global statistics, and decided whether to create `ConnectionManager` for new connections or not. @@ -105,9 +90,6 @@ pub trait SocketManager: Debug { fn track_connection_destroyed(&mut self, addr: &SocketAddr); } - - - /// dumbies implementation that is always in connected state and simply resends all packets #[derive(Debug)] struct DumbConnectionManager; @@ -126,28 +108,20 @@ impl ConnectionManager for DumbConnectionManager { Either::Left(data) } - // this forwards all packets and always stays in connected state - fn process_incoming<'s, 'a>( - &'s mut self, - packet: &'a Packet, - ) -> Result, ConnectionManagerError> { - Ok(ProcessPacketResult::new( - PacketsToSend::None, - ConnectionState::Connected, - )) + fn process_incoming( + &mut self, + packet: &Packet, + createPacket: &mut dyn FnMut(Packet) -> Result<(), String> + ) -> Result { + Ok(ConnectionState::Connected) } - // ignore connect and disconnect events, and simply forward the data - fn process_outgoing<'s, 'a>( - &'s mut self, - event: &'a ConnectionSendEvent, - ) -> ProcessPacketResult<'a> { - match &event { - ConnectionSendEvent::Packet(data) => ProcessPacketResult::new( - PacketsToSend::Same(data), - ConnectionState::Connected, - ), - _ => ProcessPacketResult::new(PacketsToSend::None, ConnectionState::Connected), - } + + fn process_outgoing( + &mut self, + event: &ConnectionSendEvent, + createPacket: &mut dyn FnMut(Packet) -> Result<(), String> + ) -> ConnectionState { + ConnectionState::Connected } } diff --git a/src/net/socket.rs b/src/net/socket.rs index 2b88c53a..4025dd85 100644 --- a/src/net/socket.rs +++ b/src/net/socket.rs @@ -1,7 +1,7 @@ use crate::{ config::Config, error::{ErrorKind, Result}, - net::{connection::ActiveConnections, events::SocketEvent, link_conditioner::LinkConditioner}, + net::{connection::ActiveConnections, events::SocketEvent, events::DestroyReason, link_conditioner::LinkConditioner}, net::managers::{SocketManager}, packet::{DeliveryGuarantee, Outgoing, Packet}, }; @@ -193,10 +193,8 @@ impl Socket { fn handle_dead_clients(&mut self) -> Result<()> { let dead_addresses = self.connections.dead_connections(); for address in dead_addresses { - self.connections.remove_connection(&address); - self.event_sender.send(SocketEvent::Timeout(address))?; + self.connections.remove_connection(&address, &self.event_sender, self.manager.as_mut(), DestroyReason::TooManyPacketsInFlight)?; } - Ok(()) } @@ -208,10 +206,8 @@ impl Socket { .connections .idle_connections(self.config.idle_connection_timeout, time); for address in idle_addresses { - self.connections.remove_connection(&address); - self.event_sender.send(SocketEvent::Timeout(address))?; + self.connections.remove_connection(&address, &self.event_sender, self.manager.as_mut(), DestroyReason::Timeout)?; } - Ok(()) } @@ -306,8 +302,7 @@ impl Socket { Some(conn) } else { if let Some(manager) = self.manager.accept_new_connection(&address, received_payload) { - // TODO emit create event - self.event_sender.send(SocketEvent::Connect(address))?; + self.event_sender.send(SocketEvent::Created(address))?; Some(self.connections.get_or_insert_connection(address, &self.config, time, manager)) } else { None diff --git a/src/net/virtual_connection.rs b/src/net/virtual_connection.rs index 9fb92ea0..c5f8cf4f 100644 --- a/src/net/virtual_connection.rs +++ b/src/net/virtual_connection.rs @@ -14,14 +14,15 @@ use crate::{ packet::{ DeliveryGuarantee, OrderingGuarantee, Outgoing, OutgoingPacket, OutgoingPacketBuilder, Packet, PacketReader, PacketType, SequenceNumber, - }, - SocketEvent, + }, + SocketEvent, DisconnectReason }; use crossbeam_channel::{self, Sender}; use std::fmt; use std::net::SocketAddr; use std::time::{Duration, Instant}; +use std::mem; /// Contains the information about a certain 'virtual connection' over udp. /// This connections also keeps track of network quality, processing packets, buffering data related to connection etc. @@ -42,6 +43,7 @@ pub struct VirtualConnection { fragmentation: Fragmentation, state_manager: Box, current_state: ConnectionState, + conn_manager_packets: Vec } impl VirtualConnection { @@ -58,7 +60,8 @@ impl VirtualConnection { fragmentation: Fragmentation::new(config), config: config.to_owned(), state_manager, - current_state: ConnectionState::Connecting + current_state: ConnectionState::Connecting, + conn_manager_packets: Default::default() } } @@ -81,6 +84,10 @@ impl VirtualConnection { time.duration_since(self.last_sent) } + pub fn get_current_state(&self) -> ConnectionState { + self.current_state + } + /// This will create a heartbeat packet that is expected to be sent over the network pub fn create_and_process_heartbeat(&mut self, time: Instant) -> OutgoingPacket<'static> { self.last_sent = time; @@ -96,7 +103,27 @@ impl VirtualConnection { .build() } - /// This will pre-process the given buffer to be sent over the network. + /// This will return all packets that are generated by connection manager + pub fn get_connection_manager_packets(&mut self, time: Instant) -> Result>> { + // TODO implement + Ok(vec!{}) + // Ok(self.conn_manager_packets + // .into_iter() + // .map(|packet| { + // match self.process_outgoing_impl( + // packet.payload(),packet.delivery_guarantee(), + // packet.order_guarantee(),None,time) { + // Ok(out) => Some(out), + // Err(err) => { + // None + // } + // } + // .into_iter() + // }) + // .flatten() + // .collect()) + } + pub fn process_outgoing<'a>( &mut self, payload: &'a [u8], @@ -104,6 +131,18 @@ impl VirtualConnection { ordering_guarantee: OrderingGuarantee, last_item_identifier: Option, time: Instant, + ) -> Result> { + self.process_outgoing_impl(payload, delivery_guarantee, ordering_guarantee, last_item_identifier, time) + } + + /// This will pre-process the given buffer to be sent over the network. + fn process_outgoing_impl<'a>( + &mut self, + payload: &'a [u8], + delivery_guarantee: DeliveryGuarantee, + ordering_guarantee: OrderingGuarantee, + last_item_identifier: Option, + time: Instant, ) -> Result> { match delivery_guarantee { DeliveryGuarantee::Unreliable => { @@ -287,6 +326,7 @@ impl VirtualConnection { if let Some(packet) = stream.arrange(arranging_header.arranging_id(), payload) { self.process_packet( + header.is_connection_manager(), sender, packet, header.delivery_guarantee(), @@ -296,6 +336,7 @@ impl VirtualConnection { return Ok(()); } self.process_packet( + header.is_connection_manager(), sender, packet_reader.read_payload(), header.delivery_guarantee(), @@ -312,6 +353,7 @@ impl VirtualConnection { { Ok(Some(payload)) => { self.process_packet( + header.is_connection_manager(), sender, payload.into_boxed_slice(), header.delivery_guarantee(), @@ -350,6 +392,7 @@ impl VirtualConnection { stream.arrange(arranging_header.arranging_id(), payload) { self.process_packet( + header.is_connection_manager(), sender, packet, header.delivery_guarantee(), @@ -373,6 +416,7 @@ impl VirtualConnection { .collect::>(); for packet in packets { self.process_packet( + header.is_connection_manager(), sender, packet, header.delivery_guarantee(), @@ -382,6 +426,7 @@ impl VirtualConnection { let payload = packet_reader.read_payload(); self.process_packet( + header.is_connection_manager(), sender, payload, header.delivery_guarantee(), @@ -403,34 +448,62 @@ impl VirtualConnection { Ok(()) } + /// Pass packet to ConnectionManager, check for state changes, and return any newly generated packets fn process_packet( &mut self, + is_connection_packet: bool, sender: &Sender, payload: Box<[u8]>, delivery: DeliveryGuarantee, ordering: OrderingGuarantee ) -> Result<()> { let packet = Packet::new(self.remote_address, payload,delivery, ordering); - let result = self.state_manager.process_incoming(&packet)?; - - if self.current_state == result.state { - if self.current_state == ConnectionState::Connected { + //let mut new_packets = vec!{}; + let new_state = self.state_manager.process_incoming(&packet, &mut |new_packet| { + // TODO refactor so, that i could remove self from here + self.process_outgoing_impl(new_packet.payload(),new_packet.delivery_guarantee(), + new_packet.order_guarantee(),None,Instant::now()); + //new_packets.push(new); + Ok(()) + })?; + + if self.current_state == new_state { + if self.current_state == ConnectionState::Connected && !is_connection_packet { sender.send(SocketEvent::Packet(packet))?; } } else { - match (&self.current_state, &result.state) { - (ConnectionState::Connecting, ConnectionState::Connected) => "", - (ConnectionState::Connecting, ConnectionState::Disconnected) => "", - (ConnectionState::Connected, ConnectionState::Disconnected) => "", - (ConnectionState::Disconnected, ConnectionState::Connecting) => "", - _ => panic!("Invalid state transition {:?} -> {:?}", self.current_state, result.state) + match (&self.current_state, new_state) { + (ConnectionState::Connecting, ConnectionState::Connected) => { + sender.send(SocketEvent::Connected)?; + if !is_connection_packet { + sender.send(SocketEvent::Packet(packet))?; + } + }, + (ConnectionState::Connecting, ConnectionState::Disconnected) => (),// no need to send any event, because socket is not connected and is will be destroyed later + (ConnectionState::Connected, ConnectionState::Disconnected) => sender.send(SocketEvent::Disconnected(DisconnectReason::ClosedByClient))?, + (ConnectionState::Disconnected, ConnectionState::Connecting) => self.reset_connection(), + _ => panic!("Invalid state transition {:?} -> {:?}", self.current_state, new_state) }; - self.current_state = result.state; + self.current_state = new_state; } // todo implement ability to return generated packets Ok(()) } + /// this function fully reset connection to initial state, after ConnectionManager switches from Disconnected to Connecting + fn reset_connection(&mut self) { + let time = Instant::now(); + self.last_heard = time; + self.last_sent= time; + self.ordering_system = OrderingSystem::new(); + self.sequencing_system = SequencingSystem::new(); + self.acknowledge_handler = AcknowledgmentHandler::new(); + self.congestion_handler = CongestionHandler::new(&self.config); + self.fragmentation = Fragmentation::new(&self.config); + self.current_state = ConnectionState::Connecting; + self.conn_manager_packets.clear(); + } + /// This will gather dropped packets from the acknowledgment handler. /// /// Note that after requesting dropped packets the dropped packets will be removed from this client. diff --git a/src/packet/enums.rs b/src/packet/enums.rs index 454f3634..d579f2e9 100644 --- a/src/packet/enums.rs +++ b/src/packet/enums.rs @@ -90,6 +90,8 @@ pub enum PacketType { Fragment = 1, /// Heartbeat packet Heartbeat = 2, + /// ConnectionManager specific header + ConnectionManager = 3 } impl EnumConverter for PacketType { @@ -107,6 +109,7 @@ impl TryFrom for PacketType { 0 => Ok(PacketType::Packet), 1 => Ok(PacketType::Fragment), 2 => Ok(PacketType::Heartbeat), + 3 => Ok(PacketType::ConnectionManager), _ => Err(ErrorKind::DecodingError(DecodingErrorKind::PacketType)), } } diff --git a/src/packet/header/standard_header.rs b/src/packet/header/standard_header.rs index 4eccaafe..c9ef955b 100644 --- a/src/packet/header/standard_header.rs +++ b/src/packet/header/standard_header.rs @@ -63,6 +63,11 @@ impl StandardHeader { self.packet_type == PacketType::Fragment } + // Returns true if packet is only used ConnectionManager + pub fn is_connection_manager(&self) -> bool { + self.packet_type == PacketType::ConnectionManager + } + /// Checks if the protocol version in the packet is a valid version pub fn is_current_protocol(&self) -> bool { ProtocolVersion::valid_version(self.protocol_version) From 2c39a0c7971e674e24bb48fc995a3ecfbfaad287 Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Fri, 13 Sep 2019 15:27:30 +0300 Subject: [PATCH 04/14] WIP once again, working on connection manager design --- src/net/connection.rs | 49 +++++-- src/net/events.rs | 11 +- src/net/managers.rs | 242 +++++++++++++++++++++++----------- src/net/socket.rs | 13 +- src/net/virtual_connection.rs | 72 +++++----- 5 files changed, 258 insertions(+), 129 deletions(-) diff --git a/src/net/connection.rs b/src/net/connection.rs index c231bda2..3d6278ee 100644 --- a/src/net/connection.rs +++ b/src/net/connection.rs @@ -4,7 +4,8 @@ use crate::{ either::Either::{self, Left, Right}, net::events::{SocketEvent, DisconnectReason, DestroyReason}, net::managers::{ConnectionState, SocketManager}, - packet::Packet, + packet::{Packet, OutgoingPacket}, + ErrorKind, }; use crossbeam_channel::{self, Sender, SendError}; @@ -19,12 +20,15 @@ use std::{ #[derive(Debug)] pub struct ActiveConnections { connections: HashMap, + buffer: Box<[u8]> } impl ActiveConnections { pub fn new() -> Self { Self { connections: HashMap::new(), + // TODO actually take from config + buffer: Box::new([0;1500]) } } @@ -73,7 +77,7 @@ impl ActiveConnections { ) -> Result> { if let Some((_, conn)) = self.connections.remove_entry(address) { manager.track_connection_destroyed(address); - if conn.get_current_state() == ConnectionState::Connected { + if let ConnectionState::Connected(_) = conn.get_current_state() { sender.send(SocketEvent::Disconnected(DisconnectReason::UnrecoverableError(reason.clone())))?; } sender.send(SocketEvent::Destroyed(reason))?; @@ -113,15 +117,38 @@ impl ActiveConnections { .map(|(_, connection)| connection) } - pub fn connection_manager_generated_packets( - &mut self - ) -> Vec { - // TODO implement - vec!{} - // self.connections - // .iter_mut() - // .map(|(_, connection)| connection.get_connection_manager_packets()) - // .flatten() + pub fn update_connections( + &mut self, + mut buffer: &mut [u8], + sender: &Sender, + manager:&mut dyn SocketManager + ) -> Vec<(SocketAddr, Box<[u8]>)> { + let time = Instant::now(); + self.connections + .iter_mut() + .filter_map(|(_, conn)| match conn.state_manager.update(&mut buffer, time) { + Some(result) => match result { + Ok(event) => match event { + Either::Left(packet) => Some((conn.remote_address, packet.contents())), + Either::Right(state) => { + conn.current_state = state.clone(); + if let Err(err) = match &conn.current_state { + ConnectionState::Connected(data) => sender.send(SocketEvent::Connected(conn.remote_address, data.clone())), + ConnectionState::Disconnected(closed_by) => sender.send(SocketEvent::Disconnected(DisconnectReason::ClosedBy(closed_by.clone()))), + _ => Ok(()),} { + manager.track_global_error(&ErrorKind::SendError(err)); + } + None + } + }, + Err(err) => { + manager.track_connection_error(&conn.remote_address, &ErrorKind::ConnectionError(err)); + None + } + }, + None => None + }) + .collect() } /// Returns true if the given connection exists. diff --git a/src/net/events.rs b/src/net/events.rs index 982f0eaf..50cf62db 100644 --- a/src/net/events.rs +++ b/src/net/events.rs @@ -24,17 +24,22 @@ pub enum DestroyReason { TooManyPacketErrors, } +#[derive(Debug, PartialEq, Clone)] +pub enum ConnectionClosedBy { + LocalHost, + RemoteHost +} + #[derive(Debug, PartialEq)] pub enum DisconnectReason { - ClosedByClient, - ClosedByHost, + ClosedBy(ConnectionClosedBy), UnrecoverableError(DestroyReason) } #[derive(Debug, PartialEq)] pub enum SocketEvent { Created(SocketAddr), - Connected, + Connected(SocketAddr, Box<[u8]>), Packet(Packet), Disconnected(DisconnectReason), Destroyed(DestroyReason), diff --git a/src/net/managers.rs b/src/net/managers.rs index f49f290a..82464a76 100644 --- a/src/net/managers.rs +++ b/src/net/managers.rs @@ -1,8 +1,10 @@ pub use crate::either::Either; -use crate::net::events::ConnectionSendEvent; -use crate::packet::Packet; +pub use crate::net::events::{ConnectionClosedBy, DestroyReason}; +use crate::packet::OutgoingPacket; +use crate::ErrorKind; use std::fmt::Debug; use std::net::SocketAddr; +use std::time::Instant; /// At any given moment, any connection can be only in these states. /// These states are only managed through `ConnectionManager`, and define behaviour for sending and receiving packets. @@ -21,11 +23,26 @@ use std::net::SocketAddr; /// * Disconnected - in this state `ConnectionManager` is not able to send or receive any packets. /// It can only process incoming events and decide if it can reset connection and change to Connecting state, /// otherwise connection will be closed when all packets-in-flight finishes sending or after connection timeout. -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, PartialEq, Clone)] pub enum ConnectionState { Connecting, - Connected, - Disconnected, + Connected(Box<[u8]>), + Disconnected(ConnectionClosedBy), +} + +impl ConnectionState { + /// Tries to change current state, returns old state if successfully changed. + pub fn try_change(&mut self, new: Self) -> Option { + match (&self, &new) { + (ConnectionState::Connecting, ConnectionState::Connected(_)) + | (ConnectionState::Connecting, ConnectionState::Disconnected(_)) + | (ConnectionState::Connected(_), ConnectionState::Disconnected(_)) + | (ConnectionState::Disconnected(_), ConnectionState::Connecting) => { + Some(std::mem::replace(self, new)) + } + _ => None, + } + } } impl Default for ConnectionState { @@ -39,112 +56,181 @@ impl Default for ConnectionState { pub struct ConnectionManagerError(String); /// It abstracts pure UDP packets, and allows to implement Connected/Disconnected states. -/// This table summary shows where exactly ConnectionManager sits in layer hierarchy. -/// | Abstraction layer | Capabilities | -/// |-------------------|-----------------------------------------------------------------------------------------------------------------------------------| -/// | Application | Can receive these events: Created->Connected->Packet->Disconnected->Destroyed, And send these events: Connect->Packet->Disconnect | -/// | ConnectionManager | Receive raw Packet events, and manage connection state, can also generate Packets to initiate connection | -/// | Laminar | Handles raw UDP bytes and provides reliability, ordering, fragmentation, etc.. capabilities | -/// | ConnectionManager | May change raw incoming and outgoing bytes | +/// This table summary shows where exactly ConnectionManager sits in between different layers. +/// | Abstraction layer | Capabilities | +/// |-------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +/// | Application | Can receive these events: Created->Connected(data)->Packet(data)->Disconnected(reason)->Destroyed(reason) Can send these events: Connect(data)->Packet->Disconnect. | +/// | ConnectionManager | Receive incoming and outgoing packets and manage connection state. Can generate new packets to initiate connection. | +/// | Laminar | Adds/Removes headers to packets, so that it could provides reliability, ordering, fragmentation, etc.. capabilities. | +/// | ConnectionManager | May change raw incoming and outgoing bytes to apply encryption, compression, etc. | +/// +/// Manager can store local buffer size of `Config.receive_buffer_max_size` if it does some kind of encryption. +/// It tries to maintain valid connection state, and it can't decide when to destroy or disconnect a connection itself. +/// Only when packet is recevied, or action is initiated by user it is allowed to change connection state. +/// From the point of view of connection manager, laminar's header + payload is interpreted as user data. +/// Distinction between user packet and protocol specific packet is encoded in laminar's packet header. pub trait ConnectionManager: Debug { - /// Pre-process incoming raw data, can be useful data needs to be decrypted before actually parsing packet - fn preprocess_incoming<'s, 'a>( - &'s self, + /// This function should be called frequently, even if there is no packets to send or receive. + /// It will always be called last, after all other methods is called, so it could send packets or comunicate errors if required. + /// It cannot change connection state explicitly, instead it can emit errors, and SocketManager will decide when to destroy connection. + /// It can generate all kinds of packets: heartbeat, user or connection protocol packets. + /// (maybe heartbeat functionality should be moved here?) + /// `buffer` is used to return packet payload, it's size is `Config.receive_buffer_max_size`. + fn update<'a>( + &mut self, + buffer: &'a mut [u8], + time: Instant, + ) -> Option, &ConnectionState>, ConnectionManagerError>>; + + /// This will be called for all incoming data, including packets that were resent by remote host. + /// If packet is accepted by laminar's reliability layer `process_protocol_data` will be called immediatelly. + /// It should return a slice where header + payload exists + fn preprocess_incoming<'a, 'b>( + &mut self, data: &'a [u8], - ) -> Result>, ConnectionManagerError>; - /// Post-process outgoing data, can be useful to encrypt data before sending - fn postprocess_outgoing<'s, 'a>(&'s self, data: &'a [u8]) -> Either<&'a [u8], Vec>; - /// Process incoming packet: - /// * can generate new packets for sending - /// * always returns current state - fn process_incoming( + buffer: &'b mut [u8], + ) -> Result<&'b [u8], ConnectionManagerError> + where + 'a: 'b; + + /// This will be called for all outgoing data, including packets that are resend. + /// Dropped packets will also go through here. + /// Accepts full packet: header + payload + fn postprocess_outgoing<'a, 'b>(&mut self, data: &'a [u8], buffer: &'b mut [u8]) -> &'b [u8] + where + 'a: 'b; + + /// This will be called only for incoming protocol specific packets, after laminar's reliability layer accepted it. + /// This is the only place where connection state can actually be changed by incomming packet. + fn process_protocol_data<'a>( &mut self, - packet: &Packet, - createPacket: &mut dyn FnMut(Packet) -> Result<(), String> - ) -> Result; - /// Process outgoing packet: - /// * can generate new packets for sending - /// * always returns current state - fn process_outgoing( - &mut self, - event: &ConnectionSendEvent, - createPacket: &mut dyn FnMut(Packet) -> Result<(), String> - ) -> ConnectionState; + data: &'a [u8], + ) -> Result<&ConnectionState, ConnectionManagerError>; + + /// This will be invoked when player sends connect request, + /// Some protocols might provide a way to pass initial connection data + /// Data is user payload, that will be received by remote host, on Connected(data) event + /// This method is not able to send packet immediatelly, instead this functionality should be handled in `update` method. + fn connect<'a>(&mut self, data: &'a [u8]); + + // This will be invoked when player sends disconnect request, + /// This method is not able to send packet immediatelly, instead this functionality should be handled in `update` method. + fn disconnect<'a>(&mut self); } /// Tracks all sorts of global statistics, and decided whether to create `ConnectionManager` for new connections or not. +/// Also decides when connections should be destroyed even if they are in connected state. pub trait SocketManager: Debug { - // answers very important question, - // can we accept/create new connection, it also accepts raw bytes of the first packet, so it can do additional decision based on this - fn accept_new_connection( - &mut self, - addr: &SocketAddr, - bytes: &[u8], - ) -> Option>; + /// Decide if it is possible to accept/create new connection connection + fn accept_new_connection(&mut self, addr: &SocketAddr) -> Option>; + + /// Returns list of connections that socket manager decided to destroy + fn destroy_connections(&mut self) -> Option>; // all sorts of statistics might be useful here to help deciding whether new connection can be created or not - fn track_connection_error(&mut self, addr: &SocketAddr, error: String) {} + fn track_connection_error(&mut self, addr: &SocketAddr, error: &ErrorKind); + fn track_global_error(&mut self, error: &ErrorKind); fn track_sent_bytes(&mut self, addr: &SocketAddr, bytes: usize); fn track_received_bytes(&mut self, addr: &SocketAddr, bytes: usize); fn track_ignored_bytes(&mut self, addr: &SocketAddr, bytes: usize); fn track_connection_destroyed(&mut self, addr: &SocketAddr); } -/// dumbies implementation that is always in connected state and simply resends all packets -#[derive(Debug)] -struct DumbConnectionManager; +/// Simple connection manager, sends "connect" and "disconnect" messages and changes states when receive either of theses messages +#[derive(Debug, Default)] +struct SimpleConnectionManager { + state: ConnectionState, + send: Option>, +} -impl ConnectionManager for DumbConnectionManager { - // simply return same data - fn preprocess_incoming<'s, 'a>( - &'s self, +use crate::packet::{DeliveryGuarantee, OrderingGuarantee, OutgoingPacketBuilder, PacketType}; + +impl ConnectionManager for SimpleConnectionManager { + fn update<'a>( + &mut self, + buffer: &'a mut [u8], + time: Instant, + ) -> Option, &ConnectionState>, ConnectionManagerError>> { + match self.send.take() { + Some(data) => { + // copy from buffer what we want to send + buffer.copy_from_slice(data.as_ref()); + // create packet and set packet type ConnectionManager, so that it can be processed by `process_protocol_data` in remote host + Some(Ok(Either::Left( + OutgoingPacketBuilder::new(buffer) + .with_default_header( + PacketType::ConnectionManager, + DeliveryGuarantee::Reliable, + OrderingGuarantee::Ordered(None), + ) + .build(), + ))) + } + None => None, + } + } + + fn preprocess_incoming<'a, 'b>( + &mut self, data: &'a [u8], - ) -> Result>, ConnectionManagerError> { - Ok(Either::Left(data)) + _buffer: &'b mut [u8], + ) -> Result<&'b [u8], ConnectionManagerError> + where + 'a: 'b, + { + Ok(data) } - // simply return same data - fn postprocess_outgoing<'s, 'a>(&'s self, data: &'a [u8]) -> Either<&'a [u8], Vec> { - Either::Left(data) + fn postprocess_outgoing<'a, 'b>(&mut self, data: &'a [u8], _buffer: &'b mut [u8]) -> &'b [u8] + where + 'a: 'b, + { + data } - fn process_incoming( + fn process_protocol_data<'a>( &mut self, - packet: &Packet, - createPacket: &mut dyn FnMut(Packet) -> Result<(), String> - ) -> Result { - Ok(ConnectionState::Connected) + data: &'a [u8], + ) -> Result<&ConnectionState, ConnectionManagerError> { + if data.starts_with("connect".as_bytes()) { + self.state + .try_change(ConnectionState::Connected(Box::from(data.split_at(7).1))); + } else if data.eq("disconnect".as_bytes()) { + self.state.try_change(ConnectionState::Disconnected(ConnectionClosedBy::RemoteHost)); + } else { + return Err(ConnectionManagerError(format!( + "Unknown message type: {:?}", + std::str::from_utf8(data) + ))); + } + Ok(&self.state) } - fn process_outgoing( - &mut self, - event: &ConnectionSendEvent, - createPacket: &mut dyn FnMut(Packet) -> Result<(), String> - ) -> ConnectionState { - ConnectionState::Connected + fn connect<'a>(&mut self, data: &'a [u8]) { + self.send = Some(Box::from(["connect".as_bytes(), data].concat())); + } + + fn disconnect<'a>(&mut self) { + self.state.try_change(ConnectionState::Disconnected(ConnectionClosedBy::LocalHost)); + self.send = Some(Box::from("disconnect".as_bytes())); } } -/// simples implementation of socket manager -/// it only does one thing, creates new connection only when he "properly" greets :) +/// Simplest implementation of socket manager, always accept connection and never destroy, no matter how many error connection can report #[derive(Debug)] struct DumbSocketManager; impl SocketManager for DumbSocketManager { - fn accept_new_connection( - &mut self, - addr: &SocketAddr, - bytes: &[u8], - ) -> Option> { - if bytes == "Yo, man, its me!".as_bytes() { - Some(Box::new(DumbConnectionManager)) - } else { - println!("Ignore this address, until he properly greets;)"); - None - } + fn accept_new_connection(&mut self, addr: &SocketAddr) -> Option> { + Some(Box::new(SimpleConnectionManager::default())) } - fn track_connection_error(&mut self, addr: &SocketAddr, error: String) {} + fn destroy_connections(&mut self) -> Option> { + None + } + + fn track_connection_error(&mut self, addr: &SocketAddr, error: &ErrorKind) {} + fn track_global_error(&mut self, error: &ErrorKind) {} fn track_sent_bytes(&mut self, addr: &SocketAddr, bytes: usize) {} fn track_received_bytes(&mut self, addr: &SocketAddr, bytes: usize) {} fn track_ignored_bytes(&mut self, addr: &SocketAddr, bytes: usize) {} diff --git a/src/net/socket.rs b/src/net/socket.rs index 4025dd85..2fd0ee15 100644 --- a/src/net/socket.rs +++ b/src/net/socket.rs @@ -159,6 +159,17 @@ impl Socket { } } + // TODO provide real buffer, from config.max_receive_buffer + // update all connections, update connection states and send generated packets + // + let mut buff = [0; 1500]; + for (address, bytes) in self.connections.update_connections(&mut buff, &self.event_sender, self.manager.as_mut()) { + match self.send_packet(&address, &bytes) { + Ok(bytes_sent) => self.manager.track_sent_bytes(&address, bytes_sent), + Err(err) => self.manager.track_connection_error(&address, &err) + } + } + // Check for idle clients if let Err(e) = self.handle_idle_clients(time) { error!("Encountered an error when sending TimeoutEvent: {:?}", e); @@ -301,7 +312,7 @@ impl Socket { let connection = if let Some(conn) = self.connections.try_get(&address) { Some(conn) } else { - if let Some(manager) = self.manager.accept_new_connection(&address, received_payload) { + if let Some(manager) = self.manager.accept_new_connection(&address) { self.event_sender.send(SocketEvent::Created(address))?; Some(self.connections.get_or_insert_connection(address, &self.config, time, manager)) } else { diff --git a/src/net/virtual_connection.rs b/src/net/virtual_connection.rs index c5f8cf4f..359db966 100644 --- a/src/net/virtual_connection.rs +++ b/src/net/virtual_connection.rs @@ -41,9 +41,9 @@ pub struct VirtualConnection { config: Config, fragmentation: Fragmentation, - state_manager: Box, - current_state: ConnectionState, - conn_manager_packets: Vec + pub state_manager: Box, + pub current_state: ConnectionState, + pub conn_manager_packets: Vec } impl VirtualConnection { @@ -84,8 +84,8 @@ impl VirtualConnection { time.duration_since(self.last_sent) } - pub fn get_current_state(&self) -> ConnectionState { - self.current_state + pub fn get_current_state(&self) -> &ConnectionState { + &self.current_state } /// This will create a heartbeat packet that is expected to be sent over the network @@ -284,10 +284,10 @@ impl VirtualConnection { sender: &Sender, time: Instant, ) -> crate::Result<()> { - match self.state_manager.preprocess_incoming(received_data)? { - Either::Left(bytes) => self.process_incoming_impl(bytes, sender, time), - Either::Right(bytes) => self.process_incoming_impl(bytes.as_slice(), sender, time) - } + // TODO pass buffer from somewhere else + let mut buffer = [0;1500]; + let bytes = self.state_manager.preprocess_incoming(received_data, &mut buffer)?; + self.process_incoming_impl(bytes, sender, time) } /// This processes the incoming data and returns a packet if the data is complete. fn process_incoming_impl( @@ -459,33 +459,33 @@ impl VirtualConnection { ) -> Result<()> { let packet = Packet::new(self.remote_address, payload,delivery, ordering); //let mut new_packets = vec!{}; - let new_state = self.state_manager.process_incoming(&packet, &mut |new_packet| { - // TODO refactor so, that i could remove self from here - self.process_outgoing_impl(new_packet.payload(),new_packet.delivery_guarantee(), - new_packet.order_guarantee(),None,Instant::now()); - //new_packets.push(new); - Ok(()) - })?; - - if self.current_state == new_state { - if self.current_state == ConnectionState::Connected && !is_connection_packet { - sender.send(SocketEvent::Packet(packet))?; - } - } else { - match (&self.current_state, new_state) { - (ConnectionState::Connecting, ConnectionState::Connected) => { - sender.send(SocketEvent::Connected)?; - if !is_connection_packet { - sender.send(SocketEvent::Packet(packet))?; - } - }, - (ConnectionState::Connecting, ConnectionState::Disconnected) => (),// no need to send any event, because socket is not connected and is will be destroyed later - (ConnectionState::Connected, ConnectionState::Disconnected) => sender.send(SocketEvent::Disconnected(DisconnectReason::ClosedByClient))?, - (ConnectionState::Disconnected, ConnectionState::Connecting) => self.reset_connection(), - _ => panic!("Invalid state transition {:?} -> {:?}", self.current_state, new_state) - }; - self.current_state = new_state; - } + // let new_state = self.state_manager.process_incoming(&packet, &mut |new_packet| { + // // TODO refactor so, that i could remove self from here + // self.process_outgoing_impl(new_packet.payload(),new_packet.delivery_guarantee(), + // new_packet.order_guarantee(),None,Instant::now()); + // //new_packets.push(new); + // Ok(()) + // })?; + + // if self.current_state == new_state { + // if self.current_state == ConnectionState::Connected && !is_connection_packet { + // sender.send(SocketEvent::Packet(packet))?; + // } + // } else { + // match (&self.current_state, new_state) { + // (ConnectionState::Connecting, ConnectionState::Connected) => { + // sender.send(SocketEvent::Connected)?; + // if !is_connection_packet { + // sender.send(SocketEvent::Packet(packet))?; + // } + // }, + // (ConnectionState::Connecting, ConnectionState::Disconnected) => (),// no need to send any event, because socket is not connected and is will be destroyed later + // (ConnectionState::Connected, ConnectionState::Disconnected) => sender.send(SocketEvent::Disconnected(DisconnectReason::ClosedByClient))?, + // (ConnectionState::Disconnected, ConnectionState::Connecting) => self.reset_connection(), + // _ => panic!("Invalid state transition {:?} -> {:?}", self.current_state, new_state) + // }; + // self.current_state = new_state; + // } // todo implement ability to return generated packets Ok(()) } From 87182bba42f665080fb6960c18ca615ba6b4ee4d Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Fri, 13 Sep 2019 16:26:10 +0300 Subject: [PATCH 05/14] WIP starting integration of connection manager and socket manager --- src/net/connection.rs | 53 +++++---- src/net/managers.rs | 45 +++++--- src/net/socket.rs | 69 +++++------- src/net/virtual_connection.rs | 154 ++++++++++++--------------- src/packet/header/standard_header.rs | 2 +- 5 files changed, 159 insertions(+), 164 deletions(-) diff --git a/src/net/connection.rs b/src/net/connection.rs index 3d6278ee..2e4bf211 100644 --- a/src/net/connection.rs +++ b/src/net/connection.rs @@ -11,7 +11,7 @@ use crossbeam_channel::{self, Sender, SendError}; use std::{ collections::HashMap, - net::SocketAddr, + net::{SocketAddr, UdpSocket}, time::{Duration, Instant}, }; @@ -73,17 +73,22 @@ impl ActiveConnections { address: &SocketAddr, sender: &Sender, manager: &mut dyn SocketManager, - reason: DestroyReason - ) -> Result> { + reason: DestroyReason, + error_context: &str + ) -> bool { if let Some((_, conn)) = self.connections.remove_entry(address) { manager.track_connection_destroyed(address); if let ConnectionState::Connected(_) = conn.get_current_state() { - sender.send(SocketEvent::Disconnected(DisconnectReason::UnrecoverableError(reason.clone())))?; + if let Err(err) = sender.send(SocketEvent::Disconnected(DisconnectReason::UnrecoverableError(reason.clone()))) { + manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(err), error_context); + } } - sender.send(SocketEvent::Destroyed(reason))?; - Ok(true) + if let Err(err) = sender.send(SocketEvent::Destroyed(reason)) { + manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(err), error_context); + } + true } else { - Ok(false) + false } } @@ -109,7 +114,7 @@ impl ActiveConnections { pub fn heartbeat_required_connections( &mut self, heartbeat_interval: Duration, - time: Instant, + time: Instant ) -> impl Iterator { self.connections .iter_mut() @@ -119,36 +124,44 @@ impl ActiveConnections { pub fn update_connections( &mut self, - mut buffer: &mut [u8], sender: &Sender, - manager:&mut dyn SocketManager - ) -> Vec<(SocketAddr, Box<[u8]>)> { + manager:&mut dyn SocketManager, + socket:&UdpSocket + ) { + // TODO provide real buffer, from config.max_receive_buffer + // update all connections, update connection states and send generated packets + let mut buffer = [0; 1500]; + let time = Instant::now(); self.connections .iter_mut() - .filter_map(|(_, conn)| match conn.state_manager.update(&mut buffer, time) { + .for_each(|(_, conn)| match conn.state_manager.update(&mut buffer, time) { Some(result) => match result { Ok(event) => match event { - Either::Left(packet) => Some((conn.remote_address, packet.contents())), + Either::Left(packet) => { + //TODO refactor, so that i could pass here link conditioner as well + match socket.send_to(&packet.contents(), conn.remote_address) { + Ok(bytes_sent) => manager.track_sent_bytes(&conn.remote_address, bytes_sent), + Err(err) => manager.track_connection_error(&conn.remote_address, &ErrorKind::IOError(err), + "sending packet from connection manager") + }; + }, Either::Right(state) => { conn.current_state = state.clone(); if let Err(err) = match &conn.current_state { ConnectionState::Connected(data) => sender.send(SocketEvent::Connected(conn.remote_address, data.clone())), ConnectionState::Disconnected(closed_by) => sender.send(SocketEvent::Disconnected(DisconnectReason::ClosedBy(closed_by.clone()))), _ => Ok(()),} { - manager.track_global_error(&ErrorKind::SendError(err)); + manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(err), "sending connection state update"); } - None } }, Err(err) => { - manager.track_connection_error(&conn.remote_address, &ErrorKind::ConnectionError(err)); - None + manager.track_connection_error(&conn.remote_address, &ErrorKind::ConnectionError(err), "recieved connection manager error"); } }, - None => None - }) - .collect() + None => {} + }); } /// Returns true if the given connection exists. diff --git a/src/net/managers.rs b/src/net/managers.rs index 82464a76..45444d59 100644 --- a/src/net/managers.rs +++ b/src/net/managers.rs @@ -32,13 +32,13 @@ pub enum ConnectionState { impl ConnectionState { /// Tries to change current state, returns old state if successfully changed. - pub fn try_change(&mut self, new: Self) -> Option { + pub fn try_change(&mut self, new: &Self) -> Option { match (&self, &new) { (ConnectionState::Connecting, ConnectionState::Connected(_)) | (ConnectionState::Connecting, ConnectionState::Disconnected(_)) | (ConnectionState::Connected(_), ConnectionState::Disconnected(_)) | (ConnectionState::Disconnected(_), ConnectionState::Connecting) => { - Some(std::mem::replace(self, new)) + Some(std::mem::replace(self, new.clone())) } _ => None, } @@ -125,17 +125,27 @@ pub trait SocketManager: Debug { fn accept_new_connection(&mut self, addr: &SocketAddr) -> Option>; /// Returns list of connections that socket manager decided to destroy - fn destroy_connections(&mut self) -> Option>; + fn collect_connections_to_destroy(&mut self) -> Option>; // all sorts of statistics might be useful here to help deciding whether new connection can be created or not - fn track_connection_error(&mut self, addr: &SocketAddr, error: &ErrorKind); - fn track_global_error(&mut self, error: &ErrorKind); + fn track_connection_error(&mut self, addr: &SocketAddr, error: &ErrorKind, error_context: &str); + fn track_global_error(&mut self, error: &ErrorKind, error_context: &str); fn track_sent_bytes(&mut self, addr: &SocketAddr, bytes: usize); fn track_received_bytes(&mut self, addr: &SocketAddr, bytes: usize); fn track_ignored_bytes(&mut self, addr: &SocketAddr, bytes: usize); fn track_connection_destroyed(&mut self, addr: &SocketAddr); } + + + +//======================================= demo implementations ========================================== + +use crate::packet::{DeliveryGuarantee, OrderingGuarantee, OutgoingPacketBuilder, PacketType}; +use log::error; +use std::io::ErrorKind::WouldBlock; + + /// Simple connection manager, sends "connect" and "disconnect" messages and changes states when receive either of theses messages #[derive(Debug, Default)] struct SimpleConnectionManager { @@ -143,8 +153,6 @@ struct SimpleConnectionManager { send: Option>, } -use crate::packet::{DeliveryGuarantee, OrderingGuarantee, OutgoingPacketBuilder, PacketType}; - impl ConnectionManager for SimpleConnectionManager { fn update<'a>( &mut self, @@ -194,9 +202,13 @@ impl ConnectionManager for SimpleConnectionManager { ) -> Result<&ConnectionState, ConnectionManagerError> { if data.starts_with("connect".as_bytes()) { self.state - .try_change(ConnectionState::Connected(Box::from(data.split_at(7).1))); + .try_change(&ConnectionState::Connected(Box::from(data.split_at(7).1))); + self.send = Some(Box::from("connected".as_bytes())); + } else if data.eq("connected".as_bytes()) { + self.state + .try_change(&ConnectionState::Connected(Box::from(data.split_at(9).1))); } else if data.eq("disconnect".as_bytes()) { - self.state.try_change(ConnectionState::Disconnected(ConnectionClosedBy::RemoteHost)); + self.state.try_change(&ConnectionState::Disconnected(ConnectionClosedBy::RemoteHost)); } else { return Err(ConnectionManagerError(format!( "Unknown message type: {:?}", @@ -211,7 +223,7 @@ impl ConnectionManager for SimpleConnectionManager { } fn disconnect<'a>(&mut self) { - self.state.try_change(ConnectionState::Disconnected(ConnectionClosedBy::LocalHost)); + self.state.try_change(&ConnectionState::Disconnected(ConnectionClosedBy::LocalHost)); self.send = Some(Box::from("disconnect".as_bytes())); } } @@ -225,12 +237,19 @@ impl SocketManager for DumbSocketManager { Some(Box::new(SimpleConnectionManager::default())) } - fn destroy_connections(&mut self) -> Option> { + fn collect_connections_to_destroy(&mut self) -> Option> { None } - fn track_connection_error(&mut self, addr: &SocketAddr, error: &ErrorKind) {} - fn track_global_error(&mut self, error: &ErrorKind) {} + fn track_connection_error(&mut self, addr: &SocketAddr, error: &ErrorKind, error_context: &str) { + match error { + ErrorKind::IOError(ref e) if e.kind() == WouldBlock => {}, + _ => error!("{} ({:?}): {:?}", error_context, addr, error) + } + } + fn track_global_error(&mut self, error: &ErrorKind, error_context: &str) { + error!("{} : {:?}", error_context, error); + } fn track_sent_bytes(&mut self, addr: &SocketAddr, bytes: usize) {} fn track_received_bytes(&mut self, addr: &SocketAddr, bytes: usize) {} fn track_ignored_bytes(&mut self, addr: &SocketAddr, bytes: usize) {} diff --git a/src/net/socket.rs b/src/net/socket.rs index 2fd0ee15..57711ba0 100644 --- a/src/net/socket.rs +++ b/src/net/socket.rs @@ -145,47 +145,35 @@ impl Socket { match self.recv_from(time) { Ok(UdpSocketState::MaybeMore) => continue, Ok(UdpSocketState::MaybeEmpty) => break, - Err(e) => error!("Encountered an error receiving data: {:?}", e), + Err(ref e) => self.manager.track_global_error(e, "receiving data from socket"), } } // Now grab all the packets waiting to be sent and send them while let Ok(p) = self.packet_receiver.try_recv() { - if let Err(e) = self.send_to(p, time) { - match e { - ErrorKind::IOError(ref e) if e.kind() == io::ErrorKind::WouldBlock => {} - _ => error!("There was an error sending packet: {:?}", e), - } + if let Err(ref e) = self.send_to(p, time) { + self.manager.track_global_error(e, "sending data from socket") } } - // TODO provide real buffer, from config.max_receive_buffer - // update all connections, update connection states and send generated packets - // - let mut buff = [0; 1500]; - for (address, bytes) in self.connections.update_connections(&mut buff, &self.event_sender, self.manager.as_mut()) { - match self.send_packet(&address, &bytes) { - Ok(bytes_sent) => self.manager.track_sent_bytes(&address, bytes_sent), - Err(err) => self.manager.track_connection_error(&address, &err) - } + self.connections.update_connections(&self.event_sender, self.manager.as_mut(), &self.socket); + + // get connections that socket manager decided to destroy + if let Some(list) = self.manager.collect_connections_to_destroy() { + for (addr, reason) in list { + self.connections.remove_connection(&addr, &self.event_sender, self.manager.as_mut(), reason, "destroyed by socket manager"); + } } // Check for idle clients - if let Err(e) = self.handle_idle_clients(time) { - error!("Encountered an error when sending TimeoutEvent: {:?}", e); - } + self.handle_idle_clients(time); // Handle any dead clients - self.handle_dead_clients().expect("Internal laminar error"); + self.handle_dead_clients(); // Finally send heartbeat packets to connections that require them, if enabled if let Some(heartbeat_interval) = self.config.heartbeat_interval { - if let Err(e) = self.send_heartbeat_packets(heartbeat_interval, time) { - match e { - ErrorKind::IOError(ref e) if e.kind() == io::ErrorKind::WouldBlock => {} - _ => error!("There was an error sending a heartbeat packet: {:?}", e), - } - } + self.send_heartbeat_packets(heartbeat_interval, time); } } @@ -201,25 +189,23 @@ impl Socket { /// Iterate through the dead connections and disconnect them by removing them from the /// connection map while informing the user of this by sending an event. - fn handle_dead_clients(&mut self) -> Result<()> { + fn handle_dead_clients(&mut self) { let dead_addresses = self.connections.dead_connections(); for address in dead_addresses { - self.connections.remove_connection(&address, &self.event_sender, self.manager.as_mut(), DestroyReason::TooManyPacketsInFlight)?; + self.connections.remove_connection(&address, &self.event_sender, self.manager.as_mut(), DestroyReason::TooManyPacketsInFlight, "removing dead clients"); } - Ok(()) } /// Iterate through all of the idle connections based on `idle_connection_timeout` config and /// remove them from the active connections. For each connection removed, we will send a /// `SocketEvent::TimeOut` event to the `event_sender` channel. - fn handle_idle_clients(&mut self, time: Instant) -> Result<()> { + fn handle_idle_clients(&mut self, time: Instant) { let idle_addresses = self .connections .idle_connections(self.config.idle_connection_timeout, time); for address in idle_addresses { - self.connections.remove_connection(&address, &self.event_sender, self.manager.as_mut(), DestroyReason::Timeout)?; + self.connections.remove_connection(&address, &self.event_sender, self.manager.as_mut(), DestroyReason::Timeout, "removing idle clients"); } - Ok(()) } /// Iterate over all connections which have not sent a packet for a duration of at least @@ -228,7 +214,7 @@ impl Socket { &mut self, heartbeat_interval: Duration, time: Instant, - ) -> Result { + ) { let heartbeat_packets_and_addrs = self .connections .heartbeat_required_connections(heartbeat_interval, time) @@ -239,16 +225,15 @@ impl Socket { ) }) .collect::>(); - - let mut bytes_sent = 0; - for (heartbeat_packet, address) in heartbeat_packets_and_addrs { if self.should_send_packet() { - bytes_sent += self.send_packet(&address, &heartbeat_packet.contents())?; + match self.send_packet(&address, &heartbeat_packet.contents()) { + Ok(bytes_sent) => self.manager.track_sent_bytes(&address, bytes_sent), + Err(ref err) => self.manager.track_connection_error(&address, err, "sending heartbeat packet") + } } } - - Ok(bytes_sent) + } // Serializes and sends a `Packet` on the socket. On success, returns the number of bytes written. @@ -314,15 +299,15 @@ impl Socket { } else { if let Some(manager) = self.manager.accept_new_connection(&address) { self.event_sender.send(SocketEvent::Created(address))?; - Some(self.connections.get_or_insert_connection(address, &self.config, time, manager)) + Some(self.connections.get_or_insert_connection(address, &self.config, time, manager)) } else { None - } + } }; if let Some(conn) = connection - { - conn.process_incoming(received_payload, &self.event_sender, time)?; + { + conn.process_incoming(received_payload, &self.event_sender, self.manager.as_mut(), time)?; } } Err(e) => { diff --git a/src/net/virtual_connection.rs b/src/net/virtual_connection.rs index 359db966..1c81c7a3 100644 --- a/src/net/virtual_connection.rs +++ b/src/net/virtual_connection.rs @@ -10,7 +10,7 @@ use crate::{ ACKED_PACKET_HEADER, DEFAULT_ORDERING_STREAM, DEFAULT_SEQUENCING_STREAM, STANDARD_HEADER_SIZE, }, - net::managers::{ConnectionManager, ConnectionState}, + net::managers::{ConnectionManager, ConnectionState, SocketManager}, packet::{ DeliveryGuarantee, OrderingGuarantee, Outgoing, OutgoingPacket, OutgoingPacketBuilder, Packet, PacketReader, PacketType, SequenceNumber, @@ -22,7 +22,6 @@ use crossbeam_channel::{self, Sender}; use std::fmt; use std::net::SocketAddr; use std::time::{Duration, Instant}; -use std::mem; /// Contains the information about a certain 'virtual connection' over udp. /// This connections also keeps track of network quality, processing packets, buffering data related to connection etc. @@ -43,7 +42,6 @@ pub struct VirtualConnection { fragmentation: Fragmentation, pub state_manager: Box, pub current_state: ConnectionState, - pub conn_manager_packets: Vec } impl VirtualConnection { @@ -60,8 +58,7 @@ impl VirtualConnection { fragmentation: Fragmentation::new(config), config: config.to_owned(), state_manager, - current_state: ConnectionState::Connecting, - conn_manager_packets: Default::default() + current_state: ConnectionState::Connecting } } @@ -93,35 +90,18 @@ impl VirtualConnection { self.last_sent = time; self.congestion_handler .process_outgoing(self.acknowledge_handler.local_sequence_num(), time); - - OutgoingPacketBuilder::new(&[]) + // TODO pass in socket and link conditioner, so that i could send immediatelly + let packet = OutgoingPacketBuilder::new(&[]) .with_default_header( PacketType::Heartbeat, DeliveryGuarantee::Unreliable, OrderingGuarantee::None, ) - .build() - } + .build(); + packet + // let mut buffer = [0;1500]; + // let res = self.state_manager.postprocess_outgoing(&packet.contents(), &mut buffer); - /// This will return all packets that are generated by connection manager - pub fn get_connection_manager_packets(&mut self, time: Instant) -> Result>> { - // TODO implement - Ok(vec!{}) - // Ok(self.conn_manager_packets - // .into_iter() - // .map(|packet| { - // match self.process_outgoing_impl( - // packet.payload(),packet.delivery_guarantee(), - // packet.order_guarantee(),None,time) { - // Ok(out) => Some(out), - // Err(err) => { - // None - // } - // } - // .into_iter() - // }) - // .flatten() - // .collect()) } pub fn process_outgoing<'a>( @@ -282,20 +262,13 @@ impl VirtualConnection { &mut self, received_data: &[u8], sender: &Sender, + socket_manager: &mut dyn SocketManager, time: Instant, ) -> crate::Result<()> { // TODO pass buffer from somewhere else let mut buffer = [0;1500]; - let bytes = self.state_manager.preprocess_incoming(received_data, &mut buffer)?; - self.process_incoming_impl(bytes, sender, time) - } - /// This processes the incoming data and returns a packet if the data is complete. - fn process_incoming_impl( - &mut self, - received_data: &[u8], - sender: &Sender, - time: Instant, - ) -> crate::Result<()> { + let received_data = self.state_manager.preprocess_incoming(received_data, &mut buffer)?; + self.last_heard = time; let mut packet_reader = PacketReader::new(received_data); @@ -325,22 +298,24 @@ impl VirtualConnection { .get_or_create_stream(arranging_header.stream_id()); if let Some(packet) = stream.arrange(arranging_header.arranging_id(), payload) { - self.process_packet( - header.is_connection_manager(), + self.process_packet_and_log_errors( + header.is_connection_manager_packet(), sender, + socket_manager, packet, header.delivery_guarantee(), - OrderingGuarantee::Sequenced(Some(arranging_header.stream_id())))?; + OrderingGuarantee::Sequenced(Some(arranging_header.stream_id()))); } return Ok(()); } - self.process_packet( - header.is_connection_manager(), + self.process_packet_and_log_errors( + header.is_connection_manager_packet(), sender, + socket_manager, packet_reader.read_payload(), header.delivery_guarantee(), - header.ordering_guarantee())?; + header.ordering_guarantee()); } DeliveryGuarantee::Reliable => { if header.is_fragment() { @@ -352,13 +327,14 @@ impl VirtualConnection { .handle_fragment(fragment_header, &payload) { Ok(Some(payload)) => { - self.process_packet( - header.is_connection_manager(), + self.process_packet_and_log_errors( + false, // connection manager packet cannot be fragmented sender, + socket_manager, payload.into_boxed_slice(), header.delivery_guarantee(), OrderingGuarantee::None, - )?; + ); } Ok(None) => return Ok(()), Err(e) => return Err(e), @@ -391,13 +367,14 @@ impl VirtualConnection { if let Some(packet) = stream.arrange(arranging_header.arranging_id(), payload) { - self.process_packet( - header.is_connection_manager(), + self.process_packet_and_log_errors( + header.is_connection_manager_packet(), sender, + socket_manager, packet, header.delivery_guarantee(), OrderingGuarantee::Sequenced(Some(arranging_header.stream_id())), - )?; + ); } } else if let OrderingGuarantee::Ordered(_id) = header.ordering_guarantee() { let arranging_header = packet_reader.read_arranging_header(u16::from( @@ -415,23 +392,25 @@ impl VirtualConnection { .chain(stream.iter_mut()) .collect::>(); for packet in packets { - self.process_packet( - header.is_connection_manager(), + self.process_packet_and_log_errors( + header.is_connection_manager_packet(), sender, + socket_manager, packet, header.delivery_guarantee(), - OrderingGuarantee::Ordered(Some(arranging_header.stream_id())))?; + OrderingGuarantee::Ordered(Some(arranging_header.stream_id()))); } } else { let payload = packet_reader.read_payload(); - self.process_packet( - header.is_connection_manager(), + self.process_packet_and_log_errors( + header.is_connection_manager_packet(), sender, + socket_manager, payload, header.delivery_guarantee(), header.ordering_guarantee(), - )?; + ); } self.congestion_handler @@ -451,45 +430,45 @@ impl VirtualConnection { /// Pass packet to ConnectionManager, check for state changes, and return any newly generated packets fn process_packet( &mut self, - is_connection_packet: bool, + is_connection_manager_packet: bool, sender: &Sender, payload: Box<[u8]>, delivery: DeliveryGuarantee, ordering: OrderingGuarantee ) -> Result<()> { - let packet = Packet::new(self.remote_address, payload,delivery, ordering); - //let mut new_packets = vec!{}; - // let new_state = self.state_manager.process_incoming(&packet, &mut |new_packet| { - // // TODO refactor so, that i could remove self from here - // self.process_outgoing_impl(new_packet.payload(),new_packet.delivery_guarantee(), - // new_packet.order_guarantee(),None,Instant::now()); - // //new_packets.push(new); - // Ok(()) - // })?; - - // if self.current_state == new_state { - // if self.current_state == ConnectionState::Connected && !is_connection_packet { - // sender.send(SocketEvent::Packet(packet))?; - // } - // } else { - // match (&self.current_state, new_state) { - // (ConnectionState::Connecting, ConnectionState::Connected) => { - // sender.send(SocketEvent::Connected)?; - // if !is_connection_packet { - // sender.send(SocketEvent::Packet(packet))?; - // } - // }, - // (ConnectionState::Connecting, ConnectionState::Disconnected) => (),// no need to send any event, because socket is not connected and is will be destroyed later - // (ConnectionState::Connected, ConnectionState::Disconnected) => sender.send(SocketEvent::Disconnected(DisconnectReason::ClosedByClient))?, - // (ConnectionState::Disconnected, ConnectionState::Connecting) => self.reset_connection(), - // _ => panic!("Invalid state transition {:?} -> {:?}", self.current_state, new_state) - // }; - // self.current_state = new_state; - // } - // todo implement ability to return generated packets + if !is_connection_manager_packet { + if let ConnectionState::Connected(_) = self.current_state { + sender.send(SocketEvent::Packet(Packet::new(self.remote_address, payload, delivery, ordering)))?; + } + } else { + let new_state = self.state_manager.process_protocol_data(payload.as_ref())?; + if let Some(_) = self.current_state.try_change(new_state) { + match &self.current_state { + ConnectionState::Connected(data) => sender.send(SocketEvent::Connected(self.remote_address, data.clone()))?, + ConnectionState::Disconnected(reason) => sender.send(SocketEvent::Disconnected(DisconnectReason::ClosedBy(reason.clone())))?, + ConnectionState::Connecting => self.reset_connection(), + }; + } else { + panic!("Invalid state transition {:?} -> {:?}", self.current_state, new_state) + } + } Ok(()) } + fn process_packet_and_log_errors( + &mut self, + is_connection_manager_packet: bool, + sender: &Sender, + socket_manager: &mut dyn SocketManager, + payload: Box<[u8]>, + delivery: DeliveryGuarantee, + ordering: OrderingGuarantee + ) { + if let Err(ref error) = self.process_packet(is_connection_manager_packet, sender, payload, delivery, ordering) { + socket_manager.track_connection_error(&self.remote_address, error, "processing incomming packet"); + } + } + /// this function fully reset connection to initial state, after ConnectionManager switches from Disconnected to Connecting fn reset_connection(&mut self) { let time = Instant::now(); @@ -501,7 +480,6 @@ impl VirtualConnection { self.congestion_handler = CongestionHandler::new(&self.config); self.fragmentation = Fragmentation::new(&self.config); self.current_state = ConnectionState::Connecting; - self.conn_manager_packets.clear(); } /// This will gather dropped packets from the acknowledgment handler. diff --git a/src/packet/header/standard_header.rs b/src/packet/header/standard_header.rs index c9ef955b..ba37eeeb 100644 --- a/src/packet/header/standard_header.rs +++ b/src/packet/header/standard_header.rs @@ -64,7 +64,7 @@ impl StandardHeader { } // Returns true if packet is only used ConnectionManager - pub fn is_connection_manager(&self) -> bool { + pub fn is_connection_manager_packet(&self) -> bool { self.packet_type == PacketType::ConnectionManager } From b1db1776a4ac70575f3857ee824a6720b0960509 Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Sat, 14 Sep 2019 23:22:02 +0300 Subject: [PATCH 06/14] WIP still lots of cleanup left, and implement SocketSendEvent --- src/net/connection.rs | 23 +++++---- src/net/events.rs | 5 +- src/net/socket.rs | 113 +++++++++++++++++++++--------------------- 3 files changed, 73 insertions(+), 68 deletions(-) diff --git a/src/net/connection.rs b/src/net/connection.rs index 2e4bf211..f4c94794 100644 --- a/src/net/connection.rs +++ b/src/net/connection.rs @@ -4,9 +4,11 @@ use crate::{ either::Either::{self, Left, Right}, net::events::{SocketEvent, DisconnectReason, DestroyReason}, net::managers::{ConnectionState, SocketManager}, + net::socket::SocketWithConditioner, packet::{Packet, OutgoingPacket}, ErrorKind, }; + use crossbeam_channel::{self, Sender, SendError}; use std::{ @@ -114,19 +116,25 @@ impl ActiveConnections { pub fn heartbeat_required_connections( &mut self, heartbeat_interval: Duration, - time: Instant - ) -> impl Iterator { + time: Instant, + manager:&mut dyn SocketManager, + socket:&mut SocketWithConditioner + ) { self.connections .iter_mut() .filter(move |(_, connection)| connection.last_sent(time) >= heartbeat_interval) - .map(|(_, connection)| connection) + .for_each(|(_, connection)| { + let packet = connection.create_and_process_heartbeat(time); + socket.send_packet_and_log(&connection.remote_address, connection.state_manager.as_mut(), &packet.contents(), manager, "sending heartbeat packet"); + }); } + pub fn update_connections( &mut self, sender: &Sender, manager:&mut dyn SocketManager, - socket:&UdpSocket + socket:&mut SocketWithConditioner ) { // TODO provide real buffer, from config.max_receive_buffer // update all connections, update connection states and send generated packets @@ -139,12 +147,7 @@ impl ActiveConnections { Some(result) => match result { Ok(event) => match event { Either::Left(packet) => { - //TODO refactor, so that i could pass here link conditioner as well - match socket.send_to(&packet.contents(), conn.remote_address) { - Ok(bytes_sent) => manager.track_sent_bytes(&conn.remote_address, bytes_sent), - Err(err) => manager.track_connection_error(&conn.remote_address, &ErrorKind::IOError(err), - "sending packet from connection manager") - }; + socket.send_packet_and_log(&conn.remote_address, conn.state_manager.as_mut(), &packet.contents(), manager, "sending packet from connection manager"); }, Either::Right(state) => { conn.current_state = state.clone(); diff --git a/src/net/events.rs b/src/net/events.rs index 50cf62db..e29136ed 100644 --- a/src/net/events.rs +++ b/src/net/events.rs @@ -36,10 +36,13 @@ pub enum DisconnectReason { UnrecoverableError(DestroyReason) } +// TODO rename to SocketReceiveEvent and create new with SocketSendEvent #[derive(Debug, PartialEq)] -pub enum SocketEvent { +pub enum SocketEvent { Created(SocketAddr), + // TODO maybe change to Vec? Connected(SocketAddr, Box<[u8]>), + // TODO probably good idea, to add SocketAddr to all these method Packet(Packet), Disconnected(DisconnectReason), Destroyed(DestroyReason), diff --git a/src/net/socket.rs b/src/net/socket.rs index 57711ba0..bcfd12d5 100644 --- a/src/net/socket.rs +++ b/src/net/socket.rs @@ -2,7 +2,7 @@ use crate::{ config::Config, error::{ErrorKind, Result}, net::{connection::ActiveConnections, events::SocketEvent, events::DestroyReason, link_conditioner::LinkConditioner}, - net::managers::{SocketManager}, + net::managers::{SocketManager, ConnectionManager}, packet::{DeliveryGuarantee, Outgoing, Packet}, }; use crossbeam_channel::{self, unbounded, Receiver, SendError, Sender, TryRecvError}; @@ -14,14 +14,53 @@ use std::{ time::{Duration, Instant}, }; +// just wrap whese too together, for easier passing around +#[derive(Debug)] +pub struct SocketWithConditioner { + buffer: Vec, + socket: UdpSocket, + link_conditioner: Option, +} + +impl SocketWithConditioner { + /// Send a single packet over the UDP socket. + /// Postprocess it using Connectionmanager`s postprocess_outgoing method, + /// and use link condition if exists, to simulate network conditions + pub fn send_packet(&mut self, addr:&SocketAddr, manager:&mut dyn ConnectionManager, payload:&[u8]) -> Result { + let payload = manager.postprocess_outgoing(payload, self.buffer.as_mut_slice()); + if let Some(ref mut link) = self.link_conditioner { + if !link.should_send() { + return Ok(0) + } + } + Ok(self.socket.send_to(payload, addr)?) + } + + pub fn send_packet_and_log(&mut self, addr:&SocketAddr, conn_man:&mut dyn ConnectionManager, payload:&[u8], sock_man:&mut dyn SocketManager, error_context:&str) -> usize { + match self.send_packet(addr, conn_man, payload) { + Ok(bytes) => { + sock_man.track_sent_bytes(addr, bytes); + bytes + }, + Err(ref err) => { + sock_man.track_connection_error(addr, err, error_context); + 0 + } + } + } + +} + + + /// A reliable UDP socket implementation with configurable reliability and ordering guarantees. #[derive(Debug)] pub struct Socket { - socket: UdpSocket, + socket: SocketWithConditioner, config: Config, connections: ActiveConnections, recv_buffer: Vec, - link_conditioner: Option, + event_sender: Sender, packet_receiver: Receiver, @@ -72,10 +111,13 @@ impl Socket { let (packet_sender, packet_receiver) = unbounded(); Ok(Socket { recv_buffer: vec![0; config.receive_buffer_max_size], - socket, + socket: SocketWithConditioner { + buffer: Vec::with_capacity(config.receive_buffer_max_size), + socket, + link_conditioner: None, + }, config, connections: ActiveConnections::new(), - link_conditioner: None, event_sender, packet_receiver, @@ -156,7 +198,7 @@ impl Socket { } } - self.connections.update_connections(&self.event_sender, self.manager.as_mut(), &self.socket); + self.connections.update_connections(&self.event_sender, self.manager.as_mut(), &mut self.socket); // get connections that socket manager decided to destroy if let Some(list) = self.manager.collect_connections_to_destroy() { @@ -173,18 +215,20 @@ impl Socket { // Finally send heartbeat packets to connections that require them, if enabled if let Some(heartbeat_interval) = self.config.heartbeat_interval { - self.send_heartbeat_packets(heartbeat_interval, time); + // Iterate over all connections which have not sent a packet for a duration of at least + // `heartbeat_interval` (from config), and send a heartbeat packet to each. + self.connections.heartbeat_required_connections(heartbeat_interval, time, self.manager.as_mut(), &mut self.socket); } } /// Set the link conditioner for this socket. See [LinkConditioner] for further details. pub fn set_link_conditioner(&mut self, link_conditioner: Option) { - self.link_conditioner = link_conditioner; + self.socket.link_conditioner = link_conditioner; } /// Get the local socket address pub fn local_addr(&self) -> Result { - Ok(self.socket.local_addr()?) + Ok(self.socket.socket.local_addr()?) } /// Iterate through the dead connections and disconnect them by removing them from the @@ -208,34 +252,6 @@ impl Socket { } } - /// Iterate over all connections which have not sent a packet for a duration of at least - /// `heartbeat_interval` (from config), and send a heartbeat packet to each. - fn send_heartbeat_packets( - &mut self, - heartbeat_interval: Duration, - time: Instant, - ) { - let heartbeat_packets_and_addrs = self - .connections - .heartbeat_required_connections(heartbeat_interval, time) - .map(|connection| { - ( - connection.create_and_process_heartbeat(time), - connection.remote_address, - ) - }) - .collect::>(); - for (heartbeat_packet, address) in heartbeat_packets_and_addrs { - if self.should_send_packet() { - match self.send_packet(&address, &heartbeat_packet.contents()) { - Ok(bytes_sent) => self.manager.track_sent_bytes(&address, bytes_sent), - Err(ref err) => self.manager.track_connection_error(&address, err, "sending heartbeat packet") - } - } - } - - } - // Serializes and sends a `Packet` on the socket. On success, returns the number of bytes written. fn send_to(&mut self, packet: Packet, time: Instant) -> Result { let mut bytes_sent = 0; @@ -268,18 +284,16 @@ impl Socket { processed_packets.push(processed_packet); for processed_packet in processed_packets { - if self.should_send_packet() { match processed_packet { Outgoing::Packet(outgoing) => { - bytes_sent += self.send_packet(address, &outgoing.contents())?; + bytes_sent += self.socket.send_packet(address, connection.state_manager.as_mut(), &outgoing.contents())?; } Outgoing::Fragments(packets) => { for outgoing in packets { - bytes_sent += self.send_packet(address, &outgoing.contents())?; + bytes_sent += self.socket.send_packet(address, connection.state_manager.as_mut(), &outgoing.contents())?; } } } - } } } Ok(bytes_sent) @@ -287,7 +301,7 @@ impl Socket { // On success the packet will be sent on the `event_sender` fn recv_from(&mut self, time: Instant) -> Result { - match self.socket.recv_from(&mut self.recv_buffer) { + match self.socket.socket.recv_from(&mut self.recv_buffer) { Ok((recv_len, address)) => { if recv_len == 0 { return Err(ErrorKind::ReceivedDataToShort)?; @@ -327,21 +341,6 @@ impl Socket { } } - // Send a single packet over the UDP socket. - fn send_packet(&self, addr: &SocketAddr, payload: &[u8]) -> Result { - let bytes_sent = self.socket.send_to(payload, addr)?; - Ok(bytes_sent) - } - - // In the presence of a link conditioner, we would like it to determine whether or not we should - // send a packet. - fn should_send_packet(&mut self) -> bool { - if let Some(link_conditioner) = &mut self.link_conditioner { - link_conditioner.should_send() - } else { - true - } - } #[cfg(test)] fn connection_count(&self) -> usize { From ef6e809dc128a3006bd1c883137aad94c7a1419e Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Sun, 15 Sep 2019 21:57:17 +0300 Subject: [PATCH 07/14] WIP ConnectionEvent in progress --- src/error.rs | 8 +++--- src/lib.rs | 2 +- src/net.rs | 2 +- src/net/connection.rs | 16 ++++++++---- src/net/events.rs | 43 ++++++++++++------------------- src/net/managers.rs | 6 ++--- src/net/socket.rs | 59 +++++++++++++++++++++++++------------------ 7 files changed, 70 insertions(+), 66 deletions(-) diff --git a/src/error.rs b/src/error.rs index d5d3d1d5..9a86dd75 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,5 @@ //! This module contains the laminar error handling logic. -use crate::SocketEvent; use crossbeam_channel::SendError; use std::{ error::Error, @@ -8,6 +7,7 @@ use std::{ io, result, }; use crate::net::managers::ConnectionManagerError; +use crate::net::event::{ConnectionEvent, ReceiveEvent}; /// Wrapped result type for Laminar errors. pub type Result = result::Result; @@ -28,7 +28,7 @@ pub enum ErrorKind { /// Protocol versions did not match ProtocolVersionMismatch, /// Could not send on `SendChannel`. - SendError(SendError), + SendError(SendError>), /// Expected header but could not be read from buffer. CouldNotReadHeader(String), /// Errors that is returned from ConnectionManager either preprocessing data or processing packet @@ -181,8 +181,8 @@ impl From for ErrorKind { } } -impl From> for ErrorKind { - fn from(inner: SendError) -> Self { +impl From>> for ErrorKind { + fn from(inner: SendError>) -> Self { ErrorKind::SendError(inner) } } diff --git a/src/lib.rs b/src/lib.rs index 5bfc19bc..9adcad45 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,6 +39,6 @@ pub use self::throughput::ThroughputMonitoring; pub use self::config::Config; pub use self::error::{ErrorKind, Result}; -pub use self::net::{LinkConditioner, Socket, SocketEvent, DisconnectReason}; +pub use self::net::{LinkConditioner, Socket, ConnectionEvent, DisconnectReason}; pub use self::packet::{DeliveryGuarantee, OrderingGuarantee, Packet}; pub use self::net::managers::{ConnectionManager, ConnectionManagerError}; diff --git a/src/net.rs b/src/net.rs index a96d2bd0..50ffb83c 100644 --- a/src/net.rs +++ b/src/net.rs @@ -11,7 +11,7 @@ mod virtual_connection; pub mod constants; pub mod managers; -pub use self::events::{SocketEvent, DisconnectReason, DestroyReason, ConnectionSendEvent, ConnectionReceiveEvent }; +pub use self::events::*; pub use self::link_conditioner::LinkConditioner; pub use self::quality::{NetworkQuality, RttMeasurer}; pub use self::socket::Socket; diff --git a/src/net/connection.rs b/src/net/connection.rs index f4c94794..2b700662 100644 --- a/src/net/connection.rs +++ b/src/net/connection.rs @@ -2,7 +2,7 @@ pub use crate::net::{NetworkQuality, RttMeasurer, VirtualConnection, managers::C use crate::{ config::Config, either::Either::{self, Left, Right}, - net::events::{SocketEvent, DisconnectReason, DestroyReason}, + net::events::{ ConnectionEvent, ReceiveEvent, DisconnectReason, DestroyReason}, net::managers::{ConnectionState, SocketManager}, net::socket::SocketWithConditioner, packet::{Packet, OutgoingPacket}, @@ -73,7 +73,7 @@ impl ActiveConnections { pub fn remove_connection( &mut self, address: &SocketAddr, - sender: &Sender, + sender: &Sender>, manager: &mut dyn SocketManager, reason: DestroyReason, error_context: &str @@ -81,11 +81,17 @@ impl ActiveConnections { if let Some((_, conn)) = self.connections.remove_entry(address) { manager.track_connection_destroyed(address); if let ConnectionState::Connected(_) = conn.get_current_state() { - if let Err(err) = sender.send(SocketEvent::Disconnected(DisconnectReason::UnrecoverableError(reason.clone()))) { + if let Err(err) = sender.send(ConnectionEvent { + addr: conn.remote_address, + event: ReceiveEvent::Disconnected(DisconnectReason::UnrecoverableError(reason.clone())) + }) { manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(err), error_context); } } - if let Err(err) = sender.send(SocketEvent::Destroyed(reason)) { + if let Err(err) = sender.send(ConnectionEvent { + addr: conn.remote_address, + event: ReceiveEvent::Destroyed(reason) + }) { manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(err), error_context); } true @@ -132,7 +138,7 @@ impl ActiveConnections { pub fn update_connections( &mut self, - sender: &Sender, + sender: &Sender>, manager:&mut dyn SocketManager, socket:&mut SocketWithConditioner ) { diff --git a/src/net/events.rs b/src/net/events.rs index e29136ed..6d42f504 100644 --- a/src/net/events.rs +++ b/src/net/events.rs @@ -16,12 +16,12 @@ use std::net::SocketAddr; // } #[derive(Debug, PartialEq, Clone)] -pub enum DestroyReason { - // because wasnt able to connect - HandshakeFailed(ConnectionManagerError), +pub enum DestroyReason { + ConnectionError(ConnectionManagerError), Timeout, TooManyPacketsInFlight, TooManyPacketErrors, + GracefullyDisconnected } #[derive(Debug, PartialEq, Clone)] @@ -36,35 +36,24 @@ pub enum DisconnectReason { UnrecoverableError(DestroyReason) } -// TODO rename to SocketReceiveEvent and create new with SocketSendEvent -#[derive(Debug, PartialEq)] -pub enum SocketEvent { - Created(SocketAddr), - // TODO maybe change to Vec? - Connected(SocketAddr, Box<[u8]>), - // TODO probably good idea, to add SocketAddr to all these method - Packet(Packet), - Disconnected(DisconnectReason), - Destroyed(DestroyReason), +#[derive(Debug)] +pub struct ConnectionEvent { + pub addr: SocketAddr, + pub event: Event } #[derive(Debug)] -pub enum ConnectionReceiveEvent { - /// When the connection is actually added to active connections list. - Created, - /// When connection manager changes to connected state - Connected, - /// Actual received packet, this should only occure after Connected state +pub enum SendEvent { + Connect(Box<[u8]>), Packet(Packet), - /// When connection manager changes to disconnected state - Disconnected(DisconnectReason), - /// When connection is actually removed from connections list. - Destroyed(DestroyReason) + Disconnect, } #[derive(Debug)] -pub enum ConnectionSendEvent { - Connect, +pub enum ReceiveEvent { + Created, + Connected(Box<[u8]>), Packet(Packet), - Disconnect, -} \ No newline at end of file + Disconnected(DisconnectReason), + Destroyed(DestroyReason), +} diff --git a/src/net/managers.rs b/src/net/managers.rs index 45444d59..d9218e9e 100644 --- a/src/net/managers.rs +++ b/src/net/managers.rs @@ -111,7 +111,7 @@ pub trait ConnectionManager: Debug { /// Some protocols might provide a way to pass initial connection data /// Data is user payload, that will be received by remote host, on Connected(data) event /// This method is not able to send packet immediatelly, instead this functionality should be handled in `update` method. - fn connect<'a>(&mut self, data: &'a [u8]); + fn connect(&mut self, data: Box<[u8]>); // This will be invoked when player sends disconnect request, /// This method is not able to send packet immediatelly, instead this functionality should be handled in `update` method. @@ -218,8 +218,8 @@ impl ConnectionManager for SimpleConnectionManager { Ok(&self.state) } - fn connect<'a>(&mut self, data: &'a [u8]) { - self.send = Some(Box::from(["connect".as_bytes(), data].concat())); + fn connect<'a>(&mut self, data: Box<[u8]>) { + self.send = Some(Box::from(["connect".as_bytes(), data.as_ref()].concat())); } fn disconnect<'a>(&mut self) { diff --git a/src/net/socket.rs b/src/net/socket.rs index bcfd12d5..80e8826f 100644 --- a/src/net/socket.rs +++ b/src/net/socket.rs @@ -1,7 +1,8 @@ use crate::{ config::Config, error::{ErrorKind, Result}, - net::{connection::ActiveConnections, events::SocketEvent, events::DestroyReason, link_conditioner::LinkConditioner}, + net::{connection::ActiveConnections, link_conditioner::LinkConditioner}, + net::events::{ConnectionEvent, SendEvent, ReceiveEvent, DestroyReason}, net::managers::{SocketManager, ConnectionManager}, packet::{DeliveryGuarantee, Outgoing, Packet}, }; @@ -61,11 +62,11 @@ pub struct Socket { connections: ActiveConnections, recv_buffer: Vec, - event_sender: Sender, - packet_receiver: Receiver, + event_sender: Sender>, + packet_receiver: Receiver>, - receiver: Receiver, - sender: Sender, + receiver: Receiver>, + sender: Sender>, manager: Box, } @@ -131,29 +132,27 @@ impl Socket { /// Returns a handle to the packet sender which provides a thread-safe way to enqueue packets /// to be processed. This should be used when the socket is busy running its polling loop in a /// separate thread. - pub fn get_packet_sender(&mut self) -> Sender { + pub fn get_packet_sender(&mut self) -> Sender> { self.sender.clone() } /// Returns a handle to the event receiver which provides a thread-safe way to retrieve events /// from the socket. This should be used when the socket is busy running its polling loop in /// a separate thread. - pub fn get_event_receiver(&mut self) -> Receiver { + pub fn get_event_receiver(&mut self) -> Receiver> { self.receiver.clone() } /// Send a packet - pub fn send(&mut self, packet: Packet) -> Result<()> { - match self.sender.send(packet) { + pub fn send(&mut self, event: ConnectionEvent) -> Result<()> { + match self.sender.send(event) { Ok(_) => Ok(()), - Err(error) => Err(ErrorKind::SendError(SendError(SocketEvent::Packet( - error.0, - )))), + Err(error) => Err(ErrorKind::SendError(error)), } } /// Receive a packet - pub fn recv(&mut self) -> Option { + pub fn recv(&mut self) -> Option> { match self.receiver.try_recv() { Ok(pkt) => Some(pkt), Err(TryRecvError::Empty) => None, @@ -253,10 +252,11 @@ impl Socket { } // Serializes and sends a `Packet` on the socket. On success, returns the number of bytes written. - fn send_to(&mut self, packet: Packet, time: Instant) -> Result { + fn send_to(&mut self, event: ConnectionEvent, time: Instant) -> Result { let mut bytes_sent = 0; - let address = &packet.addr(); + let address = &event.addr; if let Some(connection) = self.connections.try_get(address) { + // TODO maybe these should not depend on send_to method? let dropped = connection.gather_dropped_packets(); let mut processed_packets: Vec = dropped .iter() @@ -273,15 +273,21 @@ impl Socket { }) .collect(); - let processed_packet = connection.process_outgoing( - packet.payload(), - packet.delivery_guarantee(), - packet.order_guarantee(), - None, - time, - )?; - - processed_packets.push(processed_packet); + match event.event { + Connect(data) => connection.state_manager.connect(data); + Packet(data) => { + let processed_packet = connection.process_outgoing( + packet.payload(), + packet.delivery_guarantee(), + packet.order_guarantee(), + None, + time, + )?; + processed_packets.push(processed_packet); + } + Disconnect => connection.state_manager.disconnect(); + } + for processed_packet in processed_packets { match processed_packet { @@ -312,7 +318,10 @@ impl Socket { Some(conn) } else { if let Some(manager) = self.manager.accept_new_connection(&address) { - self.event_sender.send(SocketEvent::Created(address))?; + self.event_sender.send(ConnectionEvent { + addr: address, + event: ReceiveEvent::Created + })?; Some(self.connections.get_or_insert_connection(address, &self.config, time, manager)) } else { None From 201f0bf091ec97546a74c82c6e14101b05ef6912 Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Mon, 16 Sep 2019 17:44:49 +0300 Subject: [PATCH 08/14] WIP working on example --- README.md | 2 +- examples/server_client.rs | 82 ++++++++++++------ examples/simple_udp.rs | 4 +- src/either.rs | 1 + src/error.rs | 15 +++- src/lib.rs | 5 +- src/managers.rs | 4 + src/managers/simple.rs | 155 ++++++++++++++++++++++++++++++++++ src/net.rs | 5 +- src/net/connection.rs | 73 +++++++--------- src/net/events.rs | 18 ++-- src/net/managers.rs | 149 +++++--------------------------- src/net/socket.rs | 148 ++++++++++++++++++++------------ src/net/virtual_connection.rs | 65 ++++++-------- 14 files changed, 413 insertions(+), 313 deletions(-) create mode 100644 src/managers.rs create mode 100644 src/managers/simple.rs diff --git a/README.md b/README.md index db968361..d880cb03 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ use laminar::{Socket, Packet}; // create the socket let mut socket = Socket::bind("127.0.0.1:12345")?; -let packet_sender = socket.get_packet_sender(); +let packet_sender = socket.get_event_sender(); // this will start the socket, which will start a poll mechanism to receive and send messages. let _thread = thread::spawn(move || socket.start_polling()); diff --git a/examples/server_client.rs b/examples/server_client.rs index 3d47793b..0464f51b 100644 --- a/examples/server_client.rs +++ b/examples/server_client.rs @@ -5,19 +5,25 @@ use std::io::stdin; use std::thread; use std::time::Instant; -use laminar::{ErrorKind, Packet, Socket, SocketEvent}; +use laminar::{ErrorKind, Packet, Socket, SocketEventSender, managers::SimpleSocketManager, ConnectionEvent, ReceiveEvent, SendEvent}; const SERVER: &str = "127.0.0.1:12351"; fn server() -> Result<(), ErrorKind> { - let mut socket = Socket::bind(SERVER)?; - let (sender, receiver) = (socket.get_packet_sender(), socket.get_event_receiver()); + let mut socket = Socket::bind(SERVER, Box::new(SimpleSocketManager))?; + let (sender, receiver) = (SocketEventSender(socket.get_event_sender()), socket.get_event_receiver()); let _thread = thread::spawn(move || socket.start_polling()); loop { - if let Ok(event) = receiver.recv() { + if let Ok(ConnectionEvent(addr, event)) = receiver.recv() { match event { - SocketEvent::Packet(packet) => { + ReceiveEvent::Created => { + println!("Connection created {:?}", addr); + }, + ReceiveEvent::Connected(data) => { + println!("Connected {:?} with message: {}",addr, String::from_utf8_lossy(data.as_ref())); + }, + ReceiveEvent::Packet(packet) => { let msg = packet.payload(); if msg == b"Bye!" { @@ -35,11 +41,13 @@ fn server() -> Result<(), ErrorKind> { "Copy that!".as_bytes().to_vec(), )) .expect("This should send"); + }, + ReceiveEvent::Disconnected(reason) => { + println!("Disconnected {:?} reason: {:?}", addr, reason); + }, + ReceiveEvent::Destroyed(reason) => { + println!("Connection destroyed {:?} reason: {:?}", addr, reason); } - SocketEvent::Timeout(address) => { - println!("Client timed out: {}", address); - } - _ => {} } } } @@ -49,7 +57,7 @@ fn server() -> Result<(), ErrorKind> { fn client() -> Result<(), ErrorKind> { let addr = "127.0.0.1:12352"; - let mut socket = Socket::bind(addr)?; + let mut socket = Socket::bind(addr, Box::new(SimpleSocketManager))?; println!("Connected on {}", addr); let server = SERVER.parse().unwrap(); @@ -59,15 +67,12 @@ fn client() -> Result<(), ErrorKind> { let stdin = stdin(); let mut s_buffer = String::new(); - loop { - s_buffer.clear(); - stdin.read_line(&mut s_buffer)?; - let line = s_buffer.replace(|x| x == '\n' || x == '\r', ""); + s_buffer.clear(); + stdin.read_line(&mut s_buffer)?; + let line = s_buffer.replace(|x| x == '\n' || x == '\r', ""); + socket.send(ConnectionEvent(server, SendEvent::Connect(Box::from(line.as_bytes()))))?; - socket.send(Packet::reliable_unordered( - server, - line.clone().into_bytes(), - ))?; + loop { socket.manual_poll(Instant::now()); @@ -75,16 +80,41 @@ fn client() -> Result<(), ErrorKind> { break; } - match socket.recv() { - Some(SocketEvent::Packet(packet)) => { - if packet.addr() == server { - println!("Server sent: {}", String::from_utf8_lossy(packet.payload())); - } else { - println!("Unknown sender."); + if let Some(ConnectionEvent(addr, event)) = socket.recv() { + match event { + ReceiveEvent::Created => { + println!("Connection created {:?}", addr); + }, + ReceiveEvent::Connected(data) => { + println!("Connected {:?} with message: {}",addr, String::from_utf8_lossy(data.as_ref())); + socket.send(ConnectionEvent(server, SendEvent::Disconnect))?; + }, + ReceiveEvent::Packet(packet) => { + let msg = packet.payload(); + + if msg == b"Bye!" { + break; + } + + let msg = String::from_utf8_lossy(msg); + let ip = packet.addr().ip(); + + println!("Received {:?} from {:?}", msg, ip); + + // sender + // .send(Packet::reliable_unordered( + // packet.addr(), + // "Copy that!".as_bytes().to_vec(), + // )) + // .expect("This should send"); + }, + ReceiveEvent::Disconnected(reason) => { + println!("Disconnected {:?} reason: {:?}", addr, reason); + }, + ReceiveEvent::Destroyed(reason) => { + println!("Connection destroyed {:?} reason: {:?}", addr, reason); } } - Some(SocketEvent::Timeout(_)) => {} - _ => println!("Silence.."), } } diff --git a/examples/simple_udp.rs b/examples/simple_udp.rs index 678ac9ca..a2a37b93 100644 --- a/examples/simple_udp.rs +++ b/examples/simple_udp.rs @@ -3,7 +3,7 @@ //! 2. setting up client to send data. //! 3. serialize data to send and deserialize when received. use bincode::{deserialize, serialize}; -use laminar::{Packet, Socket, SocketEvent}; +use laminar::{Packet, Socket, ConnectionEventSender}; use serde_derive::{Deserialize, Serialize}; use std::net::SocketAddr; use std::time::Instant; @@ -27,7 +27,7 @@ pub fn main() { let mut server = Socket::bind(server_address()).unwrap(); /* setup or `Client` and send some test data. */ - let mut client = Socket::bind(client_address()).unwrap(); + let mut client = ConnectionEventSender(Socket::bind(client_address()).unwrap()); client.send(Packet::unreliable( server_address(), diff --git a/src/either.rs b/src/either.rs index d430cc15..21e7880e 100644 --- a/src/either.rs +++ b/src/either.rs @@ -1,3 +1,4 @@ +#[derive(Debug)] pub enum Either { Left(L), Right(R), diff --git a/src/error.rs b/src/error.rs index 9a86dd75..25a39015 100644 --- a/src/error.rs +++ b/src/error.rs @@ -7,7 +7,8 @@ use std::{ io, result, }; use crate::net::managers::ConnectionManagerError; -use crate::net::event::{ConnectionEvent, ReceiveEvent}; +use crate::net::events::{ConnectionEvent, SendEvent, ReceiveEvent}; +use crate::either::Either; /// Wrapped result type for Laminar errors. pub type Result = result::Result; @@ -28,7 +29,7 @@ pub enum ErrorKind { /// Protocol versions did not match ProtocolVersionMismatch, /// Could not send on `SendChannel`. - SendError(SendError>), + SendError(SendError, ConnectionEvent>>), /// Expected header but could not be read from buffer. CouldNotReadHeader(String), /// Errors that is returned from ConnectionManager either preprocessing data or processing packet @@ -181,9 +182,15 @@ impl From for ErrorKind { } } -impl From>> for ErrorKind { +impl From>> for ErrorKind { + fn from(inner: SendError>) -> Self { + ErrorKind::SendError(SendError(Either::Left(inner.0))) + } +} + +impl From>> for ErrorKind { fn from(inner: SendError>) -> Self { - ErrorKind::SendError(inner) + ErrorKind::SendError(SendError(Either::Right(inner.0))) } } diff --git a/src/lib.rs b/src/lib.rs index 9adcad45..911a8da0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,6 +30,7 @@ mod net; mod packet; mod protocol_version; mod sequence_buffer; +pub mod managers; #[cfg(feature = "tester")] mod throughput; @@ -39,6 +40,8 @@ pub use self::throughput::ThroughputMonitoring; pub use self::config::Config; pub use self::error::{ErrorKind, Result}; -pub use self::net::{LinkConditioner, Socket, ConnectionEvent, DisconnectReason}; +pub use self::net::{LinkConditioner, Socket, SocketEventSender}; pub use self::packet::{DeliveryGuarantee, OrderingGuarantee, Packet}; pub use self::net::managers::{ConnectionManager, ConnectionManagerError}; +// pub use self::managers::{SimpleSocketManager}; +pub use self::net::events::*; \ No newline at end of file diff --git a/src/managers.rs b/src/managers.rs new file mode 100644 index 00000000..dc3ce27d --- /dev/null +++ b/src/managers.rs @@ -0,0 +1,4 @@ +//! This module provides socket managers. +mod simple; + +pub use self::simple::SimpleSocketManager; \ No newline at end of file diff --git a/src/managers/simple.rs b/src/managers/simple.rs new file mode 100644 index 00000000..5634bcf9 --- /dev/null +++ b/src/managers/simple.rs @@ -0,0 +1,155 @@ +use crate::net::managers::*; + +use crate::packet::{DeliveryGuarantee, OrderingGuarantee, OutgoingPacketBuilder, PacketType}; +use log::error; +use std::io::ErrorKind::WouldBlock; +use std::net::SocketAddr; +use std::time::Instant; +use std::collections::VecDeque; + +/// Simple connection manager, sends "connect" and "disconnect" messages and changes states when receive either of these messages +#[derive(Debug, Default)] +struct SimpleConnectionManager { + state: ConnectionState, + changes: VecDeque, ConnectionState>>, +} + +impl SimpleConnectionManager { + fn change_state(&mut self, new: ConnectionState) { + if let Some(_) = self.state.try_change(&new) { + self.changes.push_back(Either::Right(self.state.clone())); + } + } + + fn send_packet(&mut self, payload:&[u8]) { + self.changes.push_back(Either::Left(Box::from(payload))); + } + + fn get_packet<'a> (data: Box<[u8]>, buffer: &'a mut [u8]) -> OutgoingPacket<'a> { + // get result slice + let res_buffer = &mut buffer[0..data.as_ref().len()]; + // copy from buffer what we want to send + res_buffer.copy_from_slice(data.as_ref()); + println!("===========================update send {}", String::from_utf8_lossy(res_buffer)); + // create packet + OutgoingPacketBuilder::new(res_buffer) + .with_default_header( + PacketType::ConnectionManager, + DeliveryGuarantee::Unreliable, + OrderingGuarantee::None, + ) + .build() + } +} + +impl ConnectionManager for SimpleConnectionManager { + fn update<'a>( + &mut self, + buffer: &'a mut [u8], + _time: Instant, + ) -> Option, ConnectionState>, ConnectionManagerError>> { + match self.changes.pop_front().take() { + Some(change) => { + Some(Ok(match change { + Either::Left(data) => Either::Left(SimpleConnectionManager::get_packet(data, buffer)), + Either::Right(state) => Either::Right(state) + })) + } + None => None, + } + } + + fn preprocess_incoming<'a, 'b>( + &mut self, + data: &'a [u8], + _buffer: &'b mut [u8], + ) -> Result<&'b [u8], ConnectionManagerError> + where + 'a: 'b, + { + println!( + "===========================preprocess_incoming: {}", + String::from_utf8_lossy(data) + ); + Ok(data) + } + + fn postprocess_outgoing<'a, 'b>(&mut self, data: &'a [u8], _buffer: &'b mut [u8]) -> &'b [u8] + where + 'a: 'b, + { + println!( + "===========================postprocess_outgoing: {}", + String::from_utf8_lossy(data) + ); + data + } + + fn process_protocol_data<'a>(&mut self, data: &'a [u8]) -> Result<(), ConnectionManagerError> { + println!( + "===========================process_protocol_data: {}", + String::from_utf8_lossy(data) + ); + if data.starts_with("connect-".as_bytes()) { + self.change_state(ConnectionState::Connected(Box::from(data.split_at(8).1))); + self.send_packet("connected-".as_bytes()); + } else if data.starts_with("connected-".as_bytes()) { + self.change_state(ConnectionState::Connected(Box::from(data.split_at(10).1))); + } else if data.starts_with("disconnect".as_bytes()) { + self.change_state(ConnectionState::Disconnected(TargetHost::RemoteHost)); + } else { + return Err(ConnectionManagerError(format!( + "Unknown message type: {:?}", + String::from_utf8_lossy(data) + ))); + } + Ok(()) + } + + fn connect<'a>(&mut self, data: Box<[u8]>) { + self.send_packet(["connect-".as_bytes(), data.as_ref()].concat().as_ref()); + } + + fn disconnect<'a>(&mut self) { + self.change_state(ConnectionState::Disconnected(TargetHost::LocalHost)); + self.send_packet("disconnect".as_bytes()); + } + +} + +/// Simplest implementation of socket manager, always accept connection and never destroy, no matter how many errors connection reports +#[derive(Debug)] +pub struct SimpleSocketManager; + +impl SocketManager for SimpleSocketManager { + fn accept_new_connection( + &mut self, + _addr: &SocketAddr, + _requested_by: TargetHost, + ) -> Option> { + Some(Box::new(SimpleConnectionManager::default())) + } + + fn collect_connections_to_destroy(&mut self) -> Option> { + None + } + + fn track_connection_error( + &mut self, + addr: &SocketAddr, + error: &ErrorKind, + error_context: &str, + ) { + match error { + ErrorKind::IOError(ref e) if e.kind() == WouldBlock => {} + _ => error!("Error, {} ({:?}): {:?}", error_context, addr, error), + } + } + fn track_global_error(&mut self, error: &ErrorKind, error_context: &str) { + error!("Error, {}: {:?}", error_context, error); + } + fn track_sent_bytes(&mut self, _addr: &SocketAddr, _bytes: usize) {} + fn track_received_bytes(&mut self, _addr: &SocketAddr, _bytes: usize) {} + fn track_ignored_bytes(&mut self, _addr: &SocketAddr, _bytes: usize) {} + fn track_connection_destroyed(&mut self, _addr: &SocketAddr) {} +} diff --git a/src/net.rs b/src/net.rs index 50ffb83c..69414992 100644 --- a/src/net.rs +++ b/src/net.rs @@ -2,17 +2,16 @@ //! You can think of the socket, connection management, congestion control. mod connection; -mod events; mod link_conditioner; mod quality; mod socket; mod virtual_connection; +pub mod events; pub mod constants; pub mod managers; -pub use self::events::*; pub use self::link_conditioner::LinkConditioner; pub use self::quality::{NetworkQuality, RttMeasurer}; -pub use self::socket::Socket; +pub use self::socket::{Socket, SocketEventSender}; pub use self::virtual_connection::VirtualConnection; diff --git a/src/net/connection.rs b/src/net/connection.rs index 2b700662..51edf50b 100644 --- a/src/net/connection.rs +++ b/src/net/connection.rs @@ -1,11 +1,10 @@ pub use crate::net::{NetworkQuality, RttMeasurer, VirtualConnection, managers::ConnectionManager}; use crate::{ config::Config, - either::Either::{self, Left, Right}, + either::Either, net::events::{ ConnectionEvent, ReceiveEvent, DisconnectReason, DestroyReason}, net::managers::{ConnectionState, SocketManager}, net::socket::SocketWithConditioner, - packet::{Packet, OutgoingPacket}, ErrorKind, }; @@ -13,7 +12,7 @@ use crossbeam_channel::{self, Sender, SendError}; use std::{ collections::HashMap, - net::{SocketAddr, UdpSocket}, + net::SocketAddr, time::{Duration, Instant}, }; @@ -48,23 +47,6 @@ impl ActiveConnections { .or_insert_with(|| VirtualConnection::new(address, config, time, state_manager)) } - - /// Try to get or create a [VirtualConnection] by address. If the connection does not exist, it will be - /// created and returned, but not inserted into the table of active connections. - pub(crate) fn get_or_create_connection( - &mut self, - address: SocketAddr, - config: &Config, - time: Instant, - state_manager: Box, - ) -> Either<&mut VirtualConnection, VirtualConnection> { - if let Some(connection) = self.connections.get_mut(&address) { - Left(connection) - } else { - Right(VirtualConnection::new(address, config, time, state_manager)) - } - } - pub fn try_get(&mut self, address: &SocketAddr) -> Option<&mut VirtualConnection> { self.connections.get_mut(address) } @@ -81,18 +63,12 @@ impl ActiveConnections { if let Some((_, conn)) = self.connections.remove_entry(address) { manager.track_connection_destroyed(address); if let ConnectionState::Connected(_) = conn.get_current_state() { - if let Err(err) = sender.send(ConnectionEvent { - addr: conn.remote_address, - event: ReceiveEvent::Disconnected(DisconnectReason::UnrecoverableError(reason.clone())) - }) { - manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(err), error_context); + if let Err(err) = sender.send(ConnectionEvent(conn.remote_address, ReceiveEvent::Disconnected(DisconnectReason::Destroying(reason.clone())))) { + manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(SendError(Either::Right(err.0))), error_context); } } - if let Err(err) = sender.send(ConnectionEvent { - addr: conn.remote_address, - event: ReceiveEvent::Destroyed(reason) - }) { - manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(err), error_context); + if let Err(err) = sender.send(ConnectionEvent(conn.remote_address, ReceiveEvent::Destroyed(reason))) { + manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(SendError(Either::Right(err.0))), error_context); } true } else { @@ -110,11 +86,16 @@ impl ActiveConnections { } /// Get a list of addresses of dead connections - pub fn dead_connections(&mut self) -> Vec { + pub fn dead_connections(&mut self) -> Vec<(SocketAddr, DestroyReason)> { self.connections .iter() - .filter(|(_, connection)| connection.should_be_dropped()) - .map(|(address, _)| *address) + .filter_map(|(_, connection)| if connection.should_be_dropped() { + Some((connection.remote_address, DestroyReason::TooManyPacketsInFlight)) + } else if connection.can_gracefully_disconnect() { + Some((connection.remote_address, DestroyReason::GracefullyDisconnected)) + } else { + None + }) .collect() } @@ -156,12 +137,21 @@ impl ActiveConnections { socket.send_packet_and_log(&conn.remote_address, conn.state_manager.as_mut(), &packet.contents(), manager, "sending packet from connection manager"); }, Either::Right(state) => { - conn.current_state = state.clone(); - if let Err(err) = match &conn.current_state { - ConnectionState::Connected(data) => sender.send(SocketEvent::Connected(conn.remote_address, data.clone())), - ConnectionState::Disconnected(closed_by) => sender.send(SocketEvent::Disconnected(DisconnectReason::ClosedBy(closed_by.clone()))), - _ => Ok(()),} { - manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(err), "sending connection state update"); + if let Some(_) = conn.current_state.try_change(&state) { + if let Err(err) = match &conn.current_state { + ConnectionState::Connected(data) => sender.send(ConnectionEvent(conn.remote_address, + ReceiveEvent::Connected(data.clone()))), + ConnectionState::Disconnected(closed_by) => sender.send(ConnectionEvent(conn.remote_address, + ReceiveEvent::Disconnected(DisconnectReason::ClosedBy(closed_by.clone())))), + ConnectionState::Connecting => { + conn.reset_connection(); + Ok(()) + }, + } { + manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(SendError(Either::Right(err.0))), "sending connection state update"); + } + } else { + panic!("Invalid state transition {:?} -> {:?}", conn.current_state, state); } } }, @@ -173,11 +163,6 @@ impl ActiveConnections { }); } - /// Returns true if the given connection exists. - pub fn exists(&self, address: &SocketAddr) -> bool { - self.connections.contains_key(&address) - } - /// Returns the number of connected clients. #[cfg(test)] pub(crate) fn count(&self) -> usize { diff --git a/src/net/events.rs b/src/net/events.rs index 6d42f504..7a7fa6ed 100644 --- a/src/net/events.rs +++ b/src/net/events.rs @@ -25,22 +25,23 @@ pub enum DestroyReason { } #[derive(Debug, PartialEq, Clone)] -pub enum ConnectionClosedBy { +pub enum TargetHost { LocalHost, RemoteHost } +/// Disconnect reason, received by connection #[derive(Debug, PartialEq)] pub enum DisconnectReason { - ClosedBy(ConnectionClosedBy), - UnrecoverableError(DestroyReason) + /// Disconnect was initiated by local or remote host + ClosedBy(TargetHost), + /// Socket manager decided to destroy connection for provided reason + Destroying(DestroyReason) } +/// Wraps send or receive event together with remote address #[derive(Debug)] -pub struct ConnectionEvent { - pub addr: SocketAddr, - pub event: Event -} +pub struct ConnectionEvent (pub SocketAddr, pub Event); #[derive(Debug)] pub enum SendEvent { @@ -57,3 +58,6 @@ pub enum ReceiveEvent { Disconnected(DisconnectReason), Destroyed(DestroyReason), } + +pub type ConnectionReceiveEvent = ConnectionEvent; +pub type ConnectionSendEvent = ConnectionEvent; \ No newline at end of file diff --git a/src/net/managers.rs b/src/net/managers.rs index d9218e9e..2f2b84b9 100644 --- a/src/net/managers.rs +++ b/src/net/managers.rs @@ -1,7 +1,7 @@ pub use crate::either::Either; -pub use crate::net::events::{ConnectionClosedBy, DestroyReason}; -use crate::packet::OutgoingPacket; -use crate::ErrorKind; +pub use crate::net::events::{TargetHost, DestroyReason}; +pub use crate::packet::{OutgoingPacket, PacketType, DeliveryGuarantee, OrderingGuarantee}; +pub use crate::ErrorKind; use std::fmt::Debug; use std::net::SocketAddr; use std::time::Instant; @@ -27,7 +27,7 @@ use std::time::Instant; pub enum ConnectionState { Connecting, Connected(Box<[u8]>), - Disconnected(ConnectionClosedBy), + Disconnected(TargetHost), } impl ConnectionState { @@ -39,7 +39,7 @@ impl ConnectionState { | (ConnectionState::Connected(_), ConnectionState::Disconnected(_)) | (ConnectionState::Disconnected(_), ConnectionState::Connecting) => { Some(std::mem::replace(self, new.clone())) - } + }, _ => None, } } @@ -53,7 +53,7 @@ impl Default for ConnectionState { /// Generic error type, that is used by ConnectionManager implementation #[derive(Debug, PartialEq, Clone)] -pub struct ConnectionManagerError(String); +pub struct ConnectionManagerError(pub String); /// It abstracts pure UDP packets, and allows to implement Connected/Disconnected states. /// This table summary shows where exactly ConnectionManager sits in between different layers. @@ -69,7 +69,7 @@ pub struct ConnectionManagerError(String); /// Only when packet is recevied, or action is initiated by user it is allowed to change connection state. /// From the point of view of connection manager, laminar's header + payload is interpreted as user data. /// Distinction between user packet and protocol specific packet is encoded in laminar's packet header. -pub trait ConnectionManager: Debug { +pub trait ConnectionManager: Debug + Send { /// This function should be called frequently, even if there is no packets to send or receive. /// It will always be called last, after all other methods is called, so it could send packets or comunicate errors if required. /// It cannot change connection state explicitly, instead it can emit errors, and SocketManager will decide when to destroy connection. @@ -80,7 +80,7 @@ pub trait ConnectionManager: Debug { &mut self, buffer: &'a mut [u8], time: Instant, - ) -> Option, &ConnectionState>, ConnectionManagerError>>; + ) -> Option, ConnectionState>, ConnectionManagerError>>; /// This will be called for all incoming data, including packets that were resent by remote host. /// If packet is accepted by laminar's reliability layer `process_protocol_data` will be called immediatelly. @@ -105,7 +105,7 @@ pub trait ConnectionManager: Debug { fn process_protocol_data<'a>( &mut self, data: &'a [u8], - ) -> Result<&ConnectionState, ConnectionManagerError>; + ) -> Result<(), ConnectionManagerError>; /// This will be invoked when player sends connect request, /// Some protocols might provide a way to pass initial connection data @@ -120,9 +120,9 @@ pub trait ConnectionManager: Debug { /// Tracks all sorts of global statistics, and decided whether to create `ConnectionManager` for new connections or not. /// Also decides when connections should be destroyed even if they are in connected state. -pub trait SocketManager: Debug { +pub trait SocketManager: Debug + Send { /// Decide if it is possible to accept/create new connection connection - fn accept_new_connection(&mut self, addr: &SocketAddr) -> Option>; + fn accept_new_connection(&mut self, addr: &SocketAddr, requested_by: TargetHost) -> Option>; /// Returns list of connections that socket manager decided to destroy fn collect_connections_to_destroy(&mut self) -> Option>; @@ -137,121 +137,12 @@ pub trait SocketManager: Debug { } - - -//======================================= demo implementations ========================================== - -use crate::packet::{DeliveryGuarantee, OrderingGuarantee, OutgoingPacketBuilder, PacketType}; -use log::error; -use std::io::ErrorKind::WouldBlock; - - -/// Simple connection manager, sends "connect" and "disconnect" messages and changes states when receive either of theses messages -#[derive(Debug, Default)] -struct SimpleConnectionManager { - state: ConnectionState, - send: Option>, -} - -impl ConnectionManager for SimpleConnectionManager { - fn update<'a>( - &mut self, - buffer: &'a mut [u8], - time: Instant, - ) -> Option, &ConnectionState>, ConnectionManagerError>> { - match self.send.take() { - Some(data) => { - // copy from buffer what we want to send - buffer.copy_from_slice(data.as_ref()); - // create packet and set packet type ConnectionManager, so that it can be processed by `process_protocol_data` in remote host - Some(Ok(Either::Left( - OutgoingPacketBuilder::new(buffer) - .with_default_header( - PacketType::ConnectionManager, - DeliveryGuarantee::Reliable, - OrderingGuarantee::Ordered(None), - ) - .build(), - ))) - } - None => None, - } - } - - fn preprocess_incoming<'a, 'b>( - &mut self, - data: &'a [u8], - _buffer: &'b mut [u8], - ) -> Result<&'b [u8], ConnectionManagerError> - where - 'a: 'b, - { - Ok(data) - } - - fn postprocess_outgoing<'a, 'b>(&mut self, data: &'a [u8], _buffer: &'b mut [u8]) -> &'b [u8] - where - 'a: 'b, - { - data - } - - fn process_protocol_data<'a>( - &mut self, - data: &'a [u8], - ) -> Result<&ConnectionState, ConnectionManagerError> { - if data.starts_with("connect".as_bytes()) { - self.state - .try_change(&ConnectionState::Connected(Box::from(data.split_at(7).1))); - self.send = Some(Box::from("connected".as_bytes())); - } else if data.eq("connected".as_bytes()) { - self.state - .try_change(&ConnectionState::Connected(Box::from(data.split_at(9).1))); - } else if data.eq("disconnect".as_bytes()) { - self.state.try_change(&ConnectionState::Disconnected(ConnectionClosedBy::RemoteHost)); - } else { - return Err(ConnectionManagerError(format!( - "Unknown message type: {:?}", - std::str::from_utf8(data) - ))); - } - Ok(&self.state) - } - - fn connect<'a>(&mut self, data: Box<[u8]>) { - self.send = Some(Box::from(["connect".as_bytes(), data.as_ref()].concat())); - } - - fn disconnect<'a>(&mut self) { - self.state.try_change(&ConnectionState::Disconnected(ConnectionClosedBy::LocalHost)); - self.send = Some(Box::from("disconnect".as_bytes())); - } -} - -/// Simplest implementation of socket manager, always accept connection and never destroy, no matter how many error connection can report -#[derive(Debug)] -struct DumbSocketManager; - -impl SocketManager for DumbSocketManager { - fn accept_new_connection(&mut self, addr: &SocketAddr) -> Option> { - Some(Box::new(SimpleConnectionManager::default())) - } - - fn collect_connections_to_destroy(&mut self) -> Option> { - None - } - - fn track_connection_error(&mut self, addr: &SocketAddr, error: &ErrorKind, error_context: &str) { - match error { - ErrorKind::IOError(ref e) if e.kind() == WouldBlock => {}, - _ => error!("{} ({:?}): {:?}", error_context, addr, error) - } - } - fn track_global_error(&mut self, error: &ErrorKind, error_context: &str) { - error!("{} : {:?}", error_context, error); - } - fn track_sent_bytes(&mut self, addr: &SocketAddr, bytes: usize) {} - fn track_received_bytes(&mut self, addr: &SocketAddr, bytes: usize) {} - fn track_ignored_bytes(&mut self, addr: &SocketAddr, bytes: usize) {} - fn track_connection_destroyed(&mut self, addr: &SocketAddr) {} -} +pub struct GenericPacket<'a> { + packet_type: PacketType, + /// the raw payload of the packet + payload: &'a [u8], + /// defines on how the packet will be delivered. + delivery: DeliveryGuarantee, + /// defines on how the packet will be ordered. + ordering: OrderingGuarantee, +} \ No newline at end of file diff --git a/src/net/socket.rs b/src/net/socket.rs index 80e8826f..49d8ac2d 100644 --- a/src/net/socket.rs +++ b/src/net/socket.rs @@ -1,8 +1,9 @@ use crate::{ config::Config, + either::Either, error::{ErrorKind, Result}, net::{connection::ActiveConnections, link_conditioner::LinkConditioner}, - net::events::{ConnectionEvent, SendEvent, ReceiveEvent, DestroyReason}, + net::events::{ConnectionEvent, SendEvent, ReceiveEvent, DestroyReason, TargetHost}, net::managers::{SocketManager, ConnectionManager}, packet::{DeliveryGuarantee, Outgoing, Packet}, }; @@ -52,6 +53,34 @@ impl SocketWithConditioner { } +/// Wraps crossbeam_channel sender with convenient functions: `connect`, `send`, `disconnect` +pub struct SocketEventSender(pub Sender>); + +impl SocketEventSender { + /// Constructs ConnectionEvent Packet event from provided packet + pub fn send(&self, packet: Packet) -> std::result::Result<(), SendError> { + self.0.send(ConnectionEvent(packet.addr(), SendEvent::Packet(packet))) + .map_err(|err| SendError(match (err.0).1 { + SendEvent::Packet(data) => data, + _ => unreachable!() + })) + } + + /// Constructs ConnectionEvent Connect event from provided payload + pub fn connect(&self, addr: SocketAddr, payload: Box<[u8]>) -> std::result::Result<(), SendError>> { + self.0.send(ConnectionEvent(addr, SendEvent::Connect(payload))) + .map_err(|err| SendError(match (err.0).1 { + SendEvent::Connect(data) => data, + _ => unreachable!() + })) + } + + /// Constructs ConnectionEvent Disconnect event + pub fn disconnect(&self, addr: SocketAddr) -> std::result::Result<(), SendError<()>> { + self.0.send(ConnectionEvent(addr, SendEvent::Disconnect)) + .map_err(|_| SendError(())) + } +} /// A reliable UDP socket implementation with configurable reliability and ordering guarantees. @@ -132,7 +161,7 @@ impl Socket { /// Returns a handle to the packet sender which provides a thread-safe way to enqueue packets /// to be processed. This should be used when the socket is busy running its polling loop in a /// separate thread. - pub fn get_packet_sender(&mut self) -> Sender> { + pub fn get_event_sender(&mut self) -> Sender> { self.sender.clone() } @@ -147,7 +176,7 @@ impl Socket { pub fn send(&mut self, event: ConnectionEvent) -> Result<()> { match self.sender.send(event) { Ok(_) => Ok(()), - Err(error) => Err(ErrorKind::SendError(error)), + Err(error) => Err(ErrorKind::SendError(SendError(Either::Left(error.0)))), } } @@ -234,8 +263,8 @@ impl Socket { /// connection map while informing the user of this by sending an event. fn handle_dead_clients(&mut self) { let dead_addresses = self.connections.dead_connections(); - for address in dead_addresses { - self.connections.remove_connection(&address, &self.event_sender, self.manager.as_mut(), DestroyReason::TooManyPacketsInFlight, "removing dead clients"); + for (address, reason) in dead_addresses { + self.connections.remove_connection(&address, &self.event_sender, self.manager.as_mut(), reason, "removing dead clients"); } } @@ -254,54 +283,64 @@ impl Socket { // Serializes and sends a `Packet` on the socket. On success, returns the number of bytes written. fn send_to(&mut self, event: ConnectionEvent, time: Instant) -> Result { let mut bytes_sent = 0; - let address = &event.addr; - if let Some(connection) = self.connections.try_get(address) { - // TODO maybe these should not depend on send_to method? - let dropped = connection.gather_dropped_packets(); - let mut processed_packets: Vec = dropped - .iter() - .flat_map(|waiting_packet| { - connection.process_outgoing( - &waiting_packet.payload, - // Because a delivery guarantee is only sent with reliable packets - DeliveryGuarantee::Reliable, - // This is stored with the dropped packet because they could be mixed - waiting_packet.ordering_guarantee, - waiting_packet.item_identifier, - time, - ) - }) - .collect(); - - match event.event { - Connect(data) => connection.state_manager.connect(data); - Packet(data) => { - let processed_packet = connection.process_outgoing( - packet.payload(), - packet.delivery_guarantee(), - packet.order_guarantee(), - None, - time, - )?; - processed_packets.push(processed_packet); - } - Disconnect => connection.state_manager.disconnect(); - } - - for processed_packet in processed_packets { - match processed_packet { - Outgoing::Packet(outgoing) => { - bytes_sent += self.socket.send_packet(address, connection.state_manager.as_mut(), &outgoing.contents())?; - } - Outgoing::Fragments(packets) => { - for outgoing in packets { - bytes_sent += self.socket.send_packet(address, connection.state_manager.as_mut(), &outgoing.contents())?; + match (self.connections.try_get(&event.0), event.1) { + (conn, SendEvent::Connect(data)) => { + let connection = if let Some(connection) = conn { + Some(connection) + } else if let Some(conn_manager) = self.manager.accept_new_connection(&event.0, TargetHost::LocalHost) { + self.event_sender.send(ConnectionEvent(event.0, ReceiveEvent::Created))?; + Some(self.connections.get_or_insert_connection(event.0, &self.config, time, conn_manager)) + } else { + None + }; + if let Some(conn) = connection { + conn.state_manager.connect(data); + } + }, + (Some(connection), SendEvent::Packet(packet)) => { + // TODO maybe these should not depend on send_to method? + let dropped = connection.gather_dropped_packets(); + let mut processed_packets: Vec = dropped + .iter() + .flat_map(|waiting_packet| { + connection.process_outgoing( + &waiting_packet.payload, + // Because a delivery guarantee is only sent with reliable packets + DeliveryGuarantee::Reliable, + // This is stored with the dropped packet because they could be mixed + waiting_packet.ordering_guarantee, + waiting_packet.item_identifier, + time, + ) + }) + .collect(); + + let processed_packet = connection.process_outgoing( + packet.payload(), + packet.delivery_guarantee(), + packet.order_guarantee(), + None, + time, + )?; + processed_packets.push(processed_packet); + + for processed_packet in processed_packets { + match processed_packet { + Outgoing::Packet(outgoing) => { + bytes_sent += self.socket.send_packet(&event.0, connection.state_manager.as_mut(), &outgoing.contents())?; + } + Outgoing::Fragments(packets) => { + for outgoing in packets { + bytes_sent += self.socket.send_packet(&event.0, connection.state_manager.as_mut(), &outgoing.contents())?; + } } } - } - } - } + } + }, + (Some(connection), SendEvent::Disconnect) => connection.state_manager.disconnect(), + _ => {} // ignore packet and disconnect event for non existent connection + }; Ok(bytes_sent) } @@ -317,11 +356,8 @@ impl Socket { let connection = if let Some(conn) = self.connections.try_get(&address) { Some(conn) } else { - if let Some(manager) = self.manager.accept_new_connection(&address) { - self.event_sender.send(ConnectionEvent { - addr: address, - event: ReceiveEvent::Created - })?; + if let Some(manager) = self.manager.accept_new_connection(&address, TargetHost::RemoteHost) { + self.event_sender.send(ConnectionEvent(address, ReceiveEvent::Created))?; Some(self.connections.get_or_insert_connection(address, &self.config, time, manager)) } else { None @@ -439,7 +475,7 @@ mod tests { let time = Instant::now(); - let sender = client.get_packet_sender(); + let sender = client.get_event_sender(); let receiver = server.get_event_receiver(); sender diff --git a/src/net/virtual_connection.rs b/src/net/virtual_connection.rs index 1c81c7a3..ff88f9a2 100644 --- a/src/net/virtual_connection.rs +++ b/src/net/virtual_connection.rs @@ -1,7 +1,6 @@ use crate::{ config::Config, error::{ErrorKind, PacketErrorKind, Result}, - either::Either, infrastructure::{ arranging::{Arranging, ArrangingSystem, OrderingSystem, SequencingSystem}, AcknowledgmentHandler, CongestionHandler, Fragmentation, SentPacket, @@ -15,7 +14,7 @@ use crate::{ DeliveryGuarantee, OrderingGuarantee, Outgoing, OutgoingPacket, OutgoingPacketBuilder, Packet, PacketReader, PacketType, SequenceNumber, }, - SocketEvent, DisconnectReason + net::events::{ConnectionEvent, ReceiveEvent, DisconnectReason} }; use crossbeam_channel::{self, Sender}; @@ -67,6 +66,14 @@ impl VirtualConnection { self.acknowledge_handler.packets_in_flight() > self.config.max_packets_in_flight } + pub fn can_gracefully_disconnect(&self) -> bool { + if let ConnectionState::Disconnected(_) = self.current_state { + self.acknowledge_handler.packets_in_flight() == 0 + } else { + false + } + } + /// Returns a [Duration] representing the interval since we last heard from the client pub fn last_heard(&self, time: Instant) -> Duration { // TODO: Replace with saturating_duration_since once it becomes stable. @@ -90,33 +97,18 @@ impl VirtualConnection { self.last_sent = time; self.congestion_handler .process_outgoing(self.acknowledge_handler.local_sequence_num(), time); - // TODO pass in socket and link conditioner, so that i could send immediatelly - let packet = OutgoingPacketBuilder::new(&[]) + + OutgoingPacketBuilder::new(&[]) .with_default_header( PacketType::Heartbeat, DeliveryGuarantee::Unreliable, OrderingGuarantee::None, ) - .build(); - packet - // let mut buffer = [0;1500]; - // let res = self.state_manager.postprocess_outgoing(&packet.contents(), &mut buffer); - + .build() } - pub fn process_outgoing<'a>( - &mut self, - payload: &'a [u8], - delivery_guarantee: DeliveryGuarantee, - ordering_guarantee: OrderingGuarantee, - last_item_identifier: Option, - time: Instant, - ) -> Result> { - self.process_outgoing_impl(payload, delivery_guarantee, ordering_guarantee, last_item_identifier, time) - } - /// This will pre-process the given buffer to be sent over the network. - fn process_outgoing_impl<'a>( + pub fn process_outgoing<'a>( &mut self, payload: &'a [u8], delivery_guarantee: DeliveryGuarantee, @@ -261,14 +253,14 @@ impl VirtualConnection { pub fn process_incoming( &mut self, received_data: &[u8], - sender: &Sender, + sender: &Sender>, socket_manager: &mut dyn SocketManager, time: Instant, ) -> crate::Result<()> { // TODO pass buffer from somewhere else let mut buffer = [0;1500]; let received_data = self.state_manager.preprocess_incoming(received_data, &mut buffer)?; - + self.last_heard = time; let mut packet_reader = PacketReader::new(received_data); @@ -277,7 +269,7 @@ impl VirtualConnection { if !header.is_current_protocol() { return Err(ErrorKind::ProtocolVersionMismatch); - } + } if header.is_heartbeat() { // Heartbeat packets are unreliable, unordered and empty packets. @@ -383,6 +375,8 @@ impl VirtualConnection { let payload = packet_reader.read_payload(); + println!("======================process_incoming ordered: {:?} {}", header, String::from_utf8_lossy(payload.as_ref())); + let stream = self .ordering_system .get_or_create_stream(arranging_header.stream_id()); @@ -431,26 +425,17 @@ impl VirtualConnection { fn process_packet( &mut self, is_connection_manager_packet: bool, - sender: &Sender, + sender: &Sender>, payload: Box<[u8]>, delivery: DeliveryGuarantee, ordering: OrderingGuarantee ) -> Result<()> { if !is_connection_manager_packet { if let ConnectionState::Connected(_) = self.current_state { - sender.send(SocketEvent::Packet(Packet::new(self.remote_address, payload, delivery, ordering)))?; + sender.send(ConnectionEvent(self.remote_address, ReceiveEvent::Packet(Packet::new(self.remote_address, payload, delivery, ordering))))?; } } else { - let new_state = self.state_manager.process_protocol_data(payload.as_ref())?; - if let Some(_) = self.current_state.try_change(new_state) { - match &self.current_state { - ConnectionState::Connected(data) => sender.send(SocketEvent::Connected(self.remote_address, data.clone()))?, - ConnectionState::Disconnected(reason) => sender.send(SocketEvent::Disconnected(DisconnectReason::ClosedBy(reason.clone())))?, - ConnectionState::Connecting => self.reset_connection(), - }; - } else { - panic!("Invalid state transition {:?} -> {:?}", self.current_state, new_state) - } + self.state_manager.process_protocol_data(payload.as_ref())?; } Ok(()) } @@ -458,19 +443,19 @@ impl VirtualConnection { fn process_packet_and_log_errors( &mut self, is_connection_manager_packet: bool, - sender: &Sender, + sender: &Sender>, socket_manager: &mut dyn SocketManager, payload: Box<[u8]>, delivery: DeliveryGuarantee, ordering: OrderingGuarantee - ) { + ) { if let Err(ref error) = self.process_packet(is_connection_manager_packet, sender, payload, delivery, ordering) { socket_manager.track_connection_error(&self.remote_address, error, "processing incomming packet"); } } - /// this function fully reset connection to initial state, after ConnectionManager switches from Disconnected to Connecting - fn reset_connection(&mut self) { + /// Fully reset connection to initial state, after ConnectionManager switches from Disconnected to Connecting + pub fn reset_connection(&mut self) { let time = Instant::now(); self.last_heard = time; self.last_sent= time; From 30f7807753ac95f6a733916178af970f3e69672f Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Mon, 16 Sep 2019 21:09:09 +0300 Subject: [PATCH 09/14] WIP server_client example in progress --- examples/server_client.rs | 7 +++-- src/managers/simple.rs | 49 ++++++++++++++++------------------- src/net/connection.rs | 12 +++++++-- src/net/managers.rs | 10 +++---- src/net/socket.rs | 4 ++- src/net/virtual_connection.rs | 16 +++++++++--- 6 files changed, 57 insertions(+), 41 deletions(-) diff --git a/examples/server_client.rs b/examples/server_client.rs index 0464f51b..1b7d719c 100644 --- a/examples/server_client.rs +++ b/examples/server_client.rs @@ -16,12 +16,14 @@ fn server() -> Result<(), ErrorKind> { loop { if let Ok(ConnectionEvent(addr, event)) = receiver.recv() { + match event { ReceiveEvent::Created => { println!("Connection created {:?}", addr); }, ReceiveEvent::Connected(data) => { println!("Connected {:?} with message: {}",addr, String::from_utf8_lossy(data.as_ref())); + //sender.disconnect(addr); }, ReceiveEvent::Packet(packet) => { let msg = packet.payload(); @@ -38,7 +40,7 @@ fn server() -> Result<(), ErrorKind> { sender .send(Packet::reliable_unordered( packet.addr(), - "Copy that!".as_bytes().to_vec(), + ["Echo: ".as_bytes(), msg.as_bytes()].concat(), )) .expect("This should send"); }, @@ -87,7 +89,7 @@ fn client() -> Result<(), ErrorKind> { }, ReceiveEvent::Connected(data) => { println!("Connected {:?} with message: {}",addr, String::from_utf8_lossy(data.as_ref())); - socket.send(ConnectionEvent(server, SendEvent::Disconnect))?; + socket.send(ConnectionEvent(server, SendEvent::Packet(Packet::reliable_unordered(server, "Copy that!".as_bytes().to_vec()))))?; }, ReceiveEvent::Packet(packet) => { let msg = packet.payload(); @@ -100,6 +102,7 @@ fn client() -> Result<(), ErrorKind> { let ip = packet.addr().ip(); println!("Received {:?} from {:?}", msg, ip); + socket.send(ConnectionEvent(server, SendEvent::Disconnect))?; // sender // .send(Packet::reliable_unordered( diff --git a/src/managers/simple.rs b/src/managers/simple.rs index 5634bcf9..9645c645 100644 --- a/src/managers/simple.rs +++ b/src/managers/simple.rs @@ -1,6 +1,6 @@ use crate::net::managers::*; -use crate::packet::{DeliveryGuarantee, OrderingGuarantee, OutgoingPacketBuilder, PacketType}; +use crate::packet::{DeliveryGuarantee, OrderingGuarantee, PacketType}; use log::error; use std::io::ErrorKind::WouldBlock; use std::net::SocketAddr; @@ -25,20 +25,26 @@ impl SimpleConnectionManager { self.changes.push_back(Either::Left(Box::from(payload))); } - fn get_packet<'a> (data: Box<[u8]>, buffer: &'a mut [u8]) -> OutgoingPacket<'a> { + fn get_packet<'a> (data: Box<[u8]>, buffer: &'a mut [u8]) -> GenericPacket<'a> { // get result slice - let res_buffer = &mut buffer[0..data.as_ref().len()]; + let payload = &mut buffer[0..data.as_ref().len()]; // copy from buffer what we want to send - res_buffer.copy_from_slice(data.as_ref()); - println!("===========================update send {}", String::from_utf8_lossy(res_buffer)); + payload.copy_from_slice(data.as_ref()); + if payload.len() == 0 { + return GenericPacket{ + packet_type: PacketType::ConnectionManager, + payload, + delivery: DeliveryGuarantee::Unreliable, + ordering: OrderingGuarantee::None + } + } // create packet - OutgoingPacketBuilder::new(res_buffer) - .with_default_header( - PacketType::ConnectionManager, - DeliveryGuarantee::Unreliable, - OrderingGuarantee::None, - ) - .build() + GenericPacket{ + packet_type: PacketType::ConnectionManager, + payload, + delivery: DeliveryGuarantee::Reliable, + ordering: OrderingGuarantee::None + } } } @@ -47,7 +53,7 @@ impl ConnectionManager for SimpleConnectionManager { &mut self, buffer: &'a mut [u8], _time: Instant, - ) -> Option, ConnectionState>, ConnectionManagerError>> { + ) -> Option, ConnectionState>, ConnectionManagerError>> { match self.changes.pop_front().take() { Some(change) => { Some(Ok(match change { @@ -67,10 +73,6 @@ impl ConnectionManager for SimpleConnectionManager { where 'a: 'b, { - println!( - "===========================preprocess_incoming: {}", - String::from_utf8_lossy(data) - ); Ok(data) } @@ -78,24 +80,17 @@ impl ConnectionManager for SimpleConnectionManager { where 'a: 'b, { - println!( - "===========================postprocess_outgoing: {}", - String::from_utf8_lossy(data) - ); data } fn process_protocol_data<'a>(&mut self, data: &'a [u8]) -> Result<(), ConnectionManagerError> { - println!( - "===========================process_protocol_data: {}", - String::from_utf8_lossy(data) - ); if data.starts_with("connect-".as_bytes()) { self.change_state(ConnectionState::Connected(Box::from(data.split_at(8).1))); - self.send_packet("connected-".as_bytes()); + self.send_packet("connected-".as_bytes()); } else if data.starts_with("connected-".as_bytes()) { self.change_state(ConnectionState::Connected(Box::from(data.split_at(10).1))); } else if data.starts_with("disconnect".as_bytes()) { + self.send_packet("".as_bytes()); self.change_state(ConnectionState::Disconnected(TargetHost::RemoteHost)); } else { return Err(ConnectionManagerError(format!( @@ -111,8 +106,8 @@ impl ConnectionManager for SimpleConnectionManager { } fn disconnect<'a>(&mut self) { - self.change_state(ConnectionState::Disconnected(TargetHost::LocalHost)); self.send_packet("disconnect".as_bytes()); + self.change_state(ConnectionState::Disconnected(TargetHost::LocalHost)); } } diff --git a/src/net/connection.rs b/src/net/connection.rs index 51edf50b..ddc25103 100644 --- a/src/net/connection.rs +++ b/src/net/connection.rs @@ -6,6 +6,7 @@ use crate::{ net::managers::{ConnectionState, SocketManager}, net::socket::SocketWithConditioner, ErrorKind, + packet::{Outgoing} }; use crossbeam_channel::{self, Sender, SendError}; @@ -134,7 +135,14 @@ impl ActiveConnections { Some(result) => match result { Ok(event) => match event { Either::Left(packet) => { - socket.send_packet_and_log(&conn.remote_address, conn.state_manager.as_mut(), &packet.contents(), manager, "sending packet from connection manager"); + // TODO properly handle error, instead of assert + let packet = conn.process_outgoing(packet.packet_type, packet.payload, packet.delivery, packet.ordering, None, time) + .expect("connection manager packet should not fail"); + if let Outgoing::Packet(outgoing) = packet { + socket.send_packet_and_log(&conn.remote_address, conn.state_manager.as_mut(), &outgoing.contents(), manager, "sending packet from connection manager"); + } else { + panic!("connection manager cannot send fragmented packets"); + } }, Either::Right(state) => { if let Some(_) = conn.current_state.try_change(&state) { @@ -144,7 +152,7 @@ impl ActiveConnections { ConnectionState::Disconnected(closed_by) => sender.send(ConnectionEvent(conn.remote_address, ReceiveEvent::Disconnected(DisconnectReason::ClosedBy(closed_by.clone())))), ConnectionState::Connecting => { - conn.reset_connection(); + conn.reset_connection(time); Ok(()) }, } { diff --git a/src/net/managers.rs b/src/net/managers.rs index 2f2b84b9..c4544b8f 100644 --- a/src/net/managers.rs +++ b/src/net/managers.rs @@ -80,7 +80,7 @@ pub trait ConnectionManager: Debug + Send { &mut self, buffer: &'a mut [u8], time: Instant, - ) -> Option, ConnectionState>, ConnectionManagerError>>; + ) -> Option, ConnectionState>, ConnectionManagerError>>; /// This will be called for all incoming data, including packets that were resent by remote host. /// If packet is accepted by laminar's reliability layer `process_protocol_data` will be called immediatelly. @@ -138,11 +138,11 @@ pub trait SocketManager: Debug + Send { pub struct GenericPacket<'a> { - packet_type: PacketType, + pub packet_type: PacketType, /// the raw payload of the packet - payload: &'a [u8], + pub payload: &'a [u8], /// defines on how the packet will be delivered. - delivery: DeliveryGuarantee, + pub delivery: DeliveryGuarantee, /// defines on how the packet will be ordered. - ordering: OrderingGuarantee, + pub ordering: OrderingGuarantee, } \ No newline at end of file diff --git a/src/net/socket.rs b/src/net/socket.rs index 49d8ac2d..65237b3a 100644 --- a/src/net/socket.rs +++ b/src/net/socket.rs @@ -5,7 +5,7 @@ use crate::{ net::{connection::ActiveConnections, link_conditioner::LinkConditioner}, net::events::{ConnectionEvent, SendEvent, ReceiveEvent, DestroyReason, TargetHost}, net::managers::{SocketManager, ConnectionManager}, - packet::{DeliveryGuarantee, Outgoing, Packet}, + packet::{DeliveryGuarantee, Outgoing, Packet, PacketType}, }; use crossbeam_channel::{self, unbounded, Receiver, SendError, Sender, TryRecvError}; use log::error; @@ -305,6 +305,7 @@ impl Socket { .iter() .flat_map(|waiting_packet| { connection.process_outgoing( + PacketType::Packet, &waiting_packet.payload, // Because a delivery guarantee is only sent with reliable packets DeliveryGuarantee::Reliable, @@ -317,6 +318,7 @@ impl Socket { .collect(); let processed_packet = connection.process_outgoing( + PacketType::Packet, packet.payload(), packet.delivery_guarantee(), packet.order_guarantee(), diff --git a/src/net/virtual_connection.rs b/src/net/virtual_connection.rs index ff88f9a2..9809ae7d 100644 --- a/src/net/virtual_connection.rs +++ b/src/net/virtual_connection.rs @@ -110,6 +110,7 @@ impl VirtualConnection { /// This will pre-process the given buffer to be sent over the network. pub fn process_outgoing<'a>( &mut self, + packet_type: PacketType, payload: &'a [u8], delivery_guarantee: DeliveryGuarantee, ordering_guarantee: OrderingGuarantee, @@ -120,7 +121,7 @@ impl VirtualConnection { DeliveryGuarantee::Unreliable => { if payload.len() <= self.config.receive_buffer_max_size { let mut builder = OutgoingPacketBuilder::new(payload).with_default_header( - PacketType::Packet, + packet_type, delivery_guarantee, ordering_guarantee, ); @@ -149,7 +150,7 @@ impl VirtualConnection { // spit the packet if the payload length is greater than the allowed fragment size. if payload_length <= self.config.fragment_size { let mut builder = OutgoingPacketBuilder::new(payload).with_default_header( - PacketType::Packet, + packet_type, delivery_guarantee, ordering_guarantee, ); @@ -435,6 +436,14 @@ impl VirtualConnection { sender.send(ConnectionEvent(self.remote_address, ReceiveEvent::Packet(Packet::new(self.remote_address, payload, delivery, ordering))))?; } } else { + // when we're in disconnected state, and receive empty message, + // this means that remote host got our disconnect message + if let ConnectionState::Disconnected(_) = self.current_state { + if payload.as_ref().len() == 0 { + self.acknowledge_handler = AcknowledgmentHandler::new(); + return Ok(()); + } + } self.state_manager.process_protocol_data(payload.as_ref())?; } Ok(()) @@ -455,8 +464,7 @@ impl VirtualConnection { } /// Fully reset connection to initial state, after ConnectionManager switches from Disconnected to Connecting - pub fn reset_connection(&mut self) { - let time = Instant::now(); + pub fn reset_connection(&mut self, time: Instant) { self.last_heard = time; self.last_sent= time; self.ordering_system = OrderingSystem::new(); From 766c5c882dec2799479a164f86e55ccb39f72abc Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Tue, 17 Sep 2019 15:01:25 +0300 Subject: [PATCH 10/14] WIP server_client example working + more doc comments --- examples/server_client.rs | 129 ++++++------- examples/simple_udp.rs | 2 +- src/error.rs | 8 +- src/infrastructure/fragmenter.rs | 8 +- src/lib.rs | 6 +- src/managers.rs | 2 +- src/managers/simple.rs | 158 ++++++++++------ src/net.rs | 2 +- src/net/connection.rs | 212 +++++++++++++-------- src/net/events.rs | 85 +++++---- src/net/managers.rs | 89 ++++----- src/net/socket.rs | 304 +++++++++++++++++++++---------- src/net/virtual_connection.rs | 114 ++++++------ src/packet.rs | 2 +- src/packet/enums.rs | 2 +- src/packet/packet_structure.rs | 54 +++++- 16 files changed, 723 insertions(+), 454 deletions(-) diff --git a/examples/server_client.rs b/examples/server_client.rs index 1b7d719c..17b6420e 100644 --- a/examples/server_client.rs +++ b/examples/server_client.rs @@ -5,37 +5,39 @@ use std::io::stdin; use std::thread; use std::time::Instant; -use laminar::{ErrorKind, Packet, Socket, SocketEventSender, managers::SimpleSocketManager, ConnectionEvent, ReceiveEvent, SendEvent}; +use laminar::{ + managers::SimpleSocketManager, ConnectionEvent, ErrorKind, Packet, ReceiveEvent, Socket, + SocketEventSender, +}; const SERVER: &str = "127.0.0.1:12351"; fn server() -> Result<(), ErrorKind> { - let mut socket = Socket::bind(SERVER, Box::new(SimpleSocketManager))?; - let (sender, receiver) = (SocketEventSender(socket.get_event_sender()), socket.get_event_receiver()); + // create socket manager, that will use SimpleConnectionManager, that actually initiates connection by exchanging methods + let mut socket = Socket::bind(SERVER, Box::new(SimpleSocketManager(false)))?; + let (sender, receiver) = ( + SocketEventSender(socket.get_event_sender()), + socket.get_event_receiver(), + ); let _thread = thread::spawn(move || socket.start_polling()); loop { if let Ok(ConnectionEvent(addr, event)) = receiver.recv() { - match event { - ReceiveEvent::Created => { - println!("Connection created {:?}", addr); - }, ReceiveEvent::Connected(data) => { - println!("Connected {:?} with message: {}",addr, String::from_utf8_lossy(data.as_ref())); - //sender.disconnect(addr); - }, + println!( + "{:?} -> Connected msg:{}", + addr, + String::from_utf8_lossy(data.as_ref()) + ); + } ReceiveEvent::Packet(packet) => { let msg = packet.payload(); - if msg == b"Bye!" { break; } - let msg = String::from_utf8_lossy(msg); - let ip = packet.addr().ip(); - - println!("Received {:?} from {:?}", msg, ip); + println!("{:?} -> Packet msg:{}", addr, msg); sender .send(Packet::reliable_unordered( @@ -43,13 +45,8 @@ fn server() -> Result<(), ErrorKind> { ["Echo: ".as_bytes(), msg.as_bytes()].concat(), )) .expect("This should send"); - }, - ReceiveEvent::Disconnected(reason) => { - println!("Disconnected {:?} reason: {:?}", addr, reason); - }, - ReceiveEvent::Destroyed(reason) => { - println!("Connection destroyed {:?} reason: {:?}", addr, reason); } + _ => println!("{:?} -> {:?}", addr, event), } } } @@ -59,66 +56,58 @@ fn server() -> Result<(), ErrorKind> { fn client() -> Result<(), ErrorKind> { let addr = "127.0.0.1:12352"; - let mut socket = Socket::bind(addr, Box::new(SimpleSocketManager))?; + let mut socket = Socket::bind(addr, Box::new(SimpleSocketManager(false)))?; println!("Connected on {}", addr); - let server = SERVER.parse().unwrap(); - - println!("Type a message and press Enter to send. Send `Bye!` to quit."); - - let stdin = stdin(); - let mut s_buffer = String::new(); - - s_buffer.clear(); - stdin.read_line(&mut s_buffer)?; - let line = s_buffer.replace(|x| x == '\n' || x == '\r', ""); - socket.send(ConnectionEvent(server, SendEvent::Connect(Box::from(line.as_bytes()))))?; - - loop { - + let sender = SocketEventSender(socket.get_event_sender()); + let _thread = thread::spawn(move || loop { socket.manual_poll(Instant::now()); - if line == "Bye!" { - break; - } - if let Some(ConnectionEvent(addr, event)) = socket.recv() { match event { - ReceiveEvent::Created => { - println!("Connection created {:?}", addr); - }, ReceiveEvent::Connected(data) => { - println!("Connected {:?} with message: {}",addr, String::from_utf8_lossy(data.as_ref())); - socket.send(ConnectionEvent(server, SendEvent::Packet(Packet::reliable_unordered(server, "Copy that!".as_bytes().to_vec()))))?; - }, + println!( + "{:?} -> Connected msg:{}", + addr, + String::from_utf8_lossy(data.as_ref()) + ); + } ReceiveEvent::Packet(packet) => { - let msg = packet.payload(); - - if msg == b"Bye!" { - break; - } - - let msg = String::from_utf8_lossy(msg); - let ip = packet.addr().ip(); - - println!("Received {:?} from {:?}", msg, ip); - socket.send(ConnectionEvent(server, SendEvent::Disconnect))?; - - // sender - // .send(Packet::reliable_unordered( - // packet.addr(), - // "Copy that!".as_bytes().to_vec(), - // )) - // .expect("This should send"); - }, - ReceiveEvent::Disconnected(reason) => { - println!("Disconnected {:?} reason: {:?}", addr, reason); - }, - ReceiveEvent::Destroyed(reason) => { - println!("Connection destroyed {:?} reason: {:?}", addr, reason); + let msg = String::from_utf8_lossy(packet.payload()); + println!("{:?} -> Packet msg:{}", addr, msg); } + _ => println!("{:?} -> {:?}", addr, event), } } + }); + + let stdin = stdin(); + let mut s_buffer = String::new(); + s_buffer.clear(); + + let server = SERVER.parse().unwrap(); + println!("Type a `:c` to connect"); + println!("Type a `` to send a packet"); + println!("Type a `:d` to disconnect"); + println!("Type a `:q` to quit."); + + loop { + stdin.read_line(&mut s_buffer)?; + let line = s_buffer.replace(|x| x == '\n' || x == '\r', ""); + if line == ":q" { + break; + } else if line.starts_with(":c") { + sender + .connect(server, Box::from(line.split_at(2).1.as_bytes())) + .expect("sending should not fail"); + } else if line == ":d" { + sender.disconnect(server).expect("sending should not fail"); + } else { + sender + .send(Packet::reliable_unordered(server, line.into_bytes())) + .expect("sending should not fail"); + } + s_buffer.clear(); } Ok(()) diff --git a/examples/simple_udp.rs b/examples/simple_udp.rs index a2a37b93..9feb507a 100644 --- a/examples/simple_udp.rs +++ b/examples/simple_udp.rs @@ -3,7 +3,7 @@ //! 2. setting up client to send data. //! 3. serialize data to send and deserialize when received. use bincode::{deserialize, serialize}; -use laminar::{Packet, Socket, ConnectionEventSender}; +use laminar::{ConnectionEventSender, Packet, Socket}; use serde_derive::{Deserialize, Serialize}; use std::net::SocketAddr; use std::time::Instant; diff --git a/src/error.rs b/src/error.rs index 25a39015..4d47a8e8 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,14 +1,14 @@ //! This module contains the laminar error handling logic. +use crate::either::Either; +use crate::net::events::{ConnectionEvent, ReceiveEvent, SendEvent}; +use crate::net::managers::ConnectionManagerError; use crossbeam_channel::SendError; use std::{ error::Error, fmt::{self, Display, Formatter}, io, result, }; -use crate::net::managers::ConnectionManagerError; -use crate::net::events::{ConnectionEvent, SendEvent, ReceiveEvent}; -use crate::either::Either; /// Wrapped result type for Laminar errors. pub type Result = result::Result; @@ -75,7 +75,7 @@ impl Display for ErrorKind { fmt, "Something when wrong in ConnectionManager. Reason: {:?}.", err - ) + ), } } } diff --git a/src/infrastructure/fragmenter.rs b/src/infrastructure/fragmenter.rs index 47ba8ce2..e5c62ab4 100644 --- a/src/infrastructure/fragmenter.rs +++ b/src/infrastructure/fragmenter.rs @@ -71,7 +71,7 @@ impl Fragmentation { Fragmentation::fragments_needed(payload_length, config.fragment_size) as u8; if num_fragments > config.max_fragments { - Err(FragmentErrorKind::ExceededMaxFragments)?; + return Err(FragmentErrorKind::ExceededMaxFragments.into()); } for fragment_id in 0..num_fragments { @@ -112,16 +112,16 @@ impl Fragmentation { // get entry of previous received fragments let reassembly_data = match self.fragments.get_mut(fragment_header.sequence()) { Some(val) => val, - None => Err(FragmentErrorKind::CouldNotFindFragmentById)?, + None => return Err(FragmentErrorKind::CouldNotFindFragmentById.into()), }; // Got the data if reassembly_data.num_fragments_total != fragment_header.fragment_count() { - Err(FragmentErrorKind::FragmentWithUnevenNumberOfFragemts)? + return Err(FragmentErrorKind::FragmentWithUnevenNumberOfFragemts.into()); } if reassembly_data.fragments_received[usize::from(fragment_header.id())] { - Err(FragmentErrorKind::AlreadyProcessedFragment)? + return Err(FragmentErrorKind::AlreadyProcessedFragment.into()); } // increase number of received fragments and set the specific fragment to received. diff --git a/src/lib.rs b/src/lib.rs index 911a8da0..0e800b75 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,11 +26,11 @@ mod config; mod either; mod error; mod infrastructure; +pub mod managers; mod net; mod packet; mod protocol_version; mod sequence_buffer; -pub mod managers; #[cfg(feature = "tester")] mod throughput; @@ -40,8 +40,8 @@ pub use self::throughput::ThroughputMonitoring; pub use self::config::Config; pub use self::error::{ErrorKind, Result}; +pub use self::net::managers::{ConnectionManager, ConnectionManagerError}; pub use self::net::{LinkConditioner, Socket, SocketEventSender}; pub use self::packet::{DeliveryGuarantee, OrderingGuarantee, Packet}; -pub use self::net::managers::{ConnectionManager, ConnectionManagerError}; // pub use self::managers::{SimpleSocketManager}; -pub use self::net::events::*; \ No newline at end of file +pub use self::net::events::*; diff --git a/src/managers.rs b/src/managers.rs index dc3ce27d..402d60d2 100644 --- a/src/managers.rs +++ b/src/managers.rs @@ -1,4 +1,4 @@ //! This module provides socket managers. mod simple; -pub use self::simple::SimpleSocketManager; \ No newline at end of file +pub use self::simple::SimpleSocketManager; diff --git a/src/managers/simple.rs b/src/managers/simple.rs index 9645c645..1fe97c2e 100644 --- a/src/managers/simple.rs +++ b/src/managers/simple.rs @@ -1,13 +1,61 @@ use crate::net::managers::*; -use crate::packet::{DeliveryGuarantee, OrderingGuarantee, PacketType}; +use crate::packet::{DeliveryGuarantee, OrderingGuarantee}; use log::error; +use std::collections::VecDeque; use std::io::ErrorKind::WouldBlock; use std::net::SocketAddr; use std::time::Instant; -use std::collections::VecDeque; -/// Simple connection manager, sends "connect" and "disconnect" messages and changes states when receive either of these messages +/// The simplest connection manager, that immediately goes into connected state, after creating it +#[derive(Debug, Default)] +struct AlwaysConnectedConManager { + connected: bool, +} + +impl ConnectionManager for AlwaysConnectedConManager { + fn update<'a>( + &mut self, + _buffer: &'a mut [u8], + _time: Instant, + ) -> Option, ConnectionState>, ConnectionManagerError>> { + if !self.connected { + self.connected = true; + return Some(Ok(Either::Right( + ConnectionState::Connected(Box::default()), + ))); + } + None + } + + fn preprocess_incoming<'a, 'b>( + &mut self, + data: &'a [u8], + _buffer: &'b mut [u8], + ) -> Result<&'b [u8], ConnectionManagerError> + where + 'a: 'b, + { + Ok(data) + } + + fn postprocess_outgoing<'a, 'b>(&mut self, data: &'a [u8], _buffer: &'b mut [u8]) -> &'b [u8] + where + 'a: 'b, + { + data + } + + fn process_protocol_data(&mut self, _data: &[u8]) -> Result<(), ConnectionManagerError> { + Ok(()) + } + + fn connect(&mut self, _data: Box<[u8]>) {} + + fn disconnect(&mut self) {} +} + +/// Simple connection manager, that actually tries to connect by exchanging "connect", "connected", and "disconnect" messages with remote host, #[derive(Debug, Default)] struct SimpleConnectionManager { state: ConnectionState, @@ -16,35 +64,26 @@ struct SimpleConnectionManager { impl SimpleConnectionManager { fn change_state(&mut self, new: ConnectionState) { - if let Some(_) = self.state.try_change(&new) { + if self.state.try_change(&new).is_some() { self.changes.push_back(Either::Right(self.state.clone())); } } - fn send_packet(&mut self, payload:&[u8]) { + fn send_packet(&mut self, payload: &[u8]) { self.changes.push_back(Either::Left(Box::from(payload))); } - fn get_packet<'a> (data: Box<[u8]>, buffer: &'a mut [u8]) -> GenericPacket<'a> { + fn get_packet<'a>(data: Box<[u8]>, buffer: &'a mut [u8]) -> GenericPacket<'a> { // get result slice - let payload = &mut buffer[0..data.as_ref().len()]; + let payload = &mut buffer[0..data.as_ref().len()]; // copy from buffer what we want to send payload.copy_from_slice(data.as_ref()); - if payload.len() == 0 { - return GenericPacket{ - packet_type: PacketType::ConnectionManager, - payload, - delivery: DeliveryGuarantee::Unreliable, - ordering: OrderingGuarantee::None - } - } // create packet - GenericPacket{ - packet_type: PacketType::ConnectionManager, + GenericPacket::manager_packet( payload, - delivery: DeliveryGuarantee::Reliable, - ordering: OrderingGuarantee::None - } + DeliveryGuarantee::Reliable, + OrderingGuarantee::None, + ) } } @@ -53,14 +92,14 @@ impl ConnectionManager for SimpleConnectionManager { &mut self, buffer: &'a mut [u8], _time: Instant, - ) -> Option, ConnectionState>, ConnectionManagerError>> { + ) -> Option, ConnectionState>, ConnectionManagerError>> { match self.changes.pop_front().take() { - Some(change) => { - Some(Ok(match change { - Either::Left(data) => Either::Left(SimpleConnectionManager::get_packet(data, buffer)), - Either::Right(state) => Either::Right(state) - })) - } + Some(change) => Some(Ok(match change { + Either::Left(data) => { + Either::Left(SimpleConnectionManager::get_packet(data, buffer)) + } + Either::Right(state) => Either::Right(state), + })), None => None, } } @@ -83,46 +122,63 @@ impl ConnectionManager for SimpleConnectionManager { data } - fn process_protocol_data<'a>(&mut self, data: &'a [u8]) -> Result<(), ConnectionManagerError> { - if data.starts_with("connect-".as_bytes()) { - self.change_state(ConnectionState::Connected(Box::from(data.split_at(8).1))); - self.send_packet("connected-".as_bytes()); - } else if data.starts_with("connected-".as_bytes()) { - self.change_state(ConnectionState::Connected(Box::from(data.split_at(10).1))); - } else if data.starts_with("disconnect".as_bytes()) { - self.send_packet("".as_bytes()); - self.change_state(ConnectionState::Disconnected(TargetHost::RemoteHost)); - } else { - return Err(ConnectionManagerError(format!( - "Unknown message type: {:?}", - String::from_utf8_lossy(data) - ))); + fn process_protocol_data(&mut self, data: &[u8]) -> Result<(), ConnectionManagerError> { + match self.state { + ConnectionState::Connecting => { + if data.starts_with(b"connect-") { + self.send_packet(b"connected-"); + self.change_state(ConnectionState::Connected(Box::from(data.split_at(8).1))); + } else if data.starts_with(b"connected-") { + self.change_state(ConnectionState::Connected(Box::from(data.split_at(10).1))); + } + } + ConnectionState::Connected(_) => { + if data.eq(b"disconnect") { + self.change_state(ConnectionState::Disconnected(TargetHost::RemoteHost)); + } + } + _ => panic!("In disconnected nothing can happen"), } Ok(()) } - fn connect<'a>(&mut self, data: Box<[u8]>) { - self.send_packet(["connect-".as_bytes(), data.as_ref()].concat().as_ref()); + fn connect(&mut self, data: Box<[u8]>) { + self.send_packet([b"connect-", data.as_ref()].concat().as_ref()); } - fn disconnect<'a>(&mut self) { - self.send_packet("disconnect".as_bytes()); - self.change_state(ConnectionState::Disconnected(TargetHost::LocalHost)); + fn disconnect(&mut self) { + if let ConnectionState::Connected(_) = self.state { + self.send_packet(b"disconnect"); + } + self.change_state(ConnectionState::Disconnected(TargetHost::LocalHost)); } - } /// Simplest implementation of socket manager, always accept connection and never destroy, no matter how many errors connection reports +/// It can create two types of connection managers: +/// * true - creates `AlwaysConnectedConManager` +/// * false - creates `SimpleConnectionManager` #[derive(Debug)] -pub struct SimpleSocketManager; +pub struct SimpleSocketManager(pub bool); impl SocketManager for SimpleSocketManager { - fn accept_new_connection( + fn accept_remote_connection( + &mut self, + addr: &SocketAddr, + _raw_bytes: &[u8], + ) -> Option> { + self.accept_local_connection(addr) + } + + fn accept_local_connection( &mut self, _addr: &SocketAddr, - _requested_by: TargetHost, ) -> Option> { - Some(Box::new(SimpleConnectionManager::default())) + if self.0 { + Some(Box::new(AlwaysConnectedConManager::default())) + } else { + Some(Box::new(SimpleConnectionManager::default())) + } } fn collect_connections_to_destroy(&mut self) -> Option> { diff --git a/src/net.rs b/src/net.rs index 69414992..ddc72e9f 100644 --- a/src/net.rs +++ b/src/net.rs @@ -7,8 +7,8 @@ mod quality; mod socket; mod virtual_connection; -pub mod events; pub mod constants; +pub mod events; pub mod managers; pub use self::link_conditioner::LinkConditioner; diff --git a/src/net/connection.rs b/src/net/connection.rs index ddc25103..250d817f 100644 --- a/src/net/connection.rs +++ b/src/net/connection.rs @@ -1,20 +1,20 @@ -pub use crate::net::{NetworkQuality, RttMeasurer, VirtualConnection, managers::ConnectionManager}; +pub use crate::net::{managers::ConnectionManager, NetworkQuality, RttMeasurer, VirtualConnection}; use crate::{ config::Config, either::Either, - net::events::{ ConnectionEvent, ReceiveEvent, DisconnectReason, DestroyReason}, + net::events::{ConnectionEvent, DestroyReason, DisconnectReason, ReceiveEvent}, net::managers::{ConnectionState, SocketManager}, net::socket::SocketWithConditioner, + packet::Outgoing, ErrorKind, - packet::{Outgoing} }; -use crossbeam_channel::{self, Sender, SendError}; +use crossbeam_channel::{self, SendError, Sender}; use std::{ collections::HashMap, net::SocketAddr, - time::{Duration, Instant}, + time::{Duration, Instant}, }; /// Maintains a registry of active "connections". Essentially, when we receive a packet on the @@ -22,7 +22,7 @@ use std::{ #[derive(Debug)] pub struct ActiveConnections { connections: HashMap, - buffer: Box<[u8]> + buffer: Box<[u8]>, } impl ActiveConnections { @@ -30,7 +30,7 @@ impl ActiveConnections { Self { connections: HashMap::new(), // TODO actually take from config - buffer: Box::new([0;1500]) + buffer: Box::new([0; 1500]), } } @@ -59,18 +59,32 @@ impl ActiveConnections { sender: &Sender>, manager: &mut dyn SocketManager, reason: DestroyReason, - error_context: &str + error_context: &str, ) -> bool { if let Some((_, conn)) = self.connections.remove_entry(address) { - manager.track_connection_destroyed(address); if let ConnectionState::Connected(_) = conn.get_current_state() { - if let Err(err) = sender.send(ConnectionEvent(conn.remote_address, ReceiveEvent::Disconnected(DisconnectReason::Destroying(reason.clone())))) { - manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(SendError(Either::Right(err.0))), error_context); + if let Err(err) = sender.send(ConnectionEvent( + conn.remote_address, + ReceiveEvent::Disconnected(DisconnectReason::Destroying(reason.clone())), + )) { + manager.track_connection_error( + &conn.remote_address, + &ErrorKind::SendError(SendError(Either::Right(err.0))), + error_context, + ); } } - if let Err(err) = sender.send(ConnectionEvent(conn.remote_address, ReceiveEvent::Destroyed(reason))) { - manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(SendError(Either::Right(err.0))), error_context); + if let Err(err) = sender.send(ConnectionEvent( + conn.remote_address, + ReceiveEvent::Destroyed(reason), + )) { + manager.track_connection_error( + &conn.remote_address, + &ErrorKind::SendError(SendError(Either::Right(err.0))), + error_context, + ); } + manager.track_connection_destroyed(address); true } else { false @@ -90,12 +104,20 @@ impl ActiveConnections { pub fn dead_connections(&mut self) -> Vec<(SocketAddr, DestroyReason)> { self.connections .iter() - .filter_map(|(_, connection)| if connection.should_be_dropped() { - Some((connection.remote_address, DestroyReason::TooManyPacketsInFlight)) - } else if connection.can_gracefully_disconnect() { - Some((connection.remote_address, DestroyReason::GracefullyDisconnected)) - } else { - None + .filter_map(|(_, connection)| { + if connection.should_be_dropped() { + Some(( + connection.remote_address, + DestroyReason::TooManyPacketsInFlight, + )) + } else if connection.is_disconnected() { + Some(( + connection.remote_address, + DestroyReason::GracefullyDisconnected, + )) + } else { + None + } }) .collect() } @@ -105,70 +127,117 @@ impl ActiveConnections { &mut self, heartbeat_interval: Duration, time: Instant, - manager:&mut dyn SocketManager, - socket:&mut SocketWithConditioner + manager: &mut dyn SocketManager, + socket: &mut SocketWithConditioner, ) { self.connections .iter_mut() .filter(move |(_, connection)| connection.last_sent(time) >= heartbeat_interval) .for_each(|(_, connection)| { let packet = connection.create_and_process_heartbeat(time); - socket.send_packet_and_log(&connection.remote_address, connection.state_manager.as_mut(), &packet.contents(), manager, "sending heartbeat packet"); + socket.send_packet_and_log( + &connection.remote_address, + connection.state_manager.as_mut(), + &packet.contents(), + manager, + "sending heartbeat packet", + ); }); } - - pub fn update_connections( - &mut self, + pub fn update_connection_manager( + conn: &mut VirtualConnection, sender: &Sender>, - manager:&mut dyn SocketManager, - socket:&mut SocketWithConditioner + manager: &mut dyn SocketManager, + socket: &mut SocketWithConditioner, + time: Instant, + buffer: &mut [u8], ) { - // TODO provide real buffer, from config.max_receive_buffer - // update all connections, update connection states and send generated packets - let mut buffer = [0; 1500]; - - let time = Instant::now(); - self.connections - .iter_mut() - .for_each(|(_, conn)| match conn.state_manager.update(&mut buffer, time) { - Some(result) => match result { - Ok(event) => match event { - Either::Left(packet) => { - // TODO properly handle error, instead of assert - let packet = conn.process_outgoing(packet.packet_type, packet.payload, packet.delivery, packet.ordering, None, time) - .expect("connection manager packet should not fail"); - if let Outgoing::Packet(outgoing) = packet { - socket.send_packet_and_log(&conn.remote_address, conn.state_manager.as_mut(), &outgoing.contents(), manager, "sending packet from connection manager"); - } else { - panic!("connection manager cannot send fragmented packets"); - } - }, - Either::Right(state) => { - if let Some(_) = conn.current_state.try_change(&state) { - if let Err(err) = match &conn.current_state { - ConnectionState::Connected(data) => sender.send(ConnectionEvent(conn.remote_address, - ReceiveEvent::Connected(data.clone()))), - ConnectionState::Disconnected(closed_by) => sender.send(ConnectionEvent(conn.remote_address, - ReceiveEvent::Disconnected(DisconnectReason::ClosedBy(closed_by.clone())))), - ConnectionState::Connecting => { - conn.reset_connection(time); - Ok(()) - }, - } { - manager.track_connection_error(&conn.remote_address, &ErrorKind::SendError(SendError(Either::Right(err.0))), "sending connection state update"); + while let Some(changes) = conn.state_manager.update(buffer, time) { + match changes { + Ok(event) => match event { + Either::Left(packet) => { + // TODO properly handle error, instead of assert + let packet = conn + .process_outgoing( + packet.packet_type, + packet.payload, + packet.delivery, + packet.ordering, + None, + time, + ) + .expect("connection manager packet should not fail"); + if let Outgoing::Packet(outgoing) = packet { + socket.send_packet_and_log( + &conn.remote_address, + conn.state_manager.as_mut(), + &outgoing.contents(), + manager, + "sending packet from connection manager", + ); + } else { + panic!("connection manager cannot send fragmented packets"); + } + } + Either::Right(state) => { + if let Some(old) = conn.current_state.try_change(&state) { + if let Err(err) = match &conn.current_state { + ConnectionState::Connected(data) => sender.send(ConnectionEvent( + conn.remote_address, + ReceiveEvent::Connected(data.clone()), + )), + ConnectionState::Disconnected(closed_by) => { + sender.send(ConnectionEvent( + conn.remote_address, + ReceiveEvent::Disconnected(DisconnectReason::ClosedBy( + closed_by.clone(), + )), + )) } - } else { - panic!("Invalid state transition {:?} -> {:?}", conn.current_state, state); + _ => panic!( + "Invalid state transition: {:?} -> {:?}", + old, conn.current_state + ), + } { + manager.track_connection_error( + &conn.remote_address, + &ErrorKind::SendError(SendError(Either::Right(err.0))), + "sending connection state update", + ); } + } else { + panic!( + "Invalid state transition {:?} -> {:?}", + conn.current_state, state + ); } - }, - Err(err) => { - manager.track_connection_error(&conn.remote_address, &ErrorKind::ConnectionError(err), "recieved connection manager error"); } }, - None => {} - }); + Err(err) => { + manager.track_connection_error( + &conn.remote_address, + &ErrorKind::ConnectionError(err), + "recieved connection manager error", + ); + } + } + } + } + + pub fn update_connections( + &mut self, + sender: &Sender>, + manager: &mut dyn SocketManager, + socket: &mut SocketWithConditioner, + time: Instant, + buffer: &mut [u8], + ) { + self.connections.iter_mut().for_each(|(_, conn)| { + ActiveConnections::update_connection_manager( + conn, sender, manager, socket, time, buffer, + ) + }); } /// Returns the number of connected clients. @@ -189,15 +258,10 @@ mod tests { use super::managers::ConnectionManager; - #[derive(Debug)] - struct DummyConnManager { - } - - impl ConnectionManager for DummyConnManager { - - } + struct DummyConnManager {} + impl ConnectionManager for DummyConnManager {} const ADDRESS: &str = "127.0.0.1:12345"; diff --git a/src/net/events.rs b/src/net/events.rs index 7a7fa6ed..4dffdf4d 100644 --- a/src/net/events.rs +++ b/src/net/events.rs @@ -1,33 +1,59 @@ -use crate::packet::Packet; use crate::net::managers::ConnectionManagerError; +use crate::packet::Packet; use std::net::SocketAddr; -/// Events that can occur in `laminar` and that will be pushed through the `event_receiver` returned by `Socket::bind`. -// #[derive(Debug, PartialEq)] -// pub enum SocketEvent { -// /// A packet was received from a client. -// Packet(Packet), -// /// A new client connected. -// /// Clients are uniquely identified by the ip:port combination at this layer. -// Connect(SocketAddr), -// /// The client has been idling for a configurable amount of time. -// /// You can control the timeout in the config. -// Timeout(SocketAddr), -// } +/// Events that can occur in `laminar` for a active connection. +#[derive(Debug)] +pub enum ReceiveEvent { + /// When connection is actually created, and added to active connections list. + /// Next possible event for connection is: `Connected` or `Destroyed`. + Created, + /// When `ConnectionManager` successfully establishes connection. + /// Next possible event is: `Packet` or `Disconnected`. + Connected(Box<[u8]>), + /// When connection is in Connected state, it can actually start receiving packets. + /// Next possible event is: `Packet` or `Disconnected`. + Packet(Packet), + /// When connection, that was previously in connected state, is disconnected + /// It can either be disconnected by `ConnectionManager` in this case it is "clean" disconnect, where initiator of disconnect is also specified + /// Or it can be closed by `SocketManager` if it decides to do so + Disconnected(DisconnectReason), + /// When it is removed from active connections list. + /// Cnnection can be destroyed when disconnect is initiated by `ConnectionManager` or `SocketManager` decided to destroy it. + Destroyed(DestroyReason), +} + +/// Events that are received from user. +#[derive(Debug)] +pub enum SendEvent { + /// Initiate connect request, this will call `ConnectionManager.connect` method. + Connect(Box<[u8]>), + /// Send packet to remote host. + Packet(Packet), + /// Initiate disconnect, this will call `ConnectionManager.disconnect` method. + Disconnect, +} +/// Provides a reason why connection was destroyed. #[derive(Debug, PartialEq, Clone)] -pub enum DestroyReason { +pub enum DestroyReason { + /// When `SocketManager` decided to destroy a connection for error that arrived from `ConnectionManager`. ConnectionError(ConnectionManagerError), + /// After `Config.idle_connection_timeout` connection had no activity. Timeout, + /// If there are too many non-acked packets in flight `Config.max_packets_in_flight`. TooManyPacketsInFlight, - TooManyPacketErrors, - GracefullyDisconnected + /// When `ConnectionManager` changed to `Disconnected` state. + GracefullyDisconnected, } +/// Provides convenient enum, to specify either Local or Remote host #[derive(Debug, PartialEq, Clone)] pub enum TargetHost { + /// Local host LocalHost, - RemoteHost + /// Remote host + RemoteHost, } /// Disconnect reason, received by connection @@ -36,28 +62,9 @@ pub enum DisconnectReason { /// Disconnect was initiated by local or remote host ClosedBy(TargetHost), /// Socket manager decided to destroy connection for provided reason - Destroying(DestroyReason) + Destroying(DestroyReason), } -/// Wraps send or receive event together with remote address +/// Relate send or receive events together with address. #[derive(Debug)] -pub struct ConnectionEvent (pub SocketAddr, pub Event); - -#[derive(Debug)] -pub enum SendEvent { - Connect(Box<[u8]>), - Packet(Packet), - Disconnect, -} - -#[derive(Debug)] -pub enum ReceiveEvent { - Created, - Connected(Box<[u8]>), - Packet(Packet), - Disconnected(DisconnectReason), - Destroyed(DestroyReason), -} - -pub type ConnectionReceiveEvent = ConnectionEvent; -pub type ConnectionSendEvent = ConnectionEvent; \ No newline at end of file +pub struct ConnectionEvent(pub SocketAddr, pub Event); diff --git a/src/net/managers.rs b/src/net/managers.rs index c4544b8f..c2fd248b 100644 --- a/src/net/managers.rs +++ b/src/net/managers.rs @@ -1,6 +1,8 @@ pub use crate::either::Either; -pub use crate::net::events::{TargetHost, DestroyReason}; -pub use crate::packet::{OutgoingPacket, PacketType, DeliveryGuarantee, OrderingGuarantee}; +pub use crate::net::events::{DestroyReason, TargetHost}; +pub use crate::packet::{ + DeliveryGuarantee, GenericPacket, OrderingGuarantee, OutgoingPacket, PacketType, +}; pub use crate::ErrorKind; use std::fmt::Debug; use std::net::SocketAddr; @@ -14,15 +16,12 @@ use std::time::Instant; /// | Connecting | Connected | /// | Connecting | Disconnected | /// | Connected | Disconnected | -/// | Disconnected | Connecting | /// If these rules are not satisfied, panic! will be called. /// Each state specifies what can and cannot be done: -/// * Connecting - This is initial state when socket is created, at this moment no `events` can be sent or received, -/// in this state `ConnectionManager` is able to receive and sent `packets` to properly initiate connection. -/// * Connected - Only in this state all events will be sent or received between peers. -/// * Disconnected - in this state `ConnectionManager` is not able to send or receive any packets. -/// It can only process incoming events and decide if it can reset connection and change to Connecting state, -/// otherwise connection will be closed when all packets-in-flight finishes sending or after connection timeout. +/// * Connecting - This is initial state when socket is created, at this moment no packets can be sent or received from user, +/// in this state only `ConnectionManager` is able to receive and sent packets to properly initiate connection. +/// * Connected - Only in this state all packets will be sent or received between peers. +/// * Disconnected - in this state `ConnectionManager` is not able to send or receive any packets. Connection will be destroyed immediatelly. #[derive(Debug, PartialEq, Clone)] pub enum ConnectionState { Connecting, @@ -36,10 +35,9 @@ impl ConnectionState { match (&self, &new) { (ConnectionState::Connecting, ConnectionState::Connected(_)) | (ConnectionState::Connecting, ConnectionState::Disconnected(_)) - | (ConnectionState::Connected(_), ConnectionState::Disconnected(_)) - | (ConnectionState::Disconnected(_), ConnectionState::Connecting) => { + | (ConnectionState::Connected(_), ConnectionState::Disconnected(_)) => { Some(std::mem::replace(self, new.clone())) - }, + } _ => None, } } @@ -57,25 +55,24 @@ pub struct ConnectionManagerError(pub String); /// It abstracts pure UDP packets, and allows to implement Connected/Disconnected states. /// This table summary shows where exactly ConnectionManager sits in between different layers. -/// | Abstraction layer | Capabilities | -/// |-------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| -/// | Application | Can receive these events: Created->Connected(data)->Packet(data)->Disconnected(reason)->Destroyed(reason) Can send these events: Connect(data)->Packet->Disconnect. | -/// | ConnectionManager | Receive incoming and outgoing packets and manage connection state. Can generate new packets to initiate connection. | -/// | Laminar | Adds/Removes headers to packets, so that it could provides reliability, ordering, fragmentation, etc.. capabilities. | -/// | ConnectionManager | May change raw incoming and outgoing bytes to apply encryption, compression, etc. | +/// | Abstraction layer | Capabilities | +/// |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +/// | Application | Can receive these events: Created->Connected(data)->Packet(data)->Disconnected(reason)->Destroyed(reason). Can send these events: Connect(data)->Packet->Disconnect. | +/// | ConnectionManager | Receives all, except user packets, and can report state updates, and generate new packets via `update` method | +/// | Laminar | Adds/Removes headers to packets, so that it could provides reliability, ordering, fragmentation, etc.. capabilities. | +/// | ConnectionManager | May change raw incoming and outgoing bytes to apply encryption, compression, etc. | /// -/// Manager can store local buffer size of `Config.receive_buffer_max_size` if it does some kind of encryption. -/// It tries to maintain valid connection state, and it can't decide when to destroy or disconnect a connection itself. -/// Only when packet is recevied, or action is initiated by user it is allowed to change connection state. +/// It tries to maintain valid connection state, and it can't decide when to destroy itself, only when it changes to disconnected, it will be destroyed later. /// From the point of view of connection manager, laminar's header + payload is interpreted as user data. /// Distinction between user packet and protocol specific packet is encoded in laminar's packet header. +/// Preprocess/Postprocess and Update methods always accept temporary buffer of size `Config.receive_buffer_max_size` that can be used as output. pub trait ConnectionManager: Debug + Send { + /// When instance of connection manager is created, `update` method will be called, before any other method. /// This function should be called frequently, even if there is no packets to send or receive. - /// It will always be called last, after all other methods is called, so it could send packets or comunicate errors if required. - /// It cannot change connection state explicitly, instead it can emit errors, and SocketManager will decide when to destroy connection. + /// It will always be called last, after all other methods is called, in the main laminar`s loop. /// It can generate all kinds of packets: heartbeat, user or connection protocol packets. /// (maybe heartbeat functionality should be moved here?) - /// `buffer` is used to return packet payload, it's size is `Config.receive_buffer_max_size`. + /// It will be called in the loop as long, as it returns any results. E.g. `connect` method may generate multiple results: change state and send packet. fn update<'a>( &mut self, buffer: &'a mut [u8], @@ -93,7 +90,7 @@ pub trait ConnectionManager: Debug + Send { where 'a: 'b; - /// This will be called for all outgoing data, including packets that are resend. + /// This will be called for all outgoing data, including packets that are resend. /// Dropped packets will also go through here. /// Accepts full packet: header + payload fn postprocess_outgoing<'a, 'b>(&mut self, data: &'a [u8], buffer: &'b mut [u8]) -> &'b [u8] @@ -101,33 +98,34 @@ pub trait ConnectionManager: Debug + Send { 'a: 'b; /// This will be called only for incoming protocol specific packets, after laminar's reliability layer accepted it. - /// This is the only place where connection state can actually be changed by incomming packet. - fn process_protocol_data<'a>( - &mut self, - data: &'a [u8], - ) -> Result<(), ConnectionManagerError>; + /// This is convenient place to process actual logic, because it is filtered by laminar's reliability layer and it accepts only `PacketType::ConnectionManager` messages. + fn process_protocol_data(&mut self, data: &[u8]) -> Result<(), ConnectionManagerError>; /// This will be invoked when player sends connect request, - /// Some protocols might provide a way to pass initial connection data - /// Data is user payload, that will be received by remote host, on Connected(data) event - /// This method is not able to send packet immediatelly, instead this functionality should be handled in `update` method. + /// Some protocols might provide a way to pass initial connection data, hence the `data` field. + /// This method can only be called when connection is in `Connecting` state fn connect(&mut self, data: Box<[u8]>); - // This will be invoked when player sends disconnect request, - /// This method is not able to send packet immediatelly, instead this functionality should be handled in `update` method. - fn disconnect<'a>(&mut self); + /// This will be invoked when player sends SendEvent::Disconnect request. + fn disconnect(&mut self); } /// Tracks all sorts of global statistics, and decided whether to create `ConnectionManager` for new connections or not. /// Also decides when connections should be destroyed even if they are in connected state. pub trait SocketManager: Debug + Send { - /// Decide if it is possible to accept/create new connection connection - fn accept_new_connection(&mut self, addr: &SocketAddr, requested_by: TargetHost) -> Option>; + /// Decide if it is possible to accept/create new remote connection, this is invoked when message from unknown address arives. + fn accept_remote_connection( + &mut self, + addr: &SocketAddr, + raw_bytes: &[u8], + ) -> Option>; + /// Decide if it is possible to accept/create new local connection, this is invoked when any user event is recieved for unknown address. + fn accept_local_connection(&mut self, addr: &SocketAddr) -> Option>; - /// Returns list of connections that socket manager decided to destroy + /// Returns list of connections that socket manager decided to destroy, along with destroy reason fn collect_connections_to_destroy(&mut self) -> Option>; - // all sorts of statistics might be useful here to help deciding whether new connection can be created or not + /// All sorts of statistics might be useful here to help deciding whether new connection can be created or not fn track_connection_error(&mut self, addr: &SocketAddr, error: &ErrorKind, error_context: &str); fn track_global_error(&mut self, error: &ErrorKind, error_context: &str); fn track_sent_bytes(&mut self, addr: &SocketAddr, bytes: usize); @@ -135,14 +133,3 @@ pub trait SocketManager: Debug + Send { fn track_ignored_bytes(&mut self, addr: &SocketAddr, bytes: usize); fn track_connection_destroyed(&mut self, addr: &SocketAddr); } - - -pub struct GenericPacket<'a> { - pub packet_type: PacketType, - /// the raw payload of the packet - pub payload: &'a [u8], - /// defines on how the packet will be delivered. - pub delivery: DeliveryGuarantee, - /// defines on how the packet will be ordered. - pub ordering: OrderingGuarantee, -} \ No newline at end of file diff --git a/src/net/socket.rs b/src/net/socket.rs index 65237b3a..7fc7198f 100644 --- a/src/net/socket.rs +++ b/src/net/socket.rs @@ -2,9 +2,9 @@ use crate::{ config::Config, either::Either, error::{ErrorKind, Result}, + net::events::{ConnectionEvent, DestroyReason, ReceiveEvent, SendEvent}, + net::managers::{ConnectionManager, ConnectionState, SocketManager}, net::{connection::ActiveConnections, link_conditioner::LinkConditioner}, - net::events::{ConnectionEvent, SendEvent, ReceiveEvent, DestroyReason, TargetHost}, - net::managers::{SocketManager, ConnectionManager}, packet::{DeliveryGuarantee, Outgoing, Packet, PacketType}, }; use crossbeam_channel::{self, unbounded, Receiver, SendError, Sender, TryRecvError}; @@ -19,38 +19,49 @@ use std::{ // just wrap whese too together, for easier passing around #[derive(Debug)] pub struct SocketWithConditioner { - buffer: Vec, + buffer: Vec, // this buffer is used for postprocess_outgoing socket: UdpSocket, link_conditioner: Option, } impl SocketWithConditioner { - /// Send a single packet over the UDP socket. + /// Send a single packet over the UDP socket. /// Postprocess it using Connectionmanager`s postprocess_outgoing method, /// and use link condition if exists, to simulate network conditions - pub fn send_packet(&mut self, addr:&SocketAddr, manager:&mut dyn ConnectionManager, payload:&[u8]) -> Result { + pub fn send_packet( + &mut self, + addr: &SocketAddr, + manager: &mut dyn ConnectionManager, + payload: &[u8], + ) -> Result { let payload = manager.postprocess_outgoing(payload, self.buffer.as_mut_slice()); if let Some(ref mut link) = self.link_conditioner { if !link.should_send() { - return Ok(0) + return Ok(0); } - } + } Ok(self.socket.send_to(payload, addr)?) } - pub fn send_packet_and_log(&mut self, addr:&SocketAddr, conn_man:&mut dyn ConnectionManager, payload:&[u8], sock_man:&mut dyn SocketManager, error_context:&str) -> usize { + pub fn send_packet_and_log( + &mut self, + addr: &SocketAddr, + conn_man: &mut dyn ConnectionManager, + payload: &[u8], + sock_man: &mut dyn SocketManager, + error_context: &str, + ) -> usize { match self.send_packet(addr, conn_man, payload) { Ok(bytes) => { sock_man.track_sent_bytes(addr, bytes); bytes - }, + } Err(ref err) => { sock_man.track_connection_error(addr, err, error_context); 0 } } } - } /// Wraps crossbeam_channel sender with convenient functions: `connect`, `send`, `disconnect` @@ -59,38 +70,49 @@ pub struct SocketEventSender(pub Sender>); impl SocketEventSender { /// Constructs ConnectionEvent Packet event from provided packet pub fn send(&self, packet: Packet) -> std::result::Result<(), SendError> { - self.0.send(ConnectionEvent(packet.addr(), SendEvent::Packet(packet))) - .map_err(|err| SendError(match (err.0).1 { - SendEvent::Packet(data) => data, - _ => unreachable!() - })) + self.0 + .send(ConnectionEvent(packet.addr(), SendEvent::Packet(packet))) + .map_err(|err| { + SendError(match (err.0).1 { + SendEvent::Packet(data) => data, + _ => unreachable!(), + }) + }) } /// Constructs ConnectionEvent Connect event from provided payload - pub fn connect(&self, addr: SocketAddr, payload: Box<[u8]>) -> std::result::Result<(), SendError>> { - self.0.send(ConnectionEvent(addr, SendEvent::Connect(payload))) - .map_err(|err| SendError(match (err.0).1 { - SendEvent::Connect(data) => data, - _ => unreachable!() - })) + pub fn connect( + &self, + addr: SocketAddr, + payload: Box<[u8]>, + ) -> std::result::Result<(), SendError>> { + self.0 + .send(ConnectionEvent(addr, SendEvent::Connect(payload))) + .map_err(|err| { + SendError(match (err.0).1 { + SendEvent::Connect(data) => data, + _ => unreachable!(), + }) + }) } /// Constructs ConnectionEvent Disconnect event pub fn disconnect(&self, addr: SocketAddr) -> std::result::Result<(), SendError<()>> { - self.0.send(ConnectionEvent(addr, SendEvent::Disconnect)) + self.0 + .send(ConnectionEvent(addr, SendEvent::Disconnect)) .map_err(|_| SendError(())) } } - /// A reliable UDP socket implementation with configurable reliability and ordering guarantees. #[derive(Debug)] pub struct Socket { - socket: SocketWithConditioner, + socket: SocketWithConditioner, config: Config, connections: ActiveConnections, recv_buffer: Vec, - + tmp_buffer: Vec, // this is temporary buffer, mostly used by connection manager, in cases where it needs to modify incomming/outgoing raw bytes, or create new packets for sending + event_sender: Sender>, packet_receiver: Receiver>, @@ -130,17 +152,26 @@ impl Socket { /// endpoint by looking to see if they are still sending packets or not /// /// This function allows you to configure laminar with the passed configuration. - pub fn bind_with_config(addresses: A, config: Config, manager: Box) -> Result { + pub fn bind_with_config( + addresses: A, + config: Config, + manager: Box, + ) -> Result { let socket = UdpSocket::bind(addresses)?; Self::bind_internal(socket, config, manager) } - fn bind_internal(socket: UdpSocket, config: Config, manager: Box) -> Result { + fn bind_internal( + socket: UdpSocket, + config: Config, + manager: Box, + ) -> Result { socket.set_nonblocking(!config.blocking_mode)?; let (event_sender, event_receiver) = unbounded(); let (packet_sender, packet_receiver) = unbounded(); - Ok(Socket { + Ok(Socket { recv_buffer: vec![0; config.receive_buffer_max_size], + tmp_buffer: vec![0; config.receive_buffer_max_size], socket: SocketWithConditioner { buffer: Vec::with_capacity(config.receive_buffer_max_size), socket, @@ -153,7 +184,7 @@ impl Socket { sender: packet_sender, receiver: event_receiver, - + manager, }) } @@ -215,24 +246,39 @@ impl Socket { match self.recv_from(time) { Ok(UdpSocketState::MaybeMore) => continue, Ok(UdpSocketState::MaybeEmpty) => break, - Err(ref e) => self.manager.track_global_error(e, "receiving data from socket"), + Err(ref e) => self + .manager + .track_global_error(e, "receiving data from socket"), } } // Now grab all the packets waiting to be sent and send them while let Ok(p) = self.packet_receiver.try_recv() { if let Err(ref e) = self.send_to(p, time) { - self.manager.track_global_error(e, "sending data from socket") + self.manager + .track_global_error(e, "sending data from socket") } } - self.connections.update_connections(&self.event_sender, self.manager.as_mut(), &mut self.socket); + self.connections.update_connections( + &self.event_sender, + self.manager.as_mut(), + &mut self.socket, + time, + self.tmp_buffer.as_mut(), + ); // get connections that socket manager decided to destroy if let Some(list) = self.manager.collect_connections_to_destroy() { for (addr, reason) in list { - self.connections.remove_connection(&addr, &self.event_sender, self.manager.as_mut(), reason, "destroyed by socket manager"); - } + self.connections.remove_connection( + &addr, + &self.event_sender, + self.manager.as_mut(), + reason, + "destroyed by socket manager", + ); + } } // Check for idle clients @@ -245,7 +291,12 @@ impl Socket { if let Some(heartbeat_interval) = self.config.heartbeat_interval { // Iterate over all connections which have not sent a packet for a duration of at least // `heartbeat_interval` (from config), and send a heartbeat packet to each. - self.connections.heartbeat_required_connections(heartbeat_interval, time, self.manager.as_mut(), &mut self.socket); + self.connections.heartbeat_required_connections( + heartbeat_interval, + time, + self.manager.as_mut(), + &mut self.socket, + ); } } @@ -264,7 +315,13 @@ impl Socket { fn handle_dead_clients(&mut self) { let dead_addresses = self.connections.dead_connections(); for (address, reason) in dead_addresses { - self.connections.remove_connection(&address, &self.event_sender, self.manager.as_mut(), reason, "removing dead clients"); + self.connections.remove_connection( + &address, + &self.event_sender, + self.manager.as_mut(), + reason, + "removing dead clients", + ); } } @@ -276,73 +333,103 @@ impl Socket { .connections .idle_connections(self.config.idle_connection_timeout, time); for address in idle_addresses { - self.connections.remove_connection(&address, &self.event_sender, self.manager.as_mut(), DestroyReason::Timeout, "removing idle clients"); + self.connections.remove_connection( + &address, + &self.event_sender, + self.manager.as_mut(), + DestroyReason::Timeout, + "removing idle clients", + ); } } // Serializes and sends a `Packet` on the socket. On success, returns the number of bytes written. - fn send_to(&mut self, event: ConnectionEvent, time: Instant) -> Result { + fn send_to(&mut self, event: ConnectionEvent, time: Instant) -> Result { let mut bytes_sent = 0; - match (self.connections.try_get(&event.0), event.1) { - (conn, SendEvent::Connect(data)) => { - let connection = if let Some(connection) = conn { - Some(connection) - } else if let Some(conn_manager) = self.manager.accept_new_connection(&event.0, TargetHost::LocalHost) { - self.event_sender.send(ConnectionEvent(event.0, ReceiveEvent::Created))?; - Some(self.connections.get_or_insert_connection(event.0, &self.config, time, conn_manager)) - } else { - None - }; - if let Some(conn) = connection { - conn.state_manager.connect(data); - } - }, - (Some(connection), SendEvent::Packet(packet)) => { - // TODO maybe these should not depend on send_to method? - let dropped = connection.gather_dropped_packets(); - let mut processed_packets: Vec = dropped - .iter() - .flat_map(|waiting_packet| { - connection.process_outgoing( - PacketType::Packet, - &waiting_packet.payload, - // Because a delivery guarantee is only sent with reliable packets - DeliveryGuarantee::Reliable, - // This is stored with the dropped packet because they could be mixed - waiting_packet.ordering_guarantee, - waiting_packet.item_identifier, - time, - ) - }) - .collect(); - - let processed_packet = connection.process_outgoing( - PacketType::Packet, - packet.payload(), - packet.delivery_guarantee(), - packet.order_guarantee(), - None, - time, - )?; - processed_packets.push(processed_packet); - - for processed_packet in processed_packets { + let connection = if let Some(connection) = self.connections.try_get(&event.0) { + Some(connection) + } else if let Some(conn_manager) = self.manager.accept_local_connection(&event.0) { + self.event_sender + .send(ConnectionEvent(event.0, ReceiveEvent::Created))?; + let conn = self.connections.get_or_insert_connection( + event.0, + &self.config, + time, + conn_manager, + ); + ActiveConnections::update_connection_manager( + conn, + &self.event_sender, + self.manager.as_mut(), + &mut self.socket, + time, + self.tmp_buffer.as_mut(), + ); + Some(conn) + } else { + None + }; + if let Some(connection) = connection { + match (event.1, &connection.current_state) { + (SendEvent::Packet(packet), ConnectionState::Connected(_)) => { + // TODO maybe these should not depend on send_to method? + let dropped = connection.gather_dropped_packets(); + let mut processed_packets: Vec = dropped + .iter() + .flat_map(|waiting_packet| { + connection.process_outgoing( + PacketType::Packet, + &waiting_packet.payload, + // Because a delivery guarantee is only sent with reliable packets + DeliveryGuarantee::Reliable, + // This is stored with the dropped packet because they could be mixed + waiting_packet.ordering_guarantee, + waiting_packet.item_identifier, + time, + ) + }) + .collect(); + + let processed_packet = connection.process_outgoing( + PacketType::Packet, + packet.payload(), + packet.delivery_guarantee(), + packet.order_guarantee(), + None, + time, + )?; + processed_packets.push(processed_packet); + + for processed_packet in processed_packets { match processed_packet { Outgoing::Packet(outgoing) => { - bytes_sent += self.socket.send_packet(&event.0, connection.state_manager.as_mut(), &outgoing.contents())?; + bytes_sent += self.socket.send_packet( + &event.0, + connection.state_manager.as_mut(), + &outgoing.contents(), + )?; } Outgoing::Fragments(packets) => { for outgoing in packets { - bytes_sent += self.socket.send_packet(&event.0, connection.state_manager.as_mut(), &outgoing.contents())?; + bytes_sent += self.socket.send_packet( + &event.0, + connection.state_manager.as_mut(), + &outgoing.contents(), + )?; } } } + } } - }, - (Some(connection), SendEvent::Disconnect) => connection.state_manager.disconnect(), - _ => {} // ignore packet and disconnect event for non existent connection - }; + (SendEvent::Connect(data), ConnectionState::Connecting) => { + connection.state_manager.connect(data) + } + (SendEvent::Disconnect, _) => connection.state_manager.disconnect(), + _ => {} // ignore all other combinations + }; + } + Ok(bytes_sent) } @@ -351,24 +438,44 @@ impl Socket { match self.socket.socket.recv_from(&mut self.recv_buffer) { Ok((recv_len, address)) => { if recv_len == 0 { - return Err(ErrorKind::ReceivedDataToShort)?; + return Err(ErrorKind::ReceivedDataToShort); } let received_payload = &self.recv_buffer[..recv_len]; let connection = if let Some(conn) = self.connections.try_get(&address) { Some(conn) + } else if let Some(manager) = self + .manager + .accept_remote_connection(&address, received_payload) + { + self.event_sender + .send(ConnectionEvent(address, ReceiveEvent::Created))?; + let conn = self.connections.get_or_insert_connection( + address, + &self.config, + time, + manager, + ); + ActiveConnections::update_connection_manager( + conn, + &self.event_sender, + self.manager.as_mut(), + &mut self.socket, + time, + self.tmp_buffer.as_mut(), + ); + Some(conn) } else { - if let Some(manager) = self.manager.accept_new_connection(&address, TargetHost::RemoteHost) { - self.event_sender.send(ConnectionEvent(address, ReceiveEvent::Created))?; - Some(self.connections.get_or_insert_connection(address, &self.config, time, manager)) - } else { - None - } + None }; - if let Some(conn) = connection - { - conn.process_incoming(received_payload, &self.event_sender, self.manager.as_mut(), time)?; + if let Some(conn) = connection { + conn.process_incoming( + received_payload, + &self.event_sender, + self.manager.as_mut(), + time, + )?; } } Err(e) => { @@ -388,7 +495,6 @@ impl Socket { } } - #[cfg(test)] fn connection_count(&self) -> usize { self.connections.count() diff --git a/src/net/virtual_connection.rs b/src/net/virtual_connection.rs index 9809ae7d..cfa61710 100644 --- a/src/net/virtual_connection.rs +++ b/src/net/virtual_connection.rs @@ -7,14 +7,14 @@ use crate::{ }, net::constants::{ ACKED_PACKET_HEADER, DEFAULT_ORDERING_STREAM, DEFAULT_SEQUENCING_STREAM, - STANDARD_HEADER_SIZE, + STANDARD_HEADER_SIZE, }, + net::events::{ConnectionEvent, ReceiveEvent}, net::managers::{ConnectionManager, ConnectionState, SocketManager}, packet::{ DeliveryGuarantee, OrderingGuarantee, Outgoing, OutgoingPacket, OutgoingPacketBuilder, Packet, PacketReader, PacketType, SequenceNumber, }, - net::events::{ConnectionEvent, ReceiveEvent, DisconnectReason} }; use crossbeam_channel::{self, Sender}; @@ -45,7 +45,12 @@ pub struct VirtualConnection { impl VirtualConnection { /// Creates and returns a new Connection that wraps the provided socket address - pub fn new(addr: SocketAddr, config: &Config, time: Instant, state_manager: Box) -> VirtualConnection { + pub fn new( + addr: SocketAddr, + config: &Config, + time: Instant, + state_manager: Box, + ) -> VirtualConnection { VirtualConnection { last_heard: time, last_sent: time, @@ -57,7 +62,7 @@ impl VirtualConnection { fragmentation: Fragmentation::new(config), config: config.to_owned(), state_manager, - current_state: ConnectionState::Connecting + current_state: ConnectionState::Connecting, } } @@ -66,9 +71,9 @@ impl VirtualConnection { self.acknowledge_handler.packets_in_flight() > self.config.max_packets_in_flight } - pub fn can_gracefully_disconnect(&self) -> bool { + pub fn is_disconnected(&self) -> bool { if let ConnectionState::Disconnected(_) = self.current_state { - self.acknowledge_handler.packets_in_flight() == 0 + true } else { false } @@ -110,6 +115,8 @@ impl VirtualConnection { /// This will pre-process the given buffer to be sent over the network. pub fn process_outgoing<'a>( &mut self, + // PacketType is used by connection manager, because it can send all sorts of packets, except fragmented. + // TODO would be nice to change this, so that it is impossible to provide bad value packet_type: PacketType, payload: &'a [u8], delivery_guarantee: DeliveryGuarantee, @@ -250,7 +257,6 @@ impl VirtualConnection { } } - pub fn process_incoming( &mut self, received_data: &[u8], @@ -259,9 +265,11 @@ impl VirtualConnection { time: Instant, ) -> crate::Result<()> { // TODO pass buffer from somewhere else - let mut buffer = [0;1500]; - let received_data = self.state_manager.preprocess_incoming(received_data, &mut buffer)?; - + let mut buffer = [0; 1500]; + let received_data = self + .state_manager + .preprocess_incoming(received_data, &mut buffer)?; + self.last_heard = time; let mut packet_reader = PacketReader::new(received_data); @@ -270,7 +278,7 @@ impl VirtualConnection { if !header.is_current_protocol() { return Err(ErrorKind::ProtocolVersionMismatch); - } + } if header.is_heartbeat() { // Heartbeat packets are unreliable, unordered and empty packets. @@ -295,9 +303,10 @@ impl VirtualConnection { header.is_connection_manager_packet(), sender, socket_manager, - packet, + packet, header.delivery_guarantee(), - OrderingGuarantee::Sequenced(Some(arranging_header.stream_id()))); + OrderingGuarantee::Sequenced(Some(arranging_header.stream_id())), + ); } return Ok(()); @@ -306,9 +315,10 @@ impl VirtualConnection { header.is_connection_manager_packet(), sender, socket_manager, - packet_reader.read_payload(), + packet_reader.read_payload(), header.delivery_guarantee(), - header.ordering_guarantee()); + header.ordering_guarantee(), + ); } DeliveryGuarantee::Reliable => { if header.is_fragment() { @@ -376,12 +386,11 @@ impl VirtualConnection { let payload = packet_reader.read_payload(); - println!("======================process_incoming ordered: {:?} {}", header, String::from_utf8_lossy(payload.as_ref())); - let stream = self .ordering_system .get_or_create_stream(arranging_header.stream_id()); - let arranged_packet = stream.arrange(arranging_header.arranging_id(), payload); + let arranged_packet = + stream.arrange(arranging_header.arranging_id(), payload); let packets = arranged_packet .into_iter() .chain(stream.iter_mut()) @@ -391,9 +400,10 @@ impl VirtualConnection { header.is_connection_manager_packet(), sender, socket_manager, - packet, - header.delivery_guarantee(), - OrderingGuarantee::Ordered(Some(arranging_header.stream_id()))); + packet, + header.delivery_guarantee(), + OrderingGuarantee::Ordered(Some(arranging_header.stream_id())), + ); } } else { let payload = packet_reader.read_payload(); @@ -424,57 +434,55 @@ impl VirtualConnection { /// Pass packet to ConnectionManager, check for state changes, and return any newly generated packets fn process_packet( - &mut self, + &mut self, is_connection_manager_packet: bool, sender: &Sender>, - payload: Box<[u8]>, - delivery: DeliveryGuarantee, - ordering: OrderingGuarantee + payload: Box<[u8]>, + delivery: DeliveryGuarantee, + ordering: OrderingGuarantee, ) -> Result<()> { if !is_connection_manager_packet { if let ConnectionState::Connected(_) = self.current_state { - sender.send(ConnectionEvent(self.remote_address, ReceiveEvent::Packet(Packet::new(self.remote_address, payload, delivery, ordering))))?; + sender.send(ConnectionEvent( + self.remote_address, + ReceiveEvent::Packet(Packet::new( + self.remote_address, + payload, + delivery, + ordering, + )), + ))?; } } else { - // when we're in disconnected state, and receive empty message, - // this means that remote host got our disconnect message - if let ConnectionState::Disconnected(_) = self.current_state { - if payload.as_ref().len() == 0 { - self.acknowledge_handler = AcknowledgmentHandler::new(); - return Ok(()); - } - } self.state_manager.process_protocol_data(payload.as_ref())?; } Ok(()) } fn process_packet_and_log_errors( - &mut self, + &mut self, is_connection_manager_packet: bool, sender: &Sender>, socket_manager: &mut dyn SocketManager, - payload: Box<[u8]>, - delivery: DeliveryGuarantee, - ordering: OrderingGuarantee - ) { - if let Err(ref error) = self.process_packet(is_connection_manager_packet, sender, payload, delivery, ordering) { - socket_manager.track_connection_error(&self.remote_address, error, "processing incomming packet"); + payload: Box<[u8]>, + delivery: DeliveryGuarantee, + ordering: OrderingGuarantee, + ) { + if let Err(ref error) = self.process_packet( + is_connection_manager_packet, + sender, + payload, + delivery, + ordering, + ) { + socket_manager.track_connection_error( + &self.remote_address, + error, + "processing incomming packet", + ); } } - /// Fully reset connection to initial state, after ConnectionManager switches from Disconnected to Connecting - pub fn reset_connection(&mut self, time: Instant) { - self.last_heard = time; - self.last_sent= time; - self.ordering_system = OrderingSystem::new(); - self.sequencing_system = SequencingSystem::new(); - self.acknowledge_handler = AcknowledgmentHandler::new(); - self.congestion_handler = CongestionHandler::new(&self.config); - self.fragmentation = Fragmentation::new(&self.config); - self.current_state = ConnectionState::Connecting; - } - /// This will gather dropped packets from the acknowledgment handler. /// /// Note that after requesting dropped packets the dropped packets will be removed from this client. diff --git a/src/packet.rs b/src/packet.rs index 9c5ad403..0f3fc185 100644 --- a/src/packet.rs +++ b/src/packet.rs @@ -10,7 +10,7 @@ mod packet_structure; pub use self::enums::{DeliveryGuarantee, OrderingGuarantee, PacketType}; pub use self::outgoing::{Outgoing, OutgoingPacket, OutgoingPacketBuilder}; pub use self::packet_reader::PacketReader; -pub use self::packet_structure::Packet; +pub use self::packet_structure::{GenericPacket, Packet}; pub type SequenceNumber = u16; diff --git a/src/packet/enums.rs b/src/packet/enums.rs index d579f2e9..e6f46f68 100644 --- a/src/packet/enums.rs +++ b/src/packet/enums.rs @@ -91,7 +91,7 @@ pub enum PacketType { /// Heartbeat packet Heartbeat = 2, /// ConnectionManager specific header - ConnectionManager = 3 + ConnectionManager = 3, } impl EnumConverter for PacketType { diff --git a/src/packet/packet_structure.rs b/src/packet/packet_structure.rs index 9dee0ba8..e96a5288 100644 --- a/src/packet/packet_structure.rs +++ b/src/packet/packet_structure.rs @@ -1,4 +1,4 @@ -use crate::packet::{DeliveryGuarantee, OrderingGuarantee}; +use crate::packet::{DeliveryGuarantee, OrderingGuarantee, PacketType}; use std::net::SocketAddr; #[derive(Clone, PartialEq, Eq, Debug)] @@ -176,6 +176,58 @@ impl Packet { } } +/// This packet type have similar properties to `Packet` except that it doesn't own anything, and additionally has `PacketType`. +#[derive(Debug)] +pub struct GenericPacket<'a> { + /// defines packet type, currently Fragment type should not set, + pub(crate) packet_type: PacketType, + /// the raw payload of the packet + pub(crate) payload: &'a [u8], + /// defines on how the packet will be delivered. + pub(crate) delivery: DeliveryGuarantee, + /// defines on how the packet will be ordered. + pub(crate) ordering: OrderingGuarantee, +} + +impl<'a> GenericPacket<'a> { + pub fn manager_packet( + payload: &'a [u8], + delivery: DeliveryGuarantee, + ordering: OrderingGuarantee, + ) -> Self { + Self { + packet_type: PacketType::ConnectionManager, + payload, + delivery, + ordering, + } + } + pub fn user_packet( + payload: &'a [u8], + delivery: DeliveryGuarantee, + ordering: OrderingGuarantee, + ) -> Self { + Self { + packet_type: PacketType::ConnectionManager, + payload, + delivery, + ordering, + } + } + pub fn heartbeat_packet( + payload: &'a [u8], + delivery: DeliveryGuarantee, + ordering: OrderingGuarantee, + ) -> Self { + Self { + packet_type: PacketType::ConnectionManager, + payload, + delivery, + ordering, + } + } +} + #[cfg(test)] mod tests { use crate::packet::{DeliveryGuarantee, OrderingGuarantee, Packet}; From b646bbb4cf8b966a6eb687f5769549702ccb97d1 Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Wed, 18 Sep 2019 19:39:37 +0300 Subject: [PATCH 11/14] Update src/net/events.rs Co-Authored-By: Timon --- src/net/events.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/events.rs b/src/net/events.rs index 4dffdf4d..fcb62be1 100644 --- a/src/net/events.rs +++ b/src/net/events.rs @@ -18,7 +18,7 @@ pub enum ReceiveEvent { /// It can either be disconnected by `ConnectionManager` in this case it is "clean" disconnect, where initiator of disconnect is also specified /// Or it can be closed by `SocketManager` if it decides to do so Disconnected(DisconnectReason), - /// When it is removed from active connections list. + /// When it is removed from the active connections list. /// Cnnection can be destroyed when disconnect is initiated by `ConnectionManager` or `SocketManager` decided to destroy it. Destroyed(DestroyReason), } From bac057f4a459089280fad8f5e99549648b45a557 Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Wed, 18 Sep 2019 19:48:28 +0300 Subject: [PATCH 12/14] Apply suggestions from code review Mostly grammar fixes Co-Authored-By: Timon --- src/error.rs | 4 ++-- src/managers/simple.rs | 6 +++--- src/net/events.rs | 22 +++++++++++----------- src/net/managers.rs | 34 +++++++++++++++++----------------- src/net/virtual_connection.rs | 2 +- src/packet/packet_structure.rs | 2 +- 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/error.rs b/src/error.rs index 4d47a8e8..6954c225 100644 --- a/src/error.rs +++ b/src/error.rs @@ -32,7 +32,7 @@ pub enum ErrorKind { SendError(SendError, ConnectionEvent>>), /// Expected header but could not be read from buffer. CouldNotReadHeader(String), - /// Errors that is returned from ConnectionManager either preprocessing data or processing packet + /// Errors that is returned from `ConnectionManager` either preprocessing data or processing packet ConnectionError(ConnectionManagerError), } @@ -73,7 +73,7 @@ impl Display for ErrorKind { ), ErrorKind::ConnectionError(err) => write!( fmt, - "Something when wrong in ConnectionManager. Reason: {:?}.", + "Something went wrong in ConnectionManager. Reason: {:?}.", err ), } diff --git a/src/managers/simple.rs b/src/managers/simple.rs index 1fe97c2e..bcd91b02 100644 --- a/src/managers/simple.rs +++ b/src/managers/simple.rs @@ -7,7 +7,7 @@ use std::io::ErrorKind::WouldBlock; use std::net::SocketAddr; use std::time::Instant; -/// The simplest connection manager, that immediately goes into connected state, after creating it +/// The simplest connection manager, that immediately goes into the connected state, after creating it #[derive(Debug, Default)] struct AlwaysConnectedConManager { connected: bool, @@ -55,7 +55,7 @@ impl ConnectionManager for AlwaysConnectedConManager { fn disconnect(&mut self) {} } -/// Simple connection manager, that actually tries to connect by exchanging "connect", "connected", and "disconnect" messages with remote host, +/// Simple connection manager, that actually tries to connect by exchanging 'connect', 'connected', and 'disconnect' messages with the remote host, #[derive(Debug, Default)] struct SimpleConnectionManager { state: ConnectionState, @@ -154,7 +154,7 @@ impl ConnectionManager for SimpleConnectionManager { } } -/// Simplest implementation of socket manager, always accept connection and never destroy, no matter how many errors connection reports +/// Simplest implementation of socket manager, always accept a connection and never destroy, no matter how many errors connection reports /// It can create two types of connection managers: /// * true - creates `AlwaysConnectedConManager` /// * false - creates `SimpleConnectionManager` diff --git a/src/net/events.rs b/src/net/events.rs index fcb62be1..be896c8a 100644 --- a/src/net/events.rs +++ b/src/net/events.rs @@ -5,7 +5,7 @@ use std::net::SocketAddr; /// Events that can occur in `laminar` for a active connection. #[derive(Debug)] pub enum ReceiveEvent { - /// When connection is actually created, and added to active connections list. + /// When the connection is actually created and added to the active connections list. /// Next possible event for connection is: `Connected` or `Destroyed`. Created, /// When `ConnectionManager` successfully establishes connection. @@ -14,27 +14,27 @@ pub enum ReceiveEvent { /// When connection is in Connected state, it can actually start receiving packets. /// Next possible event is: `Packet` or `Disconnected`. Packet(Packet), - /// When connection, that was previously in connected state, is disconnected - /// It can either be disconnected by `ConnectionManager` in this case it is "clean" disconnect, where initiator of disconnect is also specified - /// Or it can be closed by `SocketManager` if it decides to do so + /// When connection, that was previously in a connected state, is disconnected + /// it can either be disconnected by `ConnectionManager` in this case it is a 'clean' disconnect, where the initiator of disconnect is also specified + /// or it can be closed by `SocketManager` if it decides to do so Disconnected(DisconnectReason), /// When it is removed from the active connections list. - /// Cnnection can be destroyed when disconnect is initiated by `ConnectionManager` or `SocketManager` decided to destroy it. + /// Connection can be destroyed when the disconnect is initiated by `ConnectionManager`, or `SocketManager` decided to destroy it. Destroyed(DestroyReason), } -/// Events that are received from user. +/// Events that are received from the user. #[derive(Debug)] pub enum SendEvent { /// Initiate connect request, this will call `ConnectionManager.connect` method. Connect(Box<[u8]>), - /// Send packet to remote host. + /// Send packet to the remote host. Packet(Packet), /// Initiate disconnect, this will call `ConnectionManager.disconnect` method. Disconnect, } -/// Provides a reason why connection was destroyed. +// Provides a reason why the connection was destroyed. #[derive(Debug, PartialEq, Clone)] pub enum DestroyReason { /// When `SocketManager` decided to destroy a connection for error that arrived from `ConnectionManager`. @@ -50,16 +50,16 @@ pub enum DestroyReason { /// Provides convenient enum, to specify either Local or Remote host #[derive(Debug, PartialEq, Clone)] pub enum TargetHost { - /// Local host + /// Represents the localhost LocalHost, - /// Remote host + /// Represents the remote host RemoteHost, } /// Disconnect reason, received by connection #[derive(Debug, PartialEq)] pub enum DisconnectReason { - /// Disconnect was initiated by local or remote host + /// Disconnect was initiated by the local or remote host ClosedBy(TargetHost), /// Socket manager decided to destroy connection for provided reason Destroying(DestroyReason), diff --git a/src/net/managers.rs b/src/net/managers.rs index c2fd248b..1c9a007f 100644 --- a/src/net/managers.rs +++ b/src/net/managers.rs @@ -30,7 +30,7 @@ pub enum ConnectionState { } impl ConnectionState { - /// Tries to change current state, returns old state if successfully changed. + /// Tries to change current state and returns old state if successfully changed. pub fn try_change(&mut self, new: &Self) -> Option { match (&self, &new) { (ConnectionState::Connecting, ConnectionState::Connected(_)) @@ -62,14 +62,14 @@ pub struct ConnectionManagerError(pub String); /// | Laminar | Adds/Removes headers to packets, so that it could provides reliability, ordering, fragmentation, etc.. capabilities. | /// | ConnectionManager | May change raw incoming and outgoing bytes to apply encryption, compression, etc. | /// -/// It tries to maintain valid connection state, and it can't decide when to destroy itself, only when it changes to disconnected, it will be destroyed later. +/// It tries to maintain a valid connection state, and it can't decide when to destroy itself, only when it changes to disconnected, it will be destroyed later. /// From the point of view of connection manager, laminar's header + payload is interpreted as user data. -/// Distinction between user packet and protocol specific packet is encoded in laminar's packet header. +/// Distinction between user packet and the protocol-specific packet is encoded in laminar's packet header. /// Preprocess/Postprocess and Update methods always accept temporary buffer of size `Config.receive_buffer_max_size` that can be used as output. pub trait ConnectionManager: Debug + Send { - /// When instance of connection manager is created, `update` method will be called, before any other method. - /// This function should be called frequently, even if there is no packets to send or receive. - /// It will always be called last, after all other methods is called, in the main laminar`s loop. + /// When the instance of the connection manager is created, the `update` method will be called, before any other method. + /// This function should be called frequently, even if there are no packets to send or receive. + /// It will always be called last, after all, other methods are called, in the main laminar`s loop. /// It can generate all kinds of packets: heartbeat, user or connection protocol packets. /// (maybe heartbeat functionality should be moved here?) /// It will be called in the loop as long, as it returns any results. E.g. `connect` method may generate multiple results: change state and send packet. @@ -80,7 +80,7 @@ pub trait ConnectionManager: Debug + Send { ) -> Option, ConnectionState>, ConnectionManagerError>>; /// This will be called for all incoming data, including packets that were resent by remote host. - /// If packet is accepted by laminar's reliability layer `process_protocol_data` will be called immediatelly. + /// If the packet is accepted by laminar's reliability layer `process_protocol_data` will be called immediately. /// It should return a slice where header + payload exists fn preprocess_incoming<'a, 'b>( &mut self, @@ -90,42 +90,42 @@ pub trait ConnectionManager: Debug + Send { where 'a: 'b; - /// This will be called for all outgoing data, including packets that are resend. + /// This will be called for all outgoing data, including packets that are resent. /// Dropped packets will also go through here. /// Accepts full packet: header + payload fn postprocess_outgoing<'a, 'b>(&mut self, data: &'a [u8], buffer: &'b mut [u8]) -> &'b [u8] where 'a: 'b; - /// This will be called only for incoming protocol specific packets, after laminar's reliability layer accepted it. - /// This is convenient place to process actual logic, because it is filtered by laminar's reliability layer and it accepts only `PacketType::ConnectionManager` messages. + /// This will be called only for incoming protocol-specific packets after laminar's reliability layer accepted it. + /// This is a convenient place to process actual logic because it is filtered by laminar's reliability layer and it accepts only `PacketType::ConnectionManager` messages. fn process_protocol_data(&mut self, data: &[u8]) -> Result<(), ConnectionManagerError>; /// This will be invoked when player sends connect request, /// Some protocols might provide a way to pass initial connection data, hence the `data` field. - /// This method can only be called when connection is in `Connecting` state + /// This method can only be called when the connection is in `Connecting` state fn connect(&mut self, data: Box<[u8]>); /// This will be invoked when player sends SendEvent::Disconnect request. fn disconnect(&mut self); } -/// Tracks all sorts of global statistics, and decided whether to create `ConnectionManager` for new connections or not. -/// Also decides when connections should be destroyed even if they are in connected state. +/// Tracks all sorts of global statistics and can decided whether to create a `ConnectionManager` for new connections or not. +/// Also decides when connections should be destroyed even if they are in a connected state. pub trait SocketManager: Debug + Send { - /// Decide if it is possible to accept/create new remote connection, this is invoked when message from unknown address arives. + /// Decide if it is possible to accept/create new remote connection, this is invoked when a message from unknown address arrives. fn accept_remote_connection( &mut self, addr: &SocketAddr, raw_bytes: &[u8], ) -> Option>; - /// Decide if it is possible to accept/create new local connection, this is invoked when any user event is recieved for unknown address. + /// Decide if it is possible to accept/create new local connection, this is invoked when any user event is received for unknown address. fn accept_local_connection(&mut self, addr: &SocketAddr) -> Option>; - /// Returns list of connections that socket manager decided to destroy, along with destroy reason + /// Returns a list of connections that the socket manager decided to destroy, along with a destroying reason fn collect_connections_to_destroy(&mut self) -> Option>; - /// All sorts of statistics might be useful here to help deciding whether new connection can be created or not + /// All sorts of statistics might be useful here to help to decide whether a new connection can be created or not fn track_connection_error(&mut self, addr: &SocketAddr, error: &ErrorKind, error_context: &str); fn track_global_error(&mut self, error: &ErrorKind, error_context: &str); fn track_sent_bytes(&mut self, addr: &SocketAddr, bytes: usize); diff --git a/src/net/virtual_connection.rs b/src/net/virtual_connection.rs index cfa61710..6aaf6927 100644 --- a/src/net/virtual_connection.rs +++ b/src/net/virtual_connection.rs @@ -44,7 +44,7 @@ pub struct VirtualConnection { } impl VirtualConnection { - /// Creates and returns a new Connection that wraps the provided socket address + /// Creates and returns a new `VirtualConnection` that wraps the provided socket address pub fn new( addr: SocketAddr, config: &Config, diff --git a/src/packet/packet_structure.rs b/src/packet/packet_structure.rs index e96a5288..acd2ffd3 100644 --- a/src/packet/packet_structure.rs +++ b/src/packet/packet_structure.rs @@ -179,7 +179,7 @@ impl Packet { /// This packet type have similar properties to `Packet` except that it doesn't own anything, and additionally has `PacketType`. #[derive(Debug)] pub struct GenericPacket<'a> { - /// defines packet type, currently Fragment type should not set, + /// defines packet type, currently `Fragment` type is not supported pub(crate) packet_type: PacketType, /// the raw payload of the packet pub(crate) payload: &'a [u8], From c4831ff9046b40a6618f63eaf2d748f100f0da41 Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Thu, 19 Sep 2019 14:16:03 +0300 Subject: [PATCH 13/14] WIP connection manager changes: ConnectionManagerError now is enum, with different error severity types. Multiple missing doc updates, renames, and minor refactorings. --- src/error.rs | 6 ++ src/infrastructure/acknowledgment.rs | 8 ++- src/lib.rs | 3 +- src/managers/simple.rs | 35 ++++++----- src/net/connection.rs | 89 ++++++++++++++++++---------- src/net/events.rs | 2 +- src/net/managers.rs | 16 +++-- src/net/socket.rs | 6 +- src/net/virtual_connection.rs | 18 +++--- src/packet/enums.rs | 10 ++-- src/packet/header/standard_header.rs | 4 +- src/packet/packet_structure.rs | 9 ++- 12 files changed, 132 insertions(+), 74 deletions(-) diff --git a/src/error.rs b/src/error.rs index 6954c225..0d0262f0 100644 --- a/src/error.rs +++ b/src/error.rs @@ -112,6 +112,8 @@ impl Display for DecodingErrorKind { pub enum PacketErrorKind { /// The maximal allowed size of the packet was exceeded ExceededMaxPacketSize, + /// Only user packets (a.k.a PacketType::Packet) can be fragmented + PacketTypeCannotBeFragmented, } impl Display for PacketErrorKind { @@ -120,6 +122,10 @@ impl Display for PacketErrorKind { PacketErrorKind::ExceededMaxPacketSize => { write!(fmt, "The packet size was bigger than the max allowed size.") } + PacketErrorKind::PacketTypeCannotBeFragmented => write!( + fmt, + "Only user packets (PacketType::Packet) can be fragmented." + ), } } } diff --git a/src/infrastructure/acknowledgment.rs b/src/infrastructure/acknowledgment.rs index b268d58b..a0ce1a6d 100644 --- a/src/infrastructure/acknowledgment.rs +++ b/src/infrastructure/acknowledgment.rs @@ -1,5 +1,4 @@ -use crate::packet::OrderingGuarantee; -use crate::packet::SequenceNumber; +use crate::packet::{OrderingGuarantee, PacketType, SequenceNumber}; use crate::sequence_buffer::{sequence_greater_than, sequence_less_than, SequenceBuffer}; use std::collections::HashMap; @@ -101,6 +100,7 @@ impl AcknowledgmentHandler { /// Enqueue the outgoing packet for acknowledgment. pub fn process_outgoing( &mut self, + packet_type: PacketType, payload: &[u8], ordering_guarantee: OrderingGuarantee, item_identifier: Option, @@ -108,6 +108,7 @@ impl AcknowledgmentHandler { self.sent_packets.insert( self.sequence_number, SentPacket { + packet_type, payload: Box::from(payload), ordering_guarantee, item_identifier, @@ -138,8 +139,9 @@ impl AcknowledgmentHandler { } } -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub struct SentPacket { + pub packet_type: PacketType, pub payload: Box<[u8]>, pub ordering_guarantee: OrderingGuarantee, pub item_identifier: Option, diff --git a/src/lib.rs b/src/lib.rs index 0e800b75..2db294b8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,8 +40,7 @@ pub use self::throughput::ThroughputMonitoring; pub use self::config::Config; pub use self::error::{ErrorKind, Result}; +pub use self::net::events::*; pub use self::net::managers::{ConnectionManager, ConnectionManagerError}; pub use self::net::{LinkConditioner, Socket, SocketEventSender}; pub use self::packet::{DeliveryGuarantee, OrderingGuarantee, Packet}; -// pub use self::managers::{SimpleSocketManager}; -pub use self::net::events::*; diff --git a/src/managers/simple.rs b/src/managers/simple.rs index bcd91b02..e5d984e2 100644 --- a/src/managers/simple.rs +++ b/src/managers/simple.rs @@ -8,24 +8,31 @@ use std::net::SocketAddr; use std::time::Instant; /// The simplest connection manager, that immediately goes into the connected state, after creating it -#[derive(Debug, Default)] -struct AlwaysConnectedConManager { - connected: bool, +#[derive(Debug)] +struct AlwaysConnectedConnectionManager { + // this is used to set initial state as connected when creating an instance. + // we'll take this value on first `update` call + initial_state: Option, +} + +impl Default for AlwaysConnectedConnectionManager { + fn default() -> Self { + Self { + // initialize to connected state on creation. + initial_state: Some(ConnectionState::Connected(Box::default())), + } + } } -impl ConnectionManager for AlwaysConnectedConManager { +impl ConnectionManager for AlwaysConnectedConnectionManager { fn update<'a>( &mut self, _buffer: &'a mut [u8], _time: Instant, ) -> Option, ConnectionState>, ConnectionManagerError>> { - if !self.connected { - self.connected = true; - return Some(Ok(Either::Right( - ConnectionState::Connected(Box::default()), - ))); - } - None + self.initial_state + .take() // on first call state will be moved out. + .map(|state| Ok(Either::Right(state))) } fn preprocess_incoming<'a, 'b>( @@ -133,7 +140,7 @@ impl ConnectionManager for SimpleConnectionManager { } } ConnectionState::Connected(_) => { - if data.eq(b"disconnect") { + if data.eq(b"disconnect-") { self.change_state(ConnectionState::Disconnected(TargetHost::RemoteHost)); } } @@ -148,7 +155,7 @@ impl ConnectionManager for SimpleConnectionManager { fn disconnect(&mut self) { if let ConnectionState::Connected(_) = self.state { - self.send_packet(b"disconnect"); + self.send_packet(b"disconnect-"); } self.change_state(ConnectionState::Disconnected(TargetHost::LocalHost)); } @@ -175,7 +182,7 @@ impl SocketManager for SimpleSocketManager { _addr: &SocketAddr, ) -> Option> { if self.0 { - Some(Box::new(AlwaysConnectedConManager::default())) + Some(Box::new(AlwaysConnectedConnectionManager::default())) } else { Some(Box::new(SimpleConnectionManager::default())) } diff --git a/src/net/connection.rs b/src/net/connection.rs index 250d817f..5e46090a 100644 --- a/src/net/connection.rs +++ b/src/net/connection.rs @@ -3,7 +3,7 @@ use crate::{ config::Config, either::Either, net::events::{ConnectionEvent, DestroyReason, DisconnectReason, ReceiveEvent}, - net::managers::{ConnectionState, SocketManager}, + net::managers::{ConnectionManagerError, ConnectionState, SocketManager}, net::socket::SocketWithConditioner, packet::Outgoing, ErrorKind, @@ -22,15 +22,12 @@ use std::{ #[derive(Debug)] pub struct ActiveConnections { connections: HashMap, - buffer: Box<[u8]>, } impl ActiveConnections { pub fn new() -> Self { Self { connections: HashMap::new(), - // TODO actually take from config - buffer: Box::new([0; 1500]), } } @@ -48,6 +45,7 @@ impl ActiveConnections { .or_insert_with(|| VirtualConnection::new(address, config, time, state_manager)) } + /// Returns `VirtualConnection` or None if it doesn't exists for a given address. pub fn try_get(&mut self, address: &SocketAddr) -> Option<&mut VirtualConnection> { self.connections.get_mut(address) } @@ -145,6 +143,8 @@ impl ActiveConnections { }); } + /// Calls `update` method for `ConnectionManager`, in the loop, until it returns None + /// These updates returns either new packets to be sent, or connection state changes. pub fn update_connection_manager( conn: &mut VirtualConnection, sender: &Sender>, @@ -157,27 +157,40 @@ impl ActiveConnections { match changes { Ok(event) => match event { Either::Left(packet) => { - // TODO properly handle error, instead of assert - let packet = conn - .process_outgoing( - packet.packet_type, - packet.payload, - packet.delivery, - packet.ordering, - None, - time, - ) - .expect("connection manager packet should not fail"); - if let Outgoing::Packet(outgoing) = packet { - socket.send_packet_and_log( + match conn.process_outgoing( + packet.packet_type, + packet.payload, + packet.delivery, + packet.ordering, + None, + time, + ) { + Ok(packet) => { + if let Outgoing::Packet(outgoing) = packet { + socket.send_packet_and_log( + &conn.remote_address, + conn.state_manager.as_mut(), + &outgoing.contents(), + manager, + "sending packet from connection manager", + ); + } else { + manager.track_connection_error( + &conn.remote_address, + &ErrorKind::ConnectionError(ConnectionManagerError::Fatal( + String::from( + "connection manager cannot send fragmented packets", + ), + )), + "sending packet from connection manager", + ); + } + } + Err(err) => manager.track_connection_error( &conn.remote_address, - conn.state_manager.as_mut(), - &outgoing.contents(), - manager, + &err, "sending packet from connection manager", - ); - } else { - panic!("connection manager cannot send fragmented packets"); + ), } } Either::Right(state) => { @@ -195,10 +208,19 @@ impl ActiveConnections { )), )) } - _ => panic!( - "Invalid state transition: {:?} -> {:?}", - old, conn.current_state - ), + _ => { + manager.track_connection_error( + &conn.remote_address, + &ErrorKind::ConnectionError(ConnectionManagerError::Fatal( + format!( + "Invalid state transition: {:?} -> {:?}", + old, conn.current_state + ), + )), + "changing connection manager state", + ); + Ok(()) + } } { manager.track_connection_error( &conn.remote_address, @@ -207,9 +229,15 @@ impl ActiveConnections { ); } } else { - panic!( - "Invalid state transition {:?} -> {:?}", - conn.current_state, state + manager.track_connection_error( + &conn.remote_address, + &ErrorKind::ConnectionError(ConnectionManagerError::Fatal( + format!( + "Invalid state transition: {:?} -> {:?}", + conn.current_state, state + ), + )), + "changing connection manager state", ); } } @@ -225,6 +253,7 @@ impl ActiveConnections { } } + /// Iterates through all active connections, and `update`s each connection manager. pub fn update_connections( &mut self, sender: &Sender>, diff --git a/src/net/events.rs b/src/net/events.rs index be896c8a..966eae5d 100644 --- a/src/net/events.rs +++ b/src/net/events.rs @@ -34,7 +34,7 @@ pub enum SendEvent { Disconnect, } -// Provides a reason why the connection was destroyed. +/// Provides a reason why the connection was destroyed. #[derive(Debug, PartialEq, Clone)] pub enum DestroyReason { /// When `SocketManager` decided to destroy a connection for error that arrived from `ConnectionManager`. diff --git a/src/net/managers.rs b/src/net/managers.rs index 1c9a007f..58cdd493 100644 --- a/src/net/managers.rs +++ b/src/net/managers.rs @@ -49,9 +49,15 @@ impl Default for ConnectionState { } } -/// Generic error type, that is used by ConnectionManager implementation +/// Generic error type, that is used by ConnectionManager implementation. #[derive(Debug, PartialEq, Clone)] -pub struct ConnectionManagerError(pub String); +pub enum ConnectionManagerError { + /// Something really bad has happened, this is not a recoverable error, and the connection should be destroyed. + Fatal(String), + /// Something unexpected has happened, but the connection is still in a valid state. + /// `SocketManager` can decide when to destroy the connection if two many warnings are propagated from the same connection in a short amount of time. + Warning(String), // TODO: is it enought? or maybe we need more fields? +} /// It abstracts pure UDP packets, and allows to implement Connected/Disconnected states. /// This table summary shows where exactly ConnectionManager sits in between different layers. @@ -98,15 +104,15 @@ pub trait ConnectionManager: Debug + Send { 'a: 'b; /// This will be called only for incoming protocol-specific packets after laminar's reliability layer accepted it. - /// This is a convenient place to process actual logic because it is filtered by laminar's reliability layer and it accepts only `PacketType::ConnectionManager` messages. + /// This is a convenient place to process actual logic because it is filtered by laminar's reliability layer and it accepts only `PacketType::Connection` messages. fn process_protocol_data(&mut self, data: &[u8]) -> Result<(), ConnectionManagerError>; - /// This will be invoked when player sends connect request, + /// This will be invoked when a user sends connect request, /// Some protocols might provide a way to pass initial connection data, hence the `data` field. /// This method can only be called when the connection is in `Connecting` state fn connect(&mut self, data: Box<[u8]>); - /// This will be invoked when player sends SendEvent::Disconnect request. + /// This will be invoked when a user sends SendEvent::Disconnect request. fn disconnect(&mut self); } diff --git a/src/net/socket.rs b/src/net/socket.rs index 7fc7198f..37867771 100644 --- a/src/net/socket.rs +++ b/src/net/socket.rs @@ -374,12 +374,13 @@ impl Socket { match (event.1, &connection.current_state) { (SendEvent::Packet(packet), ConnectionState::Connected(_)) => { // TODO maybe these should not depend on send_to method? + // Maybe it should be extracted to separate method and added to `manual_poll` function. let dropped = connection.gather_dropped_packets(); let mut processed_packets: Vec = dropped .iter() .flat_map(|waiting_packet| { connection.process_outgoing( - PacketType::Packet, + waiting_packet.packet_type, &waiting_packet.payload, // Because a delivery guarantee is only sent with reliable packets DeliveryGuarantee::Reliable, @@ -470,6 +471,9 @@ impl Socket { }; if let Some(conn) = connection { + let received_payload = conn + .state_manager + .preprocess_incoming(received_payload, &mut self.tmp_buffer)?; conn.process_incoming( received_payload, &self.event_sender, diff --git a/src/net/virtual_connection.rs b/src/net/virtual_connection.rs index 6aaf6927..131433bd 100644 --- a/src/net/virtual_connection.rs +++ b/src/net/virtual_connection.rs @@ -93,6 +93,7 @@ impl VirtualConnection { time.duration_since(self.last_sent) } + // Returns current connection state: `Connecting`, `Connected` or `Disconnected` pub fn get_current_state(&self) -> &ConnectionState { &self.current_state } @@ -115,8 +116,7 @@ impl VirtualConnection { /// This will pre-process the given buffer to be sent over the network. pub fn process_outgoing<'a>( &mut self, - // PacketType is used by connection manager, because it can send all sorts of packets, except fragmented. - // TODO would be nice to change this, so that it is impossible to provide bad value + // TODO (fix somehow?) if this packet is user (Packet), and is large enough that it needs to be fragmented, then this will be changed to Fragment, when building actual packet. packet_type: PacketType, payload: &'a [u8], delivery_guarantee: DeliveryGuarantee, @@ -204,6 +204,11 @@ impl VirtualConnection { Outgoing::Packet(builder.build()) } else { + if packet_type != PacketType::Packet { + return Err(ErrorKind::PacketError( + PacketErrorKind::PacketTypeCannotBeFragmented, + )); + } Outgoing::Fragments( Fragmentation::spit_into_fragments(payload, &self.config)? .into_iter() @@ -247,6 +252,7 @@ impl VirtualConnection { self.congestion_handler .process_outgoing(self.acknowledge_handler.local_sequence_num(), time); self.acknowledge_handler.process_outgoing( + packet_type, payload, ordering_guarantee, item_identifier_value, @@ -257,6 +263,8 @@ impl VirtualConnection { } } + // TODO would be super nice to return iterator of new packets, instead of passing `sender`, and `socket_manager` inside. + // maybe callback: Fn(packet) would also be enough? pub fn process_incoming( &mut self, received_data: &[u8], @@ -264,12 +272,6 @@ impl VirtualConnection { socket_manager: &mut dyn SocketManager, time: Instant, ) -> crate::Result<()> { - // TODO pass buffer from somewhere else - let mut buffer = [0; 1500]; - let received_data = self - .state_manager - .preprocess_incoming(received_data, &mut buffer)?; - self.last_heard = time; let mut packet_reader = PacketReader::new(received_data); diff --git a/src/packet/enums.rs b/src/packet/enums.rs index e6f46f68..8c5c9266 100644 --- a/src/packet/enums.rs +++ b/src/packet/enums.rs @@ -84,14 +84,14 @@ impl TryFrom for OrderingGuarantee { #[derive(Copy, Clone, Debug, PartialOrd, PartialEq)] /// Id to identify a certain packet type. pub enum PacketType { - /// Full packet that is not fragmented + /// User full packet that is not fragmented, Packet = 0, - /// Fragment of a full packet + /// User fragment of a full packet Fragment = 1, /// Heartbeat packet Heartbeat = 2, - /// ConnectionManager specific header - ConnectionManager = 3, + /// Connection manager specific packet + Connection = 3, } impl EnumConverter for PacketType { @@ -109,7 +109,7 @@ impl TryFrom for PacketType { 0 => Ok(PacketType::Packet), 1 => Ok(PacketType::Fragment), 2 => Ok(PacketType::Heartbeat), - 3 => Ok(PacketType::ConnectionManager), + 3 => Ok(PacketType::Connection), _ => Err(ErrorKind::DecodingError(DecodingErrorKind::PacketType)), } } diff --git a/src/packet/header/standard_header.rs b/src/packet/header/standard_header.rs index ba37eeeb..5d42425b 100644 --- a/src/packet/header/standard_header.rs +++ b/src/packet/header/standard_header.rs @@ -63,9 +63,9 @@ impl StandardHeader { self.packet_type == PacketType::Fragment } - // Returns true if packet is only used ConnectionManager + /// Returns true if packet is only used by `ConnectionManager` pub fn is_connection_manager_packet(&self) -> bool { - self.packet_type == PacketType::ConnectionManager + self.packet_type == PacketType::Connection } /// Checks if the protocol version in the packet is a valid version diff --git a/src/packet/packet_structure.rs b/src/packet/packet_structure.rs index acd2ffd3..435bf00d 100644 --- a/src/packet/packet_structure.rs +++ b/src/packet/packet_structure.rs @@ -190,37 +190,40 @@ pub struct GenericPacket<'a> { } impl<'a> GenericPacket<'a> { + /// Creates a connection manager specific packet pub fn manager_packet( payload: &'a [u8], delivery: DeliveryGuarantee, ordering: OrderingGuarantee, ) -> Self { Self { - packet_type: PacketType::ConnectionManager, + packet_type: PacketType::Connection, payload, delivery, ordering, } } + /// Creates a packet that can be received by the user. pub fn user_packet( payload: &'a [u8], delivery: DeliveryGuarantee, ordering: OrderingGuarantee, ) -> Self { Self { - packet_type: PacketType::ConnectionManager, + packet_type: PacketType::Packet, payload, delivery, ordering, } } + /// Creates a heart beat packet. pub fn heartbeat_packet( payload: &'a [u8], delivery: DeliveryGuarantee, ordering: OrderingGuarantee, ) -> Self { Self { - packet_type: PacketType::ConnectionManager, + packet_type: PacketType::Heartbeat, payload, delivery, ordering, From 7ab389b853806cdbac5d5f1c30c9fa79b6b82508 Mon Sep 17 00:00:00 2001 From: Mindaugas Vinkelis Date: Sun, 22 Sep 2019 20:40:16 +0300 Subject: [PATCH 14/14] WIP connection manager, addressing multiple issues. --- examples/server_client.rs | 38 +- examples/simple_udp.rs | 61 +- examples/udp.rs | 87 +- src/error.rs | 20 +- src/lib.rs | 6 +- src/managers.rs | 2 +- src/managers/simple.rs | 71 +- src/net.rs | 6 +- src/net/connection.rs | 309 +++---- src/net/managers.rs | 41 +- src/net/metrics_collector.rs | 35 + src/net/reliability_system.rs | 999 +++++++++++++++++++++++ src/net/socket.rs | 346 +++----- src/net/virtual_connection.rs | 1121 ++++++-------------------- src/packet.rs | 2 +- src/packet/enums.rs | 2 +- src/packet/header/standard_header.rs | 16 - src/packet/outgoing.rs | 8 - src/packet/packet_structure.rs | 12 +- 19 files changed, 1644 insertions(+), 1538 deletions(-) create mode 100644 src/net/metrics_collector.rs create mode 100644 src/net/reliability_system.rs diff --git a/examples/server_client.rs b/examples/server_client.rs index 17b6420e..a3d9410a 100644 --- a/examples/server_client.rs +++ b/examples/server_client.rs @@ -6,19 +6,16 @@ use std::thread; use std::time::Instant; use laminar::{ - managers::SimpleSocketManager, ConnectionEvent, ErrorKind, Packet, ReceiveEvent, Socket, - SocketEventSender, + managers::SimpleConnectionManagerFactory, ConnectionEvent, ErrorKind, Packet, ReceiveEvent, + SendEvent, Socket, }; const SERVER: &str = "127.0.0.1:12351"; fn server() -> Result<(), ErrorKind> { // create socket manager, that will use SimpleConnectionManager, that actually initiates connection by exchanging methods - let mut socket = Socket::bind(SERVER, Box::new(SimpleSocketManager(false)))?; - let (sender, receiver) = ( - SocketEventSender(socket.get_event_sender()), - socket.get_event_receiver(), - ); + let mut socket = Socket::bind(SERVER, Box::new(SimpleConnectionManagerFactory(false)))?; + let (sender, receiver) = (socket.get_event_sender(), socket.get_event_receiver()); let _thread = thread::spawn(move || socket.start_polling()); loop { @@ -40,9 +37,12 @@ fn server() -> Result<(), ErrorKind> { println!("{:?} -> Packet msg:{}", addr, msg); sender - .send(Packet::reliable_unordered( + .send(ConnectionEvent( packet.addr(), - ["Echo: ".as_bytes(), msg.as_bytes()].concat(), + SendEvent::Packet(Packet::reliable_unordered( + packet.addr(), + [b"Echo: ", msg.as_bytes()].concat(), + )), )) .expect("This should send"); } @@ -56,10 +56,10 @@ fn server() -> Result<(), ErrorKind> { fn client() -> Result<(), ErrorKind> { let addr = "127.0.0.1:12352"; - let mut socket = Socket::bind(addr, Box::new(SimpleSocketManager(false)))?; + let mut socket = Socket::bind(addr, Box::new(SimpleConnectionManagerFactory(false)))?; println!("Connected on {}", addr); - let sender = SocketEventSender(socket.get_event_sender()); + let sender = socket.get_event_sender(); let _thread = thread::spawn(move || loop { socket.manual_poll(Instant::now()); @@ -98,13 +98,21 @@ fn client() -> Result<(), ErrorKind> { break; } else if line.starts_with(":c") { sender - .connect(server, Box::from(line.split_at(2).1.as_bytes())) + .send(ConnectionEvent( + server, + SendEvent::Connect(Box::from(line.split_at(2).1.as_bytes())), + )) .expect("sending should not fail"); } else if line == ":d" { - sender.disconnect(server).expect("sending should not fail"); + sender + .send(ConnectionEvent(server, SendEvent::Disconnect)) + .expect("sending should not fail"); } else { sender - .send(Packet::reliable_unordered(server, line.into_bytes())) + .send(ConnectionEvent( + server, + SendEvent::Packet(Packet::reliable_unordered(server, line.into_bytes())), + )) .expect("sending should not fail"); } s_buffer.clear(); @@ -121,7 +129,7 @@ fn main() -> Result<(), ErrorKind> { let mut s = String::new(); stdin.read_line(&mut s)?; - if s.starts_with("s") { + if s.starts_with('s') { println!("Starting server.."); server() } else { diff --git a/examples/simple_udp.rs b/examples/simple_udp.rs index 9feb507a..b8074580 100644 --- a/examples/simple_udp.rs +++ b/examples/simple_udp.rs @@ -3,15 +3,18 @@ //! 2. setting up client to send data. //! 3. serialize data to send and deserialize when received. use bincode::{deserialize, serialize}; -use laminar::{ConnectionEventSender, Packet, Socket}; +use laminar::{ + managers::SimpleConnectionManagerFactory, ConnectionEvent, Packet, ReceiveEvent, SendEvent, + Socket, +}; use serde_derive::{Deserialize, Serialize}; use std::net::SocketAddr; use std::time::Instant; /// The socket address of where the server is located. -const SERVER_ADDR: &'static str = "127.0.0.1:12345"; +const SERVER_ADDR: &str = "127.0.0.1:12345"; // The client address from where the data is sent. -const CLIENT_ADDR: &'static str = "127.0.0.1:12346"; +const CLIENT_ADDR: &str = "127.0.0.1:12346"; fn client_address() -> SocketAddr { CLIENT_ADDR.parse().unwrap() @@ -21,40 +24,55 @@ fn server_address() -> SocketAddr { SERVER_ADDR.parse().unwrap() } +// helper function to reduce boiler plate +fn create_packet(addr: SocketAddr, data: &T) -> ConnectionEvent +where + T: serde::Serialize, +{ + ConnectionEvent( + addr, + SendEvent::Packet(Packet::unreliable(addr, serialize(data).unwrap())), + ) +} + /// This will run an simple example with client and server communicating. #[allow(unused_must_use)] pub fn main() { - let mut server = Socket::bind(server_address()).unwrap(); + let mut server = Socket::bind( + server_address(), + Box::new(SimpleConnectionManagerFactory(true)), + ) + .unwrap(); /* setup or `Client` and send some test data. */ - let mut client = ConnectionEventSender(Socket::bind(client_address()).unwrap()); - - client.send(Packet::unreliable( + let mut client = Socket::bind( + client_address(), + Box::new(SimpleConnectionManagerFactory(true)), + ) + .unwrap(); + client.send(create_packet( server_address(), - serialize(&DataType::Coords { + &DataType::Coords { latitude: 10.55454, longitude: 10.555, altitude: 1.3, - }) - .unwrap(), + }, )); - client.send(Packet::unreliable( + client.send(create_packet( server_address(), - serialize(&DataType::Coords { + &DataType::Coords { latitude: 3.344, longitude: 5.4545, altitude: 1.33, - }) - .unwrap(), + }, )); - client.send(Packet::unreliable( + client.send(create_packet( server_address(), - serialize(&DataType::Text { + &DataType::Text { string: String::from("Some information"), - }) - .unwrap(), + }, )); // Send the queued send operations @@ -68,11 +86,8 @@ pub fn main() { // Coords { longitude: 5.4545, latitude: 3.344, altitude: 1.33 } // Text { string: "Some information" } while let Some(pkt) = server.recv() { - match pkt { - SocketEvent::Packet(pkt) => { - println!["{:?}", deserialize::(pkt.payload()).unwrap()] - } - _ => {} + if let ConnectionEvent(_addr, ReceiveEvent::Packet(pkt)) = pkt { + println!["{:?}", deserialize::(pkt.payload()).unwrap()] } } } diff --git a/examples/udp.rs b/examples/udp.rs index 72c7db3b..4022e62d 100644 --- a/examples/udp.rs +++ b/examples/udp.rs @@ -2,14 +2,18 @@ //! 1. sending data //! 2. receiving data //! 3. constructing the packet for sending. -use laminar::{Packet, Result, Socket, SocketEvent}; +use laminar::{ + managers::SimpleConnectionManagerFactory, ConnectionEvent, Packet, ReceiveEvent, Result, + SendEvent, Socket, +}; use std::net::SocketAddr; +use std::time::Instant; /// The socket address of where the server is located. -const SERVER_ADDR: &'static str = "127.0.0.1:12345"; +const SERVER_ADDR: &str = "127.0.0.1:12345"; // The client address from where the data is sent. -const CLIENT_ADDR: &'static str = "127.0.0.1:12346"; +const CLIENT_ADDR: &str = "127.0.0.1:12346"; fn client_address() -> SocketAddr { CLIENT_ADDR.parse().unwrap() @@ -20,57 +24,70 @@ fn server_address() -> SocketAddr { } /// This is an example of how to send data to an specific address. -pub fn send_data() -> Result<()> { - // Setup a udp socket and bind it to the client address. - let mut socket = Socket::bind(client_address()).unwrap(); - - let packet = construct_packet(); +pub fn send_data(socket: &mut Socket) -> Result<()> { + let (to_address, packet) = construct_packet(); // next send or packet to the endpoint we earlier putted into the packet. - socket.send(packet) + socket.send(ConnectionEvent(to_address, SendEvent::Packet(packet)))?; + + // this function processes all events and actually sends or receives packets + socket.manual_poll(Instant::now()); + Ok(()) } /// This is an example of how to receive data over udp. -pub fn receive_data() { - // setup an udp socket and bind it to the client address. - let mut socket = Socket::bind(server_address()).unwrap(); - +pub fn receive_data(socket: &mut Socket) { + // this function processes all events and actually sends or receives packets + socket.manual_poll(Instant::now()); // Next start receiving. loop { - if let Some(result) = socket.recv() { - match result { - SocketEvent::Packet(packet) => { - let endpoint: SocketAddr = packet.addr(); - let received_data: &[u8] = packet.payload(); - - // you can here deserialize your bytes into the data you have passed it when sending. - - println!( - "Received packet from: {:?} with length {}", - endpoint, - received_data.len() - ); - } - _ => {} - } + if let Some(ConnectionEvent(_addr, ReceiveEvent::Packet(packet))) = socket.recv() { + let endpoint: SocketAddr = packet.addr(); + let received_data: &[u8] = packet.payload(); + + // you can here deserialize your bytes into the data you have passed it when sending. + + println!( + "Received packet from: {:?} with length {}", + endpoint, + received_data.len() + ); break; } } } /// This is an example of how to construct a packet. -pub fn construct_packet() -> Packet { +pub fn construct_packet() -> (SocketAddr, Packet) { // this is the destination address of the packet. let destination: SocketAddr = server_address(); // lets construct some payload (raw data) for or packet. - let raw_data = "example data".as_bytes(); + let raw_data = b"example data"; // lets construct or packet by passing in the destination for this packet and the bytes needed to be send.. - let packet: Packet = Packet::reliable_unordered(destination, raw_data.to_owned()); + let packet: Packet = Packet::reliable_unordered(destination, raw_data.to_vec()); - packet + (destination, packet) } -// TODO: Use functions in example -fn main() {} +fn main() -> Result<()> { + // Setup a udp socket and bind it to the client address. + // SimpleSocketManager(true) provides connection that immediatelly goes to connected state, after any socket event is received + let mut client = Socket::bind( + client_address(), + Box::new(SimpleConnectionManagerFactory(true)), + ) + .unwrap(); + + // setup an udp socket and bind it to the server address. + let mut server = Socket::bind( + server_address(), + Box::new(SimpleConnectionManagerFactory(true)), + ) + .unwrap(); + + send_data(&mut client)?; + receive_data(&mut server); + Ok(()) +} diff --git a/src/error.rs b/src/error.rs index 0d0262f0..1ac8fca3 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,5 @@ //! This module contains the laminar error handling logic. -use crate::either::Either; use crate::net::events::{ConnectionEvent, ReceiveEvent, SendEvent}; use crate::net::managers::ConnectionManagerError; use crossbeam_channel::SendError; @@ -28,8 +27,10 @@ pub enum ErrorKind { ReceivedDataToShort, /// Protocol versions did not match ProtocolVersionMismatch, - /// Could not send on `SendChannel`. - SendError(SendError, ConnectionEvent>>), + /// Could not send a SendEvent from the user. + ChannelSendingError(SendError>), + /// Could not send an ReceiveEvent to the user. + ChannelReceivingError(SendError>), /// Expected header but could not be read from buffer. CouldNotReadHeader(String), /// Errors that is returned from `ConnectionManager` either preprocessing data or processing packet @@ -61,9 +62,14 @@ impl Display for ErrorKind { ErrorKind::ProtocolVersionMismatch => { write!(fmt, "The protocol versions do not match.") } - ErrorKind::SendError(e) => write!( + ErrorKind::ChannelSendingError(e) => write!( fmt, - "Could not sent on channel because it was closed. Reason: {:?}", + "Could not sent a SendEvent on channel because it was closed. Reason: {:?}", + e + ), + ErrorKind::ChannelReceivingError(e) => write!( + fmt, + "Could not sent a ReceiveEvent on channel because it was closed. Reason: {:?}", e ), ErrorKind::CouldNotReadHeader(header) => write!( @@ -190,13 +196,13 @@ impl From for ErrorKind { impl From>> for ErrorKind { fn from(inner: SendError>) -> Self { - ErrorKind::SendError(SendError(Either::Left(inner.0))) + ErrorKind::ChannelSendingError(inner) } } impl From>> for ErrorKind { fn from(inner: SendError>) -> Self { - ErrorKind::SendError(SendError(Either::Right(inner.0))) + ErrorKind::ChannelReceivingError(inner) } } diff --git a/src/lib.rs b/src/lib.rs index 2db294b8..9be3ec99 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,7 +40,9 @@ pub use self::throughput::ThroughputMonitoring; pub use self::config::Config; pub use self::error::{ErrorKind, Result}; -pub use self::net::events::*; +pub use self::net::events::{ + ConnectionEvent, DestroyReason, DisconnectReason, ReceiveEvent, SendEvent, TargetHost, +}; pub use self::net::managers::{ConnectionManager, ConnectionManagerError}; -pub use self::net::{LinkConditioner, Socket, SocketEventSender}; +pub use self::net::{LinkConditioner, Socket}; pub use self::packet::{DeliveryGuarantee, OrderingGuarantee, Packet}; diff --git a/src/managers.rs b/src/managers.rs index 402d60d2..70e49e6d 100644 --- a/src/managers.rs +++ b/src/managers.rs @@ -1,4 +1,4 @@ //! This module provides socket managers. mod simple; -pub use self::simple::SimpleSocketManager; +pub use self::simple::SimpleConnectionManagerFactory; diff --git a/src/managers/simple.rs b/src/managers/simple.rs index e5d984e2..dba40e0c 100644 --- a/src/managers/simple.rs +++ b/src/managers/simple.rs @@ -1,9 +1,7 @@ use crate::net::managers::*; use crate::packet::{DeliveryGuarantee, OrderingGuarantee}; -use log::error; use std::collections::VecDeque; -use std::io::ErrorKind::WouldBlock; use std::net::SocketAddr; use std::time::Instant; @@ -29,10 +27,10 @@ impl ConnectionManager for AlwaysConnectedConnectionManager { &mut self, _buffer: &'a mut [u8], _time: Instant, - ) -> Option, ConnectionState>, ConnectionManagerError>> { + ) -> Option> { self.initial_state .take() // on first call state will be moved out. - .map(|state| Ok(Either::Right(state))) + .map(ConnectionManagerEvent::NewState) } fn preprocess_incoming<'a, 'b>( @@ -86,7 +84,7 @@ impl SimpleConnectionManager { // copy from buffer what we want to send payload.copy_from_slice(data.as_ref()); // create packet - GenericPacket::manager_packet( + GenericPacket::connection_packet( payload, DeliveryGuarantee::Reliable, OrderingGuarantee::None, @@ -99,16 +97,16 @@ impl ConnectionManager for SimpleConnectionManager { &mut self, buffer: &'a mut [u8], _time: Instant, - ) -> Option, ConnectionState>, ConnectionManagerError>> { - match self.changes.pop_front().take() { - Some(change) => Some(Ok(match change { - Either::Left(data) => { - Either::Left(SimpleConnectionManager::get_packet(data, buffer)) - } - Either::Right(state) => Either::Right(state), - })), - None => None, - } + ) -> Option> { + self.changes + .pop_front() + .take() + .map(move |event| match event { + Either::Left(data) => ConnectionManagerEvent::NewPacket( + SimpleConnectionManager::get_packet(data, buffer), + ), + Either::Right(state) => ConnectionManagerEvent::NewState(state), + }) } fn preprocess_incoming<'a, 'b>( @@ -163,51 +161,28 @@ impl ConnectionManager for SimpleConnectionManager { /// Simplest implementation of socket manager, always accept a connection and never destroy, no matter how many errors connection reports /// It can create two types of connection managers: -/// * true - creates `AlwaysConnectedConManager` +/// * true - creates `AlwaysConnectedConnectionManager` /// * false - creates `SimpleConnectionManager` #[derive(Debug)] -pub struct SimpleSocketManager(pub bool); +pub struct SimpleConnectionManagerFactory(pub bool); -impl SocketManager for SimpleSocketManager { - fn accept_remote_connection( +impl ConnectionManagerFactory for SimpleConnectionManagerFactory { + fn create_remote_connection_manager( &mut self, addr: &SocketAddr, _raw_bytes: &[u8], - ) -> Option> { - self.accept_local_connection(addr) + ) -> Box { + self.create_local_connection_manager(addr) } - fn accept_local_connection( + fn create_local_connection_manager( &mut self, _addr: &SocketAddr, - ) -> Option> { + ) -> Box { if self.0 { - Some(Box::new(AlwaysConnectedConnectionManager::default())) + Box::new(AlwaysConnectedConnectionManager::default()) } else { - Some(Box::new(SimpleConnectionManager::default())) - } - } - - fn collect_connections_to_destroy(&mut self) -> Option> { - None - } - - fn track_connection_error( - &mut self, - addr: &SocketAddr, - error: &ErrorKind, - error_context: &str, - ) { - match error { - ErrorKind::IOError(ref e) if e.kind() == WouldBlock => {} - _ => error!("Error, {} ({:?}): {:?}", error_context, addr, error), + Box::new(SimpleConnectionManager::default()) } } - fn track_global_error(&mut self, error: &ErrorKind, error_context: &str) { - error!("Error, {}: {:?}", error_context, error); - } - fn track_sent_bytes(&mut self, _addr: &SocketAddr, _bytes: usize) {} - fn track_received_bytes(&mut self, _addr: &SocketAddr, _bytes: usize) {} - fn track_ignored_bytes(&mut self, _addr: &SocketAddr, _bytes: usize) {} - fn track_connection_destroyed(&mut self, _addr: &SocketAddr) {} } diff --git a/src/net.rs b/src/net.rs index ddc72e9f..0ffe22d6 100644 --- a/src/net.rs +++ b/src/net.rs @@ -3,7 +3,9 @@ mod connection; mod link_conditioner; +mod metrics_collector; mod quality; +mod reliability_system; mod socket; mod virtual_connection; @@ -12,6 +14,8 @@ pub mod events; pub mod managers; pub use self::link_conditioner::LinkConditioner; +pub use self::metrics_collector::MetricsCollector; pub use self::quality::{NetworkQuality, RttMeasurer}; -pub use self::socket::{Socket, SocketEventSender}; +pub use self::reliability_system::{IncomingPackets, OutgoingPackets, ReliabilitySystem}; +pub use self::socket::{Socket, SocketWithConditioner}; pub use self::virtual_connection::VirtualConnection; diff --git a/src/net/connection.rs b/src/net/connection.rs index 5e46090a..86bac45e 100644 --- a/src/net/connection.rs +++ b/src/net/connection.rs @@ -1,15 +1,15 @@ -pub use crate::net::{managers::ConnectionManager, NetworkQuality, RttMeasurer, VirtualConnection}; +pub use crate::net::{ + managers::ConnectionManager, NetworkQuality, ReliabilitySystem, RttMeasurer, VirtualConnection, +}; use crate::{ - config::Config, - either::Either, net::events::{ConnectionEvent, DestroyReason, DisconnectReason, ReceiveEvent}, - net::managers::{ConnectionManagerError, ConnectionState, SocketManager}, + net::managers::ConnectionState, net::socket::SocketWithConditioner, - packet::Outgoing, + net::MetricsCollector, ErrorKind, }; -use crossbeam_channel::{self, SendError, Sender}; +use crossbeam_channel::{self, Sender}; use std::{ collections::HashMap, @@ -17,32 +17,48 @@ use std::{ time::{Duration, Instant}, }; -/// Maintains a registry of active "connections". Essentially, when we receive a packet on the -/// socket from a particular `SocketAddr`, we will track information about it here. +/// Maintains a registry of active "connections". #[derive(Debug)] pub struct ActiveConnections { connections: HashMap, } impl ActiveConnections { + /// Initialized active connection list. pub fn new() -> Self { Self { connections: HashMap::new(), } } - /// Try to get a `VirtualConnection` by address. If the connection does not exist, it will be - /// inserted and returned. - pub fn get_or_insert_connection( + /// Inserts new connection, and calls `update` method on `ConnectionManager` to initialized it. + pub fn insert_and_init_connection( &mut self, - address: SocketAddr, - config: &Config, + connection: VirtualConnection, + socket: &mut SocketWithConditioner, + event_sender: &Sender>, + metrics: &mut MetricsCollector, time: Instant, - state_manager: Box, + tmp_buffer: &mut [u8], ) -> &mut VirtualConnection { - self.connections - .entry(address) - .or_insert_with(|| VirtualConnection::new(address, config, time, state_manager)) + let conn = self + .connections + .entry(connection.remote_address()) + .or_insert(connection); + + conn.update_connection_manager(event_sender, metrics, socket, time, tmp_buffer); + metrics.track_connection_created(&conn.remote_address()); + if let Err(err) = event_sender.send(ConnectionEvent( + conn.remote_address(), + ReceiveEvent::Created, + )) { + metrics.track_connection_error( + &conn.remote_address(), + &ErrorKind::from(err), + "sending connection create event", + ); + } + conn } /// Returns `VirtualConnection` or None if it doesn't exists for a given address. @@ -50,225 +66,98 @@ impl ActiveConnections { self.connections.get_mut(address) } - /// Removes the connection from `ActiveConnections` by socket address. - pub fn remove_connection( + /// Iterates through all active connections, and `update`s each connection manager. + pub fn update_connections( + &mut self, + sender: &Sender>, + metrics: &mut MetricsCollector, + socket: &mut SocketWithConditioner, + time: Instant, + buffer: &mut [u8], + ) { + self.connections.iter_mut().for_each(|(_, conn)| { + conn.update_connection_manager(sender, metrics, socket, time, buffer) + }); + } + + /// Iterate through all of the connections and check if any of them should be dropped. + /// Remove dropped connections from the active connections. For each connection removed, we will send an event to the `event_sender` channel. + pub fn handle_dead_clients( + &mut self, + time: Instant, + idle_connection_timeout: Duration, + sender: &Sender>, + metrics: &mut MetricsCollector, + ) { + let drop_list: Vec<_> = self + .connections + .iter_mut() + .filter_map(|(_, connection)| { + connection + .should_be_dropped(idle_connection_timeout, time) + .map(|reason| (connection.remote_address(), reason)) + }) + .collect(); + + for (address, reason) in drop_list { + self.remove_connection(&address, sender, metrics, reason, "removing dead clients"); + } + } + + pub fn handle_heartbeat( + &mut self, + time: Instant, + heartbeat_interval: Duration, + socket: &mut SocketWithConditioner, + metrics: &mut MetricsCollector, + ) { + // Iterate over all connections which have not sent a packet for a duration of at least + // `heartbeat_interval` (from config), and send a heartbeat packet to each. + let connections = self.connections.iter_mut(); + connections.for_each(|(_, connection)| { + connection.handle_heartbeat(time, heartbeat_interval, socket, metrics); + }); + } + + /// Removes the connection from `ActiveConnections` by socket address, and sends appropriate events. + fn remove_connection( &mut self, address: &SocketAddr, sender: &Sender>, - manager: &mut dyn SocketManager, + metrics: &mut MetricsCollector, reason: DestroyReason, error_context: &str, ) -> bool { if let Some((_, conn)) = self.connections.remove_entry(address) { if let ConnectionState::Connected(_) = conn.get_current_state() { if let Err(err) = sender.send(ConnectionEvent( - conn.remote_address, + conn.remote_address(), ReceiveEvent::Disconnected(DisconnectReason::Destroying(reason.clone())), )) { - manager.track_connection_error( - &conn.remote_address, - &ErrorKind::SendError(SendError(Either::Right(err.0))), + metrics.track_connection_error( + &conn.remote_address(), + &ErrorKind::from(err), error_context, ); } } if let Err(err) = sender.send(ConnectionEvent( - conn.remote_address, + conn.remote_address(), ReceiveEvent::Destroyed(reason), )) { - manager.track_connection_error( - &conn.remote_address, - &ErrorKind::SendError(SendError(Either::Right(err.0))), + metrics.track_connection_error( + &conn.remote_address(), + &ErrorKind::from(err), error_context, ); } - manager.track_connection_destroyed(address); + metrics.track_connection_destroyed(address); true } else { false } } - /// Check for and return `VirtualConnection`s which have been idling longer than `max_idle_time`. - pub fn idle_connections(&mut self, max_idle_time: Duration, time: Instant) -> Vec { - self.connections - .iter() - .filter(|(_, connection)| connection.last_heard(time) >= max_idle_time) - .map(|(address, _)| *address) - .collect() - } - - /// Get a list of addresses of dead connections - pub fn dead_connections(&mut self) -> Vec<(SocketAddr, DestroyReason)> { - self.connections - .iter() - .filter_map(|(_, connection)| { - if connection.should_be_dropped() { - Some(( - connection.remote_address, - DestroyReason::TooManyPacketsInFlight, - )) - } else if connection.is_disconnected() { - Some(( - connection.remote_address, - DestroyReason::GracefullyDisconnected, - )) - } else { - None - } - }) - .collect() - } - - /// Check for and return `VirtualConnection`s which have not sent anything for a duration of at least `heartbeat_interval`. - pub fn heartbeat_required_connections( - &mut self, - heartbeat_interval: Duration, - time: Instant, - manager: &mut dyn SocketManager, - socket: &mut SocketWithConditioner, - ) { - self.connections - .iter_mut() - .filter(move |(_, connection)| connection.last_sent(time) >= heartbeat_interval) - .for_each(|(_, connection)| { - let packet = connection.create_and_process_heartbeat(time); - socket.send_packet_and_log( - &connection.remote_address, - connection.state_manager.as_mut(), - &packet.contents(), - manager, - "sending heartbeat packet", - ); - }); - } - - /// Calls `update` method for `ConnectionManager`, in the loop, until it returns None - /// These updates returns either new packets to be sent, or connection state changes. - pub fn update_connection_manager( - conn: &mut VirtualConnection, - sender: &Sender>, - manager: &mut dyn SocketManager, - socket: &mut SocketWithConditioner, - time: Instant, - buffer: &mut [u8], - ) { - while let Some(changes) = conn.state_manager.update(buffer, time) { - match changes { - Ok(event) => match event { - Either::Left(packet) => { - match conn.process_outgoing( - packet.packet_type, - packet.payload, - packet.delivery, - packet.ordering, - None, - time, - ) { - Ok(packet) => { - if let Outgoing::Packet(outgoing) = packet { - socket.send_packet_and_log( - &conn.remote_address, - conn.state_manager.as_mut(), - &outgoing.contents(), - manager, - "sending packet from connection manager", - ); - } else { - manager.track_connection_error( - &conn.remote_address, - &ErrorKind::ConnectionError(ConnectionManagerError::Fatal( - String::from( - "connection manager cannot send fragmented packets", - ), - )), - "sending packet from connection manager", - ); - } - } - Err(err) => manager.track_connection_error( - &conn.remote_address, - &err, - "sending packet from connection manager", - ), - } - } - Either::Right(state) => { - if let Some(old) = conn.current_state.try_change(&state) { - if let Err(err) = match &conn.current_state { - ConnectionState::Connected(data) => sender.send(ConnectionEvent( - conn.remote_address, - ReceiveEvent::Connected(data.clone()), - )), - ConnectionState::Disconnected(closed_by) => { - sender.send(ConnectionEvent( - conn.remote_address, - ReceiveEvent::Disconnected(DisconnectReason::ClosedBy( - closed_by.clone(), - )), - )) - } - _ => { - manager.track_connection_error( - &conn.remote_address, - &ErrorKind::ConnectionError(ConnectionManagerError::Fatal( - format!( - "Invalid state transition: {:?} -> {:?}", - old, conn.current_state - ), - )), - "changing connection manager state", - ); - Ok(()) - } - } { - manager.track_connection_error( - &conn.remote_address, - &ErrorKind::SendError(SendError(Either::Right(err.0))), - "sending connection state update", - ); - } - } else { - manager.track_connection_error( - &conn.remote_address, - &ErrorKind::ConnectionError(ConnectionManagerError::Fatal( - format!( - "Invalid state transition: {:?} -> {:?}", - conn.current_state, state - ), - )), - "changing connection manager state", - ); - } - } - }, - Err(err) => { - manager.track_connection_error( - &conn.remote_address, - &ErrorKind::ConnectionError(err), - "recieved connection manager error", - ); - } - } - } - } - - /// Iterates through all active connections, and `update`s each connection manager. - pub fn update_connections( - &mut self, - sender: &Sender>, - manager: &mut dyn SocketManager, - socket: &mut SocketWithConditioner, - time: Instant, - buffer: &mut [u8], - ) { - self.connections.iter_mut().for_each(|(_, conn)| { - ActiveConnections::update_connection_manager( - conn, sender, manager, socket, time, buffer, - ) - }); - } - /// Returns the number of connected clients. #[cfg(test)] pub(crate) fn count(&self) -> usize { @@ -285,8 +174,6 @@ mod tests { time::{Duration, Instant}, }; - use super::managers::ConnectionManager; - #[derive(Debug)] struct DummyConnManager {} diff --git a/src/net/managers.rs b/src/net/managers.rs index 58cdd493..3c322550 100644 --- a/src/net/managers.rs +++ b/src/net/managers.rs @@ -45,7 +45,7 @@ impl ConnectionState { impl Default for ConnectionState { fn default() -> Self { - Self::Connecting + ConnectionState::Connecting } } @@ -59,6 +59,17 @@ pub enum ConnectionManagerError { Warning(String), // TODO: is it enought? or maybe we need more fields? } +/// Events that are generated by `ConnectionManager`s update method. +#[derive(Debug)] +pub enum ConnectionManagerEvent<'a> { + /// Generated new packet, that will be sent immediately. + NewPacket(GenericPacket<'a>), + /// `ConnectionState` state changed. + NewState(ConnectionState), + /// Error occured within `ConnectionManager` + Error(ConnectionManagerError), +} + /// It abstracts pure UDP packets, and allows to implement Connected/Disconnected states. /// This table summary shows where exactly ConnectionManager sits in between different layers. /// | Abstraction layer | Capabilities | @@ -83,7 +94,7 @@ pub trait ConnectionManager: Debug + Send { &mut self, buffer: &'a mut [u8], time: Instant, - ) -> Option, ConnectionState>, ConnectionManagerError>>; + ) -> Option>; /// This will be called for all incoming data, including packets that were resent by remote host. /// If the packet is accepted by laminar's reliability layer `process_protocol_data` will be called immediately. @@ -116,26 +127,14 @@ pub trait ConnectionManager: Debug + Send { fn disconnect(&mut self); } -/// Tracks all sorts of global statistics and can decided whether to create a `ConnectionManager` for new connections or not. -/// Also decides when connections should be destroyed even if they are in a connected state. -pub trait SocketManager: Debug + Send { - /// Decide if it is possible to accept/create new remote connection, this is invoked when a message from unknown address arrives. - fn accept_remote_connection( +/// Factory that knows how to create a ConnectionManager for a connection. +pub trait ConnectionManagerFactory: Debug + Send { + /// This is invoked when a message from unknown address arrives. + fn create_remote_connection_manager( &mut self, addr: &SocketAddr, raw_bytes: &[u8], - ) -> Option>; - /// Decide if it is possible to accept/create new local connection, this is invoked when any user event is received for unknown address. - fn accept_local_connection(&mut self, addr: &SocketAddr) -> Option>; - - /// Returns a list of connections that the socket manager decided to destroy, along with a destroying reason - fn collect_connections_to_destroy(&mut self) -> Option>; - - /// All sorts of statistics might be useful here to help to decide whether a new connection can be created or not - fn track_connection_error(&mut self, addr: &SocketAddr, error: &ErrorKind, error_context: &str); - fn track_global_error(&mut self, error: &ErrorKind, error_context: &str); - fn track_sent_bytes(&mut self, addr: &SocketAddr, bytes: usize); - fn track_received_bytes(&mut self, addr: &SocketAddr, bytes: usize); - fn track_ignored_bytes(&mut self, addr: &SocketAddr, bytes: usize); - fn track_connection_destroyed(&mut self, addr: &SocketAddr); + ) -> Box; + /// This is invoked when any user event is received for unknown address. + fn create_local_connection_manager(&mut self, addr: &SocketAddr) -> Box; } diff --git a/src/net/metrics_collector.rs b/src/net/metrics_collector.rs new file mode 100644 index 00000000..b1b519c3 --- /dev/null +++ b/src/net/metrics_collector.rs @@ -0,0 +1,35 @@ +use crate::error::ErrorKind; + +use log::error; +use std::io::ErrorKind::WouldBlock; +use std::net::SocketAddr; + +/// Tracks all sorts of global statistics +// TODO write implementation of this +#[derive(Debug)] +pub struct MetricsCollector {} + +impl MetricsCollector { + pub fn track_connection_error( + &mut self, + addr: &SocketAddr, + error: &ErrorKind, + error_context: &str, + ) { + match error { + ErrorKind::IOError(ref e) if e.kind() == WouldBlock => {} + _ => error!("Error, {} ({:?}): {:?}", error_context, addr, error), + } + } + pub fn track_global_error(&mut self, error: &ErrorKind, error_context: &str) { + error!("Error, {}: {:?}", error_context, error); + } + + pub fn track_sent_bytes(&mut self, _addr: &SocketAddr, _bytes: usize) {} + pub fn track_received_bytes(&mut self, _addr: &SocketAddr, _bytes: usize) {} + pub fn track_ignored_bytes(&mut self, _addr: &SocketAddr, _bytes: usize) {} + + pub fn track_connection_created(&mut self, _addr: &SocketAddr) {} + + pub fn track_connection_destroyed(&mut self, _addr: &SocketAddr) {} +} diff --git a/src/net/reliability_system.rs b/src/net/reliability_system.rs new file mode 100644 index 00000000..116a7109 --- /dev/null +++ b/src/net/reliability_system.rs @@ -0,0 +1,999 @@ +use crate::{ + config::Config, + either::Either, + error::{ErrorKind, PacketErrorKind, Result}, + infrastructure::{ + arranging::{Arranging, ArrangingSystem, OrderingSystem, SequencingSystem}, + AcknowledgmentHandler, CongestionHandler, Fragmentation, SentPacket, + }, + net::constants::{ + ACKED_PACKET_HEADER, DEFAULT_ORDERING_STREAM, DEFAULT_SEQUENCING_STREAM, + STANDARD_HEADER_SIZE, + }, + packet::{ + DeliveryGuarantee, GenericPacket, OrderingGuarantee, OutgoingPacket, OutgoingPacketBuilder, + Packet, PacketReader, PacketType, SequenceNumber, + }, +}; + +use std::collections::VecDeque; + +use std::net::SocketAddr; +use std::time::{Duration, Instant}; + +/// Helper class that implement Iterator, and is used to return incomming (from bytes to packets) or outgoing (from packet to bytes) packets. +/// It is used as optimization in cases, where most of the time there is only one element to iterate, and we don't want to create vector for it +pub struct ZeroOrMore { + pub data: Either, VecDeque>, +} + +impl ZeroOrMore { + pub fn zero() -> Self { + Self { + data: Either::Left(None), + } + } + + pub fn one(data: T) -> Self { + Self { + data: Either::Left(Some(data)), + } + } + + pub fn many(vec: VecDeque) -> Self { + Self { + data: Either::Right(vec), + } + } +} + +impl Iterator for ZeroOrMore { + type Item = T; + + fn next(&mut self) -> Option { + match &mut self.data { + Either::Left(option) => option.take(), + Either::Right(vec) => vec.pop_front(), + } + } +} + +/// Stores packets with headers that will be sent to network +pub struct OutgoingPackets<'a> { + data: ZeroOrMore>, +} + +impl<'a> OutgoingPackets<'a> { + pub fn one(packet: OutgoingPacket<'a>) -> Self { + Self { + data: ZeroOrMore::one(packet), + } + } + pub fn many(packets: VecDeque>) -> Self { + Self { + data: ZeroOrMore::many(packets), + } + } +} + +impl<'a> IntoIterator for OutgoingPackets<'a> { + type Item = OutgoingPacket<'a>; + type IntoIter = ZeroOrMore; + + fn into_iter(self) -> Self::IntoIter { + self.data + } +} + +/// Stores parsed packets with their types, that was received from network +pub struct IncomingPackets { + data: ZeroOrMore<(Packet, PacketType)>, +} + +impl IncomingPackets { + pub fn zero() -> Self { + Self { + data: ZeroOrMore::zero(), + } + } + + pub fn one(packet: Packet, packet_type: PacketType) -> Self { + Self { + data: ZeroOrMore::one((packet, packet_type)), + } + } + + pub fn many(vec: VecDeque<(Packet, PacketType)>) -> Self { + Self { + data: ZeroOrMore::many(vec), + } + } +} + +impl IntoIterator for IncomingPackets { + type Item = (Packet, PacketType); + type IntoIter = ZeroOrMore; + + fn into_iter(self) -> Self::IntoIter { + self.data + } +} + +/// Keeps reliability information about connections. +/// It exposes various, connection quality related functions, and provides functions that assembles and disassembles packet with reliability information in it. +pub struct ReliabilitySystem { + last_heard: Instant, + last_sent: Instant, + ordering_system: OrderingSystem<(Box<[u8]>, PacketType)>, + sequencing_system: SequencingSystem>, + acknowledge_handler: AcknowledgmentHandler, + congestion_handler: CongestionHandler, + + config: Config, + fragmentation: Fragmentation, +} + +impl ReliabilitySystem { + /// Creates and returns a new `VirtualConnection` that wraps the provided socket address + pub fn new(config: &Config, time: Instant) -> ReliabilitySystem { + ReliabilitySystem { + last_heard: time, + last_sent: time, + ordering_system: OrderingSystem::new(), + sequencing_system: SequencingSystem::new(), + acknowledge_handler: AcknowledgmentHandler::new(), + congestion_handler: CongestionHandler::new(config), + fragmentation: Fragmentation::new(config), + config: config.to_owned(), + } + } + + /// Determine if this connection should be dropped due to its state + pub fn should_be_dropped(&self) -> bool { + self.acknowledge_handler.packets_in_flight() > self.config.max_packets_in_flight + } + + /// Returns a [Duration] representing the interval since we last heard from the client + pub fn last_heard(&self, time: Instant) -> Duration { + // TODO: Replace with saturating_duration_since once it becomes stable. + // This function panics if the user supplies a time instant earlier than last_heard. + time.duration_since(self.last_heard) + } + + /// Returns a [Duration] representing the interval since we last sent to the client + pub fn last_sent(&self, time: Instant) -> Duration { + // TODO: Replace with saturating_duration_since once it becomes stable. + // This function panics if the user supplies a time instant earlier than last_heard. + time.duration_since(self.last_sent) + } + + /// Constructs outgoing packet(s) from dropped packet. + pub fn process_dropped<'a>( + &mut self, + packet: &'a SentPacket, + time: Instant, + ) -> Result> { + self.process_outgoing_impl( + GenericPacket { + packet_type: packet.packet_type, + payload: &packet.payload, + // Because a delivery guarantee is only sent with reliable packets + delivery: DeliveryGuarantee::Reliable, + // This is stored with the dropped packet because they could be mixed + ordering: packet.ordering_guarantee, + }, + packet.item_identifier, + time, + ) + } + + /// Constructs outgoing packet(s) with provided reliability information, for sending over the network. + pub fn process_outgoing<'a>( + &mut self, + packet: GenericPacket<'a>, + time: Instant, + ) -> Result> { + self.last_sent = time; + self.process_outgoing_impl(packet, None, time) + } + + fn process_outgoing_impl<'a>( + &mut self, + packet: GenericPacket<'a>, + last_item_identifier: Option, + time: Instant, + ) -> Result> { + match packet.delivery { + DeliveryGuarantee::Unreliable => { + if packet.payload.len() <= self.config.receive_buffer_max_size { + if packet.packet_type == PacketType::Heartbeat { + // TODO (bug?) is this required here? + self.congestion_handler + .process_outgoing(self.acknowledge_handler.local_sequence_num(), time); + } + + let mut builder = OutgoingPacketBuilder::new(packet.payload) + .with_default_header(packet.packet_type, packet.delivery, packet.ordering); + + if let OrderingGuarantee::Sequenced(stream_id) = packet.ordering { + let item_identifier = self + .sequencing_system + .get_or_create_stream(stream_id.unwrap_or(DEFAULT_SEQUENCING_STREAM)) + .new_item_identifier(); + + builder = builder.with_sequencing_header(item_identifier as u16, stream_id); + }; + + Ok(OutgoingPackets::one(builder.build())) + } else { + Err(ErrorKind::PacketError( + PacketErrorKind::ExceededMaxPacketSize, + )) + } + } + DeliveryGuarantee::Reliable => { + let payload_length = packet.payload.len() as u16; + + let mut item_identifier_value = None; + let outgoing = { + // spit the packet if the payload length is greater than the allowed fragment size. + if payload_length <= self.config.fragment_size { + let mut builder = OutgoingPacketBuilder::new(packet.payload) + .with_default_header( + packet.packet_type, + packet.delivery, + packet.ordering, + ); + + builder = builder.with_acknowledgment_header( + self.acknowledge_handler.local_sequence_num(), + self.acknowledge_handler.remote_sequence_num(), + self.acknowledge_handler.ack_bitfield(), + ); + + if let OrderingGuarantee::Ordered(stream_id) = packet.ordering { + let item_identifier = + if let Some(item_identifier) = last_item_identifier { + item_identifier + } else { + self.ordering_system + .get_or_create_stream( + stream_id.unwrap_or(DEFAULT_ORDERING_STREAM), + ) + .new_item_identifier() + }; + + item_identifier_value = Some(item_identifier); + + builder = builder.with_ordering_header(item_identifier, stream_id); + }; + + if let OrderingGuarantee::Sequenced(stream_id) = packet.ordering { + let item_identifier = + if let Some(item_identifier) = last_item_identifier { + item_identifier + } else { + self.sequencing_system + .get_or_create_stream( + stream_id.unwrap_or(DEFAULT_SEQUENCING_STREAM), + ) + .new_item_identifier() + }; + + item_identifier_value = Some(item_identifier); + + builder = builder.with_sequencing_header(item_identifier, stream_id); + }; + + OutgoingPackets::one(builder.build()) + } else { + if packet.packet_type != PacketType::Packet { + return Err(ErrorKind::PacketError( + PacketErrorKind::PacketTypeCannotBeFragmented, + )); + } + OutgoingPackets::many( + Fragmentation::spit_into_fragments(packet.payload, &self.config)? + .into_iter() + .enumerate() + .map(|(fragment_id, fragment)| { + let fragments_needed = Fragmentation::fragments_needed( + payload_length, + self.config.fragment_size, + ) + as u8; + + let mut builder = OutgoingPacketBuilder::new(fragment) + .with_default_header( + PacketType::Fragment, + packet.delivery, + packet.ordering, + ); + + builder = builder.with_fragment_header( + self.acknowledge_handler.local_sequence_num(), + fragment_id as u8, + fragments_needed, + ); + if fragment_id == 0 { + // TODO (bug?) why there is no ordering/sequencing for fragmented packet? + builder = builder.with_acknowledgment_header( + self.acknowledge_handler.local_sequence_num(), + self.acknowledge_handler.remote_sequence_num(), + self.acknowledge_handler.ack_bitfield(), + ); + } + + builder.build() + }) + .collect(), + ) + } + }; + + self.congestion_handler + .process_outgoing(self.acknowledge_handler.local_sequence_num(), time); + self.acknowledge_handler.process_outgoing( + packet.packet_type, + packet.payload, + packet.ordering, + item_identifier_value, + ); + + Ok(outgoing) + } + } + } + + /// Constructs incoming packet(s) from bytes. + pub fn process_incoming( + &mut self, + addr: SocketAddr, + received_data: &[u8], + time: Instant, + ) -> Result { + self.last_heard = time; + let mut packet_reader = PacketReader::new(received_data); + + let header = packet_reader.read_standard_header()?; + + if !header.is_current_protocol() { + return Err(ErrorKind::ProtocolVersionMismatch); + } + + if header.packet_type() == PacketType::Heartbeat { + // Heartbeat packets are unreliable, unordered and empty packets. + // We already updated our `self.last_heard` time, nothing else to be done. + return Ok(IncomingPackets::zero()); + } + + match header.delivery_guarantee() { + DeliveryGuarantee::Unreliable => { + if let OrderingGuarantee::Sequenced(_id) = header.ordering_guarantee() { + let arranging_header = + packet_reader.read_arranging_header(u16::from(STANDARD_HEADER_SIZE))?; + + let payload = packet_reader.read_payload(); + + let stream = self + .sequencing_system + .get_or_create_stream(arranging_header.stream_id()); + + if let Some(packet) = stream.arrange(arranging_header.arranging_id(), payload) { + return Ok(IncomingPackets::one( + Packet::new( + addr, + packet, + header.delivery_guarantee(), + OrderingGuarantee::Sequenced(Some(arranging_header.stream_id())), + ), + header.packet_type(), + )); + } + + return Ok(IncomingPackets::zero()); + } + return Ok(IncomingPackets::one( + Packet::new( + addr, + packet_reader.read_payload(), + header.delivery_guarantee(), + header.ordering_guarantee(), + ), + header.packet_type(), + )); + } + DeliveryGuarantee::Reliable => { + if header.packet_type() == PacketType::Fragment { + if let Ok((fragment_header, acked_header)) = packet_reader.read_fragment() { + let payload = packet_reader.read_payload(); + + if let Some(acked_header) = acked_header { + // TODO (bug?) should we also check for sequencing/ordering? + self.congestion_handler + .process_incoming(acked_header.sequence()); + self.acknowledge_handler.process_incoming( + acked_header.sequence(), + acked_header.ack_seq(), + acked_header.ack_field(), + ); + } + + match self + .fragmentation + .handle_fragment(fragment_header, &payload) + { + Ok(Some(payload)) => { + return Ok(IncomingPackets::one( + Packet::new( + addr, + payload.into_boxed_slice(), + header.delivery_guarantee(), + OrderingGuarantee::None, + ), + PacketType::Packet, // change to `Packet`, because we do inverse action in `process_outgoing`. + )); + } + Ok(None) => return Ok(IncomingPackets::zero()), + Err(e) => return Err(e), + }; + } + } else { + let acked_header = packet_reader.read_acknowledge_header()?; + + self.congestion_handler + .process_incoming(acked_header.sequence()); + self.acknowledge_handler.process_incoming( + acked_header.sequence(), + acked_header.ack_seq(), + acked_header.ack_field(), + ); + + if let OrderingGuarantee::Sequenced(_) = header.ordering_guarantee() { + let arranging_header = packet_reader.read_arranging_header(u16::from( + STANDARD_HEADER_SIZE + ACKED_PACKET_HEADER, + ))?; + + let payload = packet_reader.read_payload(); + + let stream = self + .sequencing_system + .get_or_create_stream(arranging_header.stream_id()); + + if let Some(packet) = + stream.arrange(arranging_header.arranging_id(), payload) + { + return Ok(IncomingPackets::one( + Packet::new( + addr, + packet, + header.delivery_guarantee(), + OrderingGuarantee::Sequenced(Some( + arranging_header.stream_id(), + )), + ), + header.packet_type(), + )); + } + } else if let OrderingGuarantee::Ordered(_id) = header.ordering_guarantee() { + let arranging_header = packet_reader.read_arranging_header(u16::from( + STANDARD_HEADER_SIZE + ACKED_PACKET_HEADER, + ))?; + + let payload = packet_reader.read_payload(); + + let stream = self + .ordering_system + .get_or_create_stream(arranging_header.stream_id()); + let arranged_packet = stream.arrange( + arranging_header.arranging_id(), + (payload, header.packet_type()), + ); + return Ok(IncomingPackets::many( + arranged_packet + .into_iter() + .chain(stream.iter_mut()) + .map(|(packet, packet_type)| { + ( + Packet::new( + addr, + packet, + header.delivery_guarantee(), + OrderingGuarantee::Ordered(Some( + arranging_header.stream_id(), + )), + ), + packet_type, + ) + }) + .collect(), + )); + } else { + let payload = packet_reader.read_payload(); + return Ok(IncomingPackets::one( + Packet::new( + addr, + payload, + header.delivery_guarantee(), + header.ordering_guarantee(), + ), + header.packet_type(), + )); + } + } + } + } + + Ok(IncomingPackets::zero()) + } + + /// This will gather dropped packets from the acknowledgment handler. + /// + /// Note that after requesting dropped packets the dropped packets will be removed from this client. + pub fn gather_dropped_packets(&mut self) -> Vec { + self.acknowledge_handler.dropped_packets() + } +} + +#[cfg(test)] +mod tests { + use super::VirtualConnection; + use crate::config::Config; + use crate::net::constants; + use crate::packet::header::{AckedPacketHeader, ArrangingHeader, HeaderWriter, StandardHeader}; + use crate::packet::{DeliveryGuarantee, OrderingGuarantee, Outgoing, Packet, PacketType}; + use crate::protocol_version::ProtocolVersion; + use crate::SocketEvent; + use byteorder::{BigEndian, WriteBytesExt}; + use crossbeam_channel::{unbounded, TryRecvError}; + use std::io::Write; + use std::time::Instant; + + const PAYLOAD: [u8; 4] = [1, 2, 3, 4]; + + #[test] + fn assure_right_fragmentation() { + let mut protocol_version = Vec::new(); + protocol_version + .write_u16::(ProtocolVersion::get_crc16()) + .unwrap(); + + let standard_header = [protocol_version, vec![1, 1, 2]].concat(); + + let acked_header = vec![1, 0, 0, 2, 0, 0, 0, 3]; + let first_fragment = vec![0, 1, 1, 3]; + let second_fragment = vec![0, 1, 2, 3]; + let third_fragment = vec![0, 1, 3, 3]; + + let (tx, rx) = unbounded::(); + + let mut connection = create_virtual_connection(); + connection + .process_incoming( + [standard_header.as_slice(), acked_header.as_slice()] + .concat() + .as_slice(), + &tx, + Instant::now(), + ) + .unwrap(); + assert!(rx.try_recv().is_err()); + connection + .process_incoming( + [ + standard_header.as_slice(), + first_fragment.as_slice(), + &PAYLOAD, + ] + .concat() + .as_slice(), + &tx, + Instant::now(), + ) + .unwrap(); + assert!(rx.try_recv().is_err()); + connection + .process_incoming( + [ + standard_header.as_slice(), + second_fragment.as_slice(), + &PAYLOAD, + ] + .concat() + .as_slice(), + &tx, + Instant::now(), + ) + .unwrap(); + assert!(rx.try_recv().is_err()); + connection + .process_incoming( + [ + standard_header.as_slice(), + third_fragment.as_slice(), + &PAYLOAD, + ] + .concat() + .as_slice(), + &tx, + Instant::now(), + ) + .unwrap(); + + let complete_fragment = rx.try_recv().unwrap(); + + match complete_fragment { + SocketEvent::Packet(fragment) => assert_eq!( + fragment.payload(), + &*[PAYLOAD, PAYLOAD, PAYLOAD].concat().into_boxed_slice() + ), + _ => { + panic!("Expected fragment other result."); + } + } + } + + #[test] + fn expect_fragmentation() { + let mut connection = create_virtual_connection(); + + let buffer = vec![1; 4000]; + + let outgoing = connection + .process_outgoing( + &buffer, + DeliveryGuarantee::Reliable, + OrderingGuarantee::Ordered(None), + None, + Instant::now(), + ) + .unwrap(); + + match outgoing { + Outgoing::Packet(_) => panic!("Expected fragment got packet"), + Outgoing::Fragments(fragments) => { + assert_eq!(fragments.len(), 4); + } + } + } + + #[test] + fn assure_correct_outgoing_processing() { + let mut connection = create_virtual_connection(); + + let buffer = vec![1; 1000]; + + connection + .process_outgoing( + &buffer, + DeliveryGuarantee::Unreliable, + OrderingGuarantee::None, + None, + Instant::now(), + ) + .unwrap(); + + connection + .process_outgoing( + &buffer, + DeliveryGuarantee::Unreliable, + OrderingGuarantee::Sequenced(None), + None, + Instant::now(), + ) + .unwrap(); + + connection + .process_outgoing( + &buffer, + DeliveryGuarantee::Reliable, + OrderingGuarantee::Ordered(None), + None, + Instant::now(), + ) + .unwrap(); + + connection + .process_outgoing( + &buffer, + DeliveryGuarantee::Reliable, + OrderingGuarantee::Sequenced(None), + None, + Instant::now(), + ) + .unwrap(); + } + + #[test] + fn assure_right_sequencing() { + let mut connection = create_virtual_connection(); + + assert_incoming_with_order( + DeliveryGuarantee::Unreliable, + OrderingGuarantee::Sequenced(Some(1)), + &mut connection, + Ok(SocketEvent::Packet(Packet::unreliable_sequenced( + get_fake_addr(), + PAYLOAD.to_vec(), + Some(1), + ))), + 1, + ); + + assert_incoming_with_order( + DeliveryGuarantee::Unreliable, + OrderingGuarantee::Sequenced(Some(1)), + &mut connection, + Ok(SocketEvent::Packet(Packet::unreliable_sequenced( + get_fake_addr(), + PAYLOAD.to_vec(), + Some(1), + ))), + 3, + ); + + assert_incoming_with_order( + DeliveryGuarantee::Unreliable, + OrderingGuarantee::Sequenced(Some(1)), + &mut connection, + Err(TryRecvError::Empty), + 2, + ); + + assert_incoming_with_order( + DeliveryGuarantee::Unreliable, + OrderingGuarantee::Sequenced(Some(1)), + &mut connection, + Ok(SocketEvent::Packet(Packet::unreliable_sequenced( + get_fake_addr(), + PAYLOAD.to_vec(), + Some(1), + ))), + 4, + ); + + assert_incoming_with_order( + DeliveryGuarantee::Reliable, + OrderingGuarantee::Sequenced(Some(1)), + &mut connection, + Ok(SocketEvent::Packet(Packet::reliable_sequenced( + get_fake_addr(), + PAYLOAD.to_vec(), + Some(1), + ))), + 5, + ); + } + + #[test] + fn assure_right_ordering() { + let mut connection = create_virtual_connection(); + + assert_incoming_with_order( + DeliveryGuarantee::Reliable, + OrderingGuarantee::Ordered(Some(1)), + &mut connection, + Ok(SocketEvent::Packet(Packet::reliable_ordered( + get_fake_addr(), + PAYLOAD.to_vec(), + Some(1), + ))), + 0, + ); + + assert_incoming_with_order( + DeliveryGuarantee::Reliable, + OrderingGuarantee::Ordered(Some(1)), + &mut connection, + Err(TryRecvError::Empty), + 2, + ); + + assert_incoming_with_order( + DeliveryGuarantee::Reliable, + OrderingGuarantee::Ordered(Some(1)), + &mut connection, + Err(TryRecvError::Empty), + 3, + ); + + assert_incoming_with_order( + DeliveryGuarantee::Reliable, + OrderingGuarantee::Ordered(Some(1)), + &mut connection, + Ok(SocketEvent::Packet(Packet::reliable_ordered( + get_fake_addr(), + PAYLOAD.to_vec(), + Some(1), + ))), + 1, + ); + } + + #[test] + fn assure_correct_processing_of_incoming() { + let mut connection = create_virtual_connection(); + + assert_incoming_without_order( + DeliveryGuarantee::Unreliable, + &mut connection, + SocketEvent::Packet(Packet::unreliable(get_fake_addr(), PAYLOAD.to_vec())), + ); + + assert_incoming_without_order( + DeliveryGuarantee::Reliable, + &mut connection, + SocketEvent::Packet(Packet::reliable_unordered( + get_fake_addr(), + PAYLOAD.to_vec(), + )), + ); + + assert_incoming_with_order( + DeliveryGuarantee::Unreliable, + OrderingGuarantee::Sequenced(Some(1)), + &mut connection, + Ok(SocketEvent::Packet(Packet::unreliable_sequenced( + get_fake_addr(), + PAYLOAD.to_vec(), + Some(1), + ))), + 1, + ); + + assert_incoming_with_order( + DeliveryGuarantee::Reliable, + OrderingGuarantee::Ordered(Some(1)), + &mut connection, + Ok(SocketEvent::Packet(Packet::reliable_ordered( + get_fake_addr(), + PAYLOAD.to_vec(), + Some(1), + ))), + 0, + ); + } + + #[test] + fn assure_right_header_size() { + assert_right_header_size( + DeliveryGuarantee::Unreliable, + OrderingGuarantee::None, + (constants::STANDARD_HEADER_SIZE) as usize, + ); + assert_right_header_size( + DeliveryGuarantee::Unreliable, + OrderingGuarantee::Sequenced(None), + (constants::STANDARD_HEADER_SIZE + constants::ARRANGING_PACKET_HEADER) as usize, + ); + assert_right_header_size( + DeliveryGuarantee::Reliable, + OrderingGuarantee::None, + (constants::STANDARD_HEADER_SIZE + constants::ACKED_PACKET_HEADER) as usize, + ); + assert_right_header_size( + DeliveryGuarantee::Reliable, + OrderingGuarantee::Ordered(None), + (constants::STANDARD_HEADER_SIZE + + constants::ACKED_PACKET_HEADER + + constants::ARRANGING_PACKET_HEADER) as usize, + ); + } + + /// ======= helper functions ========= + fn create_virtual_connection() -> VirtualConnection { + VirtualConnection::new(get_fake_addr(), &Config::default(), Instant::now()) + } + + fn get_fake_addr() -> std::net::SocketAddr { + "127.0.0.1:0".parse().unwrap() + } + + // assert that the processing of the given `DeliveryGuarantee` and `OrderingGuarantee` results into the given `result_event` + fn assert_incoming_with_order( + delivery: DeliveryGuarantee, + ordering: OrderingGuarantee, + connection: &mut VirtualConnection, + result_event: Result, + order_id: u16, + ) { + let mut packet = Vec::new(); + + // configure the right header based on specified guarantees. + let header = StandardHeader::new(delivery, ordering, PacketType::Packet); + header.parse(&mut packet).unwrap(); + + if let OrderingGuarantee::Sequenced(val) = ordering { + if delivery == DeliveryGuarantee::Reliable { + let ack_header = AckedPacketHeader::new(1, 2, 3); + ack_header.parse(&mut packet).unwrap(); + } + + let order_header = ArrangingHeader::new(order_id, val.unwrap()); + order_header.parse(&mut packet).unwrap(); + } + + if let OrderingGuarantee::Ordered(val) = ordering { + if delivery == DeliveryGuarantee::Reliable { + let ack_header = AckedPacketHeader::new(1, 2, 3); + let order_header = ArrangingHeader::new(order_id, val.unwrap()); + ack_header.parse(&mut packet).unwrap(); + order_header.parse(&mut packet).unwrap(); + } + } + + if let OrderingGuarantee::None = ordering { + if delivery == DeliveryGuarantee::Reliable { + let ack_header = AckedPacketHeader::new(1, 2, 3); + ack_header.parse(&mut packet).unwrap(); + } + } + + packet.write_all(&PAYLOAD).unwrap(); + + let (tx, rx) = unbounded::(); + + connection + .process_incoming(packet.as_slice(), &tx, Instant::now()) + .unwrap(); + + let event = rx.try_recv(); + + match event { + Ok(val) => assert_eq!(val, result_event.unwrap()), + Err(e) => assert_eq!(e, result_event.err().unwrap()), + } + } + + // assert that the given `DeliveryGuarantee` results into the given `SocketEvent` after processing. + fn assert_incoming_without_order( + delivery: DeliveryGuarantee, + connection: &mut VirtualConnection, + result_event: SocketEvent, + ) { + let mut packet = Vec::new(); + + // configure the right header based on specified guarantees. + let header = StandardHeader::new(delivery, OrderingGuarantee::None, PacketType::Packet); + header.parse(&mut packet).unwrap(); + + if delivery == DeliveryGuarantee::Reliable { + let ack_header = AckedPacketHeader::new(1, 2, 3); + ack_header.parse(&mut packet).unwrap(); + } + + packet.write_all(&PAYLOAD).unwrap(); + + let (tx, rx) = unbounded::(); + + connection + .process_incoming(packet.as_slice(), &tx, Instant::now()) + .unwrap(); + + let event = rx.try_recv(); + + assert_eq!(event, Ok(result_event)); + } + + // assert that the size of the processed header is the same as the given one. + fn assert_right_header_size( + delivery: DeliveryGuarantee, + ordering: OrderingGuarantee, + expected_header_size: usize, + ) { + let mut connection = create_virtual_connection(); + + let buffer = vec![1; 500]; + + let outgoing = connection + .process_outgoing(&buffer, delivery, ordering, None, Instant::now()) + .unwrap(); + + match outgoing { + Outgoing::Packet(packet) => { + assert_eq!(packet.contents().len() - buffer.len(), expected_header_size); + } + Outgoing::Fragments(_) => panic!("Expected packet got fragment"), + } + } +} diff --git a/src/net/socket.rs b/src/net/socket.rs index 37867771..2e312429 100644 --- a/src/net/socket.rs +++ b/src/net/socket.rs @@ -1,14 +1,15 @@ use crate::{ config::Config, - either::Either, error::{ErrorKind, Result}, - net::events::{ConnectionEvent, DestroyReason, ReceiveEvent, SendEvent}, - net::managers::{ConnectionManager, ConnectionState, SocketManager}, - net::{connection::ActiveConnections, link_conditioner::LinkConditioner}, - packet::{DeliveryGuarantee, Outgoing, Packet, PacketType}, + net::events::{ConnectionEvent, ReceiveEvent, SendEvent}, + net::managers::{ConnectionManager, ConnectionManagerFactory, ConnectionState}, + net::MetricsCollector, + net::{ + connection::ActiveConnections, link_conditioner::LinkConditioner, ReliabilitySystem, + VirtualConnection, + }, }; -use crossbeam_channel::{self, unbounded, Receiver, SendError, Sender, TryRecvError}; -use log::error; +use crossbeam_channel::{self, unbounded, Receiver, Sender, TryRecvError}; use std::{ self, io, net::{Ipv4Addr, SocketAddr, SocketAddrV4, ToSocketAddrs, UdpSocket}, @@ -19,7 +20,7 @@ use std::{ // just wrap whese too together, for easier passing around #[derive(Debug)] pub struct SocketWithConditioner { - buffer: Vec, // this buffer is used for postprocess_outgoing + postprocess_buffer: Vec, // this buffer is used for postprocess_outgoing socket: UdpSocket, link_conditioner: Option, } @@ -28,79 +29,33 @@ impl SocketWithConditioner { /// Send a single packet over the UDP socket. /// Postprocess it using Connectionmanager`s postprocess_outgoing method, /// and use link condition if exists, to simulate network conditions - pub fn send_packet( + pub fn send_packet_and_log( &mut self, addr: &SocketAddr, - manager: &mut dyn ConnectionManager, + conn_man: &mut dyn ConnectionManager, payload: &[u8], - ) -> Result { - let payload = manager.postprocess_outgoing(payload, self.buffer.as_mut_slice()); - if let Some(ref mut link) = self.link_conditioner { - if !link.should_send() { - return Ok(0); - } + metrics: &mut MetricsCollector, + error_context: &str, + ) { + match self.send_packet(addr, conn_man, payload) { + Ok(bytes) => metrics.track_sent_bytes(addr, bytes), + Err(ref err) => metrics.track_connection_error(addr, err, error_context), } - Ok(self.socket.send_to(payload, addr)?) } - pub fn send_packet_and_log( + fn send_packet( &mut self, addr: &SocketAddr, - conn_man: &mut dyn ConnectionManager, + manager: &mut dyn ConnectionManager, payload: &[u8], - sock_man: &mut dyn SocketManager, - error_context: &str, - ) -> usize { - match self.send_packet(addr, conn_man, payload) { - Ok(bytes) => { - sock_man.track_sent_bytes(addr, bytes); - bytes - } - Err(ref err) => { - sock_man.track_connection_error(addr, err, error_context); - 0 + ) -> Result { + let payload = manager.postprocess_outgoing(payload, self.postprocess_buffer.as_mut_slice()); + if let Some(ref mut link) = self.link_conditioner { + if !link.should_send() { + return Ok(0); } } - } -} - -/// Wraps crossbeam_channel sender with convenient functions: `connect`, `send`, `disconnect` -pub struct SocketEventSender(pub Sender>); - -impl SocketEventSender { - /// Constructs ConnectionEvent Packet event from provided packet - pub fn send(&self, packet: Packet) -> std::result::Result<(), SendError> { - self.0 - .send(ConnectionEvent(packet.addr(), SendEvent::Packet(packet))) - .map_err(|err| { - SendError(match (err.0).1 { - SendEvent::Packet(data) => data, - _ => unreachable!(), - }) - }) - } - - /// Constructs ConnectionEvent Connect event from provided payload - pub fn connect( - &self, - addr: SocketAddr, - payload: Box<[u8]>, - ) -> std::result::Result<(), SendError>> { - self.0 - .send(ConnectionEvent(addr, SendEvent::Connect(payload))) - .map_err(|err| { - SendError(match (err.0).1 { - SendEvent::Connect(data) => data, - _ => unreachable!(), - }) - }) - } - - /// Constructs ConnectionEvent Disconnect event - pub fn disconnect(&self, addr: SocketAddr) -> std::result::Result<(), SendError<()>> { - self.0 - .send(ConnectionEvent(addr, SendEvent::Disconnect)) - .map_err(|_| SendError(())) + Ok(self.socket.send_to(payload, addr)?) } } @@ -111,14 +66,15 @@ pub struct Socket { config: Config, connections: ActiveConnections, recv_buffer: Vec, - tmp_buffer: Vec, // this is temporary buffer, mostly used by connection manager, in cases where it needs to modify incomming/outgoing raw bytes, or create new packets for sending + preprocess_buffer: Vec, // this is temporary buffer, used by connection manager, in cases where it needs to modify incomming raw bytes, or create new packets for sending event_sender: Sender>, packet_receiver: Receiver>, receiver: Receiver>, sender: Sender>, - manager: Box, + metrics: MetricsCollector, + conn_manager_factory: Box, } enum UdpSocketState { @@ -130,21 +86,27 @@ impl Socket { /// Binds to the socket and then sets up `ActiveConnections` to manage the "connections". /// Because UDP connections are not persistent, we can only infer the status of the remote /// endpoint by looking to see if they are still sending packets or not - pub fn bind(addresses: A, manager: Box) -> Result { - Socket::bind_with_config(addresses, Config::default(), manager) + pub fn bind( + addresses: A, + factory: Box, + ) -> Result { + Socket::bind_with_config(addresses, Config::default(), factory) } /// Bind to any local port on the system, if available - pub fn bind_any(manager: Box) -> Result { - Self::bind_any_with_config(Config::default(), manager) + pub fn bind_any(factory: Box) -> Result { + Self::bind_any_with_config(Config::default(), factory) } /// Bind to any local port on the system, if available, with a given config - pub fn bind_any_with_config(config: Config, manager: Box) -> Result { + pub fn bind_any_with_config( + config: Config, + factory: Box, + ) -> Result { let loopback = Ipv4Addr::new(127, 0, 0, 1); let address = SocketAddrV4::new(loopback, 0); let socket = UdpSocket::bind(address)?; - Self::bind_internal(socket, config, manager) + Self::bind_internal(socket, config, factory) } /// Binds to the socket and then sets up `ActiveConnections` to manage the "connections". @@ -155,25 +117,25 @@ impl Socket { pub fn bind_with_config( addresses: A, config: Config, - manager: Box, + factory: Box, ) -> Result { let socket = UdpSocket::bind(addresses)?; - Self::bind_internal(socket, config, manager) + Self::bind_internal(socket, config, factory) } fn bind_internal( socket: UdpSocket, config: Config, - manager: Box, + factory: Box, ) -> Result { socket.set_nonblocking(!config.blocking_mode)?; let (event_sender, event_receiver) = unbounded(); let (packet_sender, packet_receiver) = unbounded(); Ok(Socket { recv_buffer: vec![0; config.receive_buffer_max_size], - tmp_buffer: vec![0; config.receive_buffer_max_size], + preprocess_buffer: vec![0; config.receive_buffer_max_size], socket: SocketWithConditioner { - buffer: Vec::with_capacity(config.receive_buffer_max_size), + postprocess_buffer: Vec::with_capacity(config.receive_buffer_max_size), socket, link_conditioner: None, }, @@ -184,8 +146,8 @@ impl Socket { sender: packet_sender, receiver: event_receiver, - - manager, + metrics: MetricsCollector {}, + conn_manager_factory: factory, }) } @@ -207,7 +169,7 @@ impl Socket { pub fn send(&mut self, event: ConnectionEvent) -> Result<()> { match self.sender.send(event) { Ok(_) => Ok(()), - Err(error) => Err(ErrorKind::SendError(SendError(Either::Left(error.0)))), + Err(error) => Err(ErrorKind::from(error)), } } @@ -247,56 +209,40 @@ impl Socket { Ok(UdpSocketState::MaybeMore) => continue, Ok(UdpSocketState::MaybeEmpty) => break, Err(ref e) => self - .manager + .metrics .track_global_error(e, "receiving data from socket"), } } // Now grab all the packets waiting to be sent and send them while let Ok(p) = self.packet_receiver.try_recv() { - if let Err(ref e) = self.send_to(p, time) { - self.manager - .track_global_error(e, "sending data from socket") - } + self.send_to(p, time); } self.connections.update_connections( &self.event_sender, - self.manager.as_mut(), + &mut self.metrics, &mut self.socket, time, - self.tmp_buffer.as_mut(), + &mut self.preprocess_buffer, ); - // get connections that socket manager decided to destroy - if let Some(list) = self.manager.collect_connections_to_destroy() { - for (addr, reason) in list { - self.connections.remove_connection( - &addr, - &self.event_sender, - self.manager.as_mut(), - reason, - "destroyed by socket manager", - ); - } - } - - // Check for idle clients - self.handle_idle_clients(time); - - // Handle any dead clients - self.handle_dead_clients(); + // Check for all dead connections + self.connections.handle_dead_clients( + time, + self.config.idle_connection_timeout, + &self.event_sender, + &mut self.metrics, + ); // Finally send heartbeat packets to connections that require them, if enabled if let Some(heartbeat_interval) = self.config.heartbeat_interval { - // Iterate over all connections which have not sent a packet for a duration of at least - // `heartbeat_interval` (from config), and send a heartbeat packet to each. - self.connections.heartbeat_required_connections( - heartbeat_interval, + self.connections.handle_heartbeat( time, - self.manager.as_mut(), + heartbeat_interval, &mut self.socket, - ); + &mut self.metrics, + ) } } @@ -310,128 +256,40 @@ impl Socket { Ok(self.socket.socket.local_addr()?) } - /// Iterate through the dead connections and disconnect them by removing them from the - /// connection map while informing the user of this by sending an event. - fn handle_dead_clients(&mut self) { - let dead_addresses = self.connections.dead_connections(); - for (address, reason) in dead_addresses { - self.connections.remove_connection( - &address, - &self.event_sender, - self.manager.as_mut(), - reason, - "removing dead clients", - ); - } - } - - /// Iterate through all of the idle connections based on `idle_connection_timeout` config and - /// remove them from the active connections. For each connection removed, we will send a - /// `SocketEvent::TimeOut` event to the `event_sender` channel. - fn handle_idle_clients(&mut self, time: Instant) { - let idle_addresses = self - .connections - .idle_connections(self.config.idle_connection_timeout, time); - for address in idle_addresses { - self.connections.remove_connection( - &address, - &self.event_sender, - self.manager.as_mut(), - DestroyReason::Timeout, - "removing idle clients", - ); - } - } - // Serializes and sends a `Packet` on the socket. On success, returns the number of bytes written. - fn send_to(&mut self, event: ConnectionEvent, time: Instant) -> Result { - let mut bytes_sent = 0; - + fn send_to(&mut self, event: ConnectionEvent, time: Instant) { let connection = if let Some(connection) = self.connections.try_get(&event.0) { Some(connection) - } else if let Some(conn_manager) = self.manager.accept_local_connection(&event.0) { - self.event_sender - .send(ConnectionEvent(event.0, ReceiveEvent::Created))?; - let conn = self.connections.get_or_insert_connection( - event.0, - &self.config, - time, - conn_manager, - ); - ActiveConnections::update_connection_manager( - conn, - &self.event_sender, - self.manager.as_mut(), + } else { + let conn = self.connections.insert_and_init_connection( + VirtualConnection::new( + event.0, + ReliabilitySystem::new(&self.config, time), + self.conn_manager_factory + .create_local_connection_manager(&event.0), + ), &mut self.socket, + &self.event_sender, + &mut self.metrics, time, - self.tmp_buffer.as_mut(), + &mut self.preprocess_buffer, ); Some(conn) - } else { - None }; + if let Some(connection) = connection { - match (event.1, &connection.current_state) { + match (event.1, connection.get_current_state()) { (SendEvent::Packet(packet), ConnectionState::Connected(_)) => { // TODO maybe these should not depend on send_to method? // Maybe it should be extracted to separate method and added to `manual_poll` function. - let dropped = connection.gather_dropped_packets(); - let mut processed_packets: Vec = dropped - .iter() - .flat_map(|waiting_packet| { - connection.process_outgoing( - waiting_packet.packet_type, - &waiting_packet.payload, - // Because a delivery guarantee is only sent with reliable packets - DeliveryGuarantee::Reliable, - // This is stored with the dropped packet because they could be mixed - waiting_packet.ordering_guarantee, - waiting_packet.item_identifier, - time, - ) - }) - .collect(); - - let processed_packet = connection.process_outgoing( - PacketType::Packet, - packet.payload(), - packet.delivery_guarantee(), - packet.order_guarantee(), - None, - time, - )?; - processed_packets.push(processed_packet); - - for processed_packet in processed_packets { - match processed_packet { - Outgoing::Packet(outgoing) => { - bytes_sent += self.socket.send_packet( - &event.0, - connection.state_manager.as_mut(), - &outgoing.contents(), - )?; - } - Outgoing::Fragments(packets) => { - for outgoing in packets { - bytes_sent += self.socket.send_packet( - &event.0, - connection.state_manager.as_mut(), - &outgoing.contents(), - )?; - } - } - } - } - } - (SendEvent::Connect(data), ConnectionState::Connecting) => { - connection.state_manager.connect(data) + connection.resend_dropped_packets(time, &mut self.socket, &mut self.metrics); + connection.process_outgoing(&packet, time, &mut self.socket, &mut self.metrics); } - (SendEvent::Disconnect, _) => connection.state_manager.disconnect(), + (SendEvent::Connect(data), ConnectionState::Connecting) => connection.connect(data), + (SendEvent::Disconnect, _) => connection.disconnect(), _ => {} // ignore all other combinations }; } - - Ok(bytes_sent) } // On success the packet will be sent on the `event_sender` @@ -445,46 +303,38 @@ impl Socket { let connection = if let Some(conn) = self.connections.try_get(&address) { Some(conn) - } else if let Some(manager) = self - .manager - .accept_remote_connection(&address, received_payload) - { - self.event_sender - .send(ConnectionEvent(address, ReceiveEvent::Created))?; - let conn = self.connections.get_or_insert_connection( - address, - &self.config, - time, - manager, - ); - ActiveConnections::update_connection_manager( - conn, - &self.event_sender, - self.manager.as_mut(), + } else { + let conn = self.connections.insert_and_init_connection( + VirtualConnection::new( + address, + ReliabilitySystem::new(&self.config, time), + self.conn_manager_factory + .create_remote_connection_manager(&address, received_payload), + ), &mut self.socket, + &self.event_sender, + &mut self.metrics, time, - self.tmp_buffer.as_mut(), + &mut self.preprocess_buffer, ); Some(conn) - } else { - None }; if let Some(conn) = connection { - let received_payload = conn - .state_manager - .preprocess_incoming(received_payload, &mut self.tmp_buffer)?; + self.metrics.track_received_bytes(&address, recv_len); conn.process_incoming( received_payload, - &self.event_sender, - self.manager.as_mut(), + &mut self.preprocess_buffer, time, + &self.event_sender, + &mut self.metrics, )?; + } else { + self.metrics.track_ignored_bytes(&address, recv_len); } } Err(e) => { if e.kind() != io::ErrorKind::WouldBlock { - error!("Encountered an error receiving data: {:?}", e); return Err(e.into()); } else { return Ok(UdpSocketState::MaybeEmpty); diff --git a/src/net/virtual_connection.rs b/src/net/virtual_connection.rs index 131433bd..da518be3 100644 --- a/src/net/virtual_connection.rs +++ b/src/net/virtual_connection.rs @@ -1,20 +1,12 @@ use crate::{ - config::Config, - error::{ErrorKind, PacketErrorKind, Result}, - infrastructure::{ - arranging::{Arranging, ArrangingSystem, OrderingSystem, SequencingSystem}, - AcknowledgmentHandler, CongestionHandler, Fragmentation, SentPacket, - }, - net::constants::{ - ACKED_PACKET_HEADER, DEFAULT_ORDERING_STREAM, DEFAULT_SEQUENCING_STREAM, - STANDARD_HEADER_SIZE, - }, - net::events::{ConnectionEvent, ReceiveEvent}, - net::managers::{ConnectionManager, ConnectionState, SocketManager}, - packet::{ - DeliveryGuarantee, OrderingGuarantee, Outgoing, OutgoingPacket, OutgoingPacketBuilder, - Packet, PacketReader, PacketType, SequenceNumber, + error::{ErrorKind, Result}, + net::events::{ConnectionEvent, DestroyReason, DisconnectReason, ReceiveEvent}, + net::managers::{ + ConnectionManager, ConnectionManagerError, ConnectionManagerEvent, ConnectionState, }, + net::{MetricsCollector, SocketWithConditioner}, + net::{OutgoingPackets, ReliabilitySystem}, + packet::{GenericPacket, Packet, PacketType}, }; use crossbeam_channel::{self, Sender}; @@ -23,474 +15,257 @@ use std::net::SocketAddr; use std::time::{Duration, Instant}; /// Contains the information about a certain 'virtual connection' over udp. -/// This connections also keeps track of network quality, processing packets, buffering data related to connection etc. +/// This connections has core components that manages connection state, and has reliability information. pub struct VirtualConnection { - /// Last time we received a packet from this client - pub last_heard: Instant, - /// Last time we sent a packet to this client - pub last_sent: Instant, - /// The address of the remote endpoint - pub remote_address: SocketAddr, - - ordering_system: OrderingSystem>, - sequencing_system: SequencingSystem>, - acknowledge_handler: AcknowledgmentHandler, - congestion_handler: CongestionHandler, - - config: Config, - fragmentation: Fragmentation, - pub state_manager: Box, - pub current_state: ConnectionState, + remote_address: SocketAddr, + reliability_system: ReliabilitySystem, + connection_manager: Box, + current_state: ConnectionState, } impl VirtualConnection { - /// Creates and returns a new `VirtualConnection` that wraps the provided socket address + /// Creates new VirtualConnection. pub fn new( - addr: SocketAddr, - config: &Config, - time: Instant, - state_manager: Box, + remote_address: SocketAddr, + reliability_system: ReliabilitySystem, + connection_manager: Box, ) -> VirtualConnection { - VirtualConnection { - last_heard: time, - last_sent: time, - remote_address: addr, - ordering_system: OrderingSystem::new(), - sequencing_system: SequencingSystem::new(), - acknowledge_handler: AcknowledgmentHandler::new(), - congestion_handler: CongestionHandler::new(config), - fragmentation: Fragmentation::new(config), - config: config.to_owned(), - state_manager, - current_state: ConnectionState::Connecting, + Self { + remote_address, + reliability_system, + connection_manager, + current_state: Default::default(), } } - /// Determine if this connection should be dropped due to its state - pub fn should_be_dropped(&self) -> bool { - self.acknowledge_handler.packets_in_flight() > self.config.max_packets_in_flight - } - - pub fn is_disconnected(&self) -> bool { - if let ConnectionState::Disconnected(_) = self.current_state { - true - } else { - false - } + /// Returns connection remote address. + pub fn remote_address(&self) -> SocketAddr { + self.remote_address } - /// Returns a [Duration] representing the interval since we last heard from the client - pub fn last_heard(&self, time: Instant) -> Duration { - // TODO: Replace with saturating_duration_since once it becomes stable. - // This function panics if the user supplies a time instant earlier than last_heard. - time.duration_since(self.last_heard) + /// Invokes connect request on ConnectionManager. + pub fn connect(&mut self, payload: Box<[u8]>) { + self.connection_manager.connect(payload); } - /// Returns a [Duration] representing the interval since we last sent to the client - pub fn last_sent(&self, time: Instant) -> Duration { - // TODO: Replace with saturating_duration_since once it becomes stable. - // This function panics if the user supplies a time instant earlier than last_heard. - time.duration_since(self.last_sent) + /// Invokes disconnect request on ConnectionManager. + pub fn disconnect(&mut self) { + self.connection_manager.disconnect(); } - // Returns current connection state: `Connecting`, `Connected` or `Disconnected` + /// Returns current connection state: `Connecting`, `Connected` or `Disconnected`. pub fn get_current_state(&self) -> &ConnectionState { &self.current_state } - /// This will create a heartbeat packet that is expected to be sent over the network - pub fn create_and_process_heartbeat(&mut self, time: Instant) -> OutgoingPacket<'static> { - self.last_sent = time; - self.congestion_handler - .process_outgoing(self.acknowledge_handler.local_sequence_num(), time); - - OutgoingPacketBuilder::new(&[]) - .with_default_header( - PacketType::Heartbeat, - DeliveryGuarantee::Unreliable, - OrderingGuarantee::None, - ) - .build() + /// Returns whether connection should be dropped, and provides a drop reason. + pub fn should_be_dropped( + &self, + max_idle_time: Duration, + time: Instant, + ) -> Option { + if self.reliability_system.should_be_dropped() { + Some(DestroyReason::TooManyPacketsInFlight) + } else if let ConnectionState::Disconnected(_) = self.current_state { + Some(DestroyReason::GracefullyDisconnected) + } else if self.reliability_system.last_heard(time) >= max_idle_time { + Some(DestroyReason::Timeout) + } else { + None + } } - /// This will pre-process the given buffer to be sent over the network. - pub fn process_outgoing<'a>( + /// Checks if connection should send heartbeat packet and sends it. + pub fn handle_heartbeat( &mut self, - // TODO (fix somehow?) if this packet is user (Packet), and is large enough that it needs to be fragmented, then this will be changed to Fragment, when building actual packet. - packet_type: PacketType, - payload: &'a [u8], - delivery_guarantee: DeliveryGuarantee, - ordering_guarantee: OrderingGuarantee, - last_item_identifier: Option, time: Instant, - ) -> Result> { - match delivery_guarantee { - DeliveryGuarantee::Unreliable => { - if payload.len() <= self.config.receive_buffer_max_size { - let mut builder = OutgoingPacketBuilder::new(payload).with_default_header( - packet_type, - delivery_guarantee, - ordering_guarantee, - ); - - if let OrderingGuarantee::Sequenced(stream_id) = ordering_guarantee { - let item_identifier = self - .sequencing_system - .get_or_create_stream(stream_id.unwrap_or(DEFAULT_SEQUENCING_STREAM)) - .new_item_identifier(); - - builder = builder.with_sequencing_header(item_identifier as u16, stream_id); - }; - - Ok(Outgoing::Packet(builder.build())) - } else { - Err(ErrorKind::PacketError( - PacketErrorKind::ExceededMaxPacketSize, - )) - } - } - DeliveryGuarantee::Reliable => { - let payload_length = payload.len() as u16; - - let mut item_identifier_value = None; - let outgoing = { - // spit the packet if the payload length is greater than the allowed fragment size. - if payload_length <= self.config.fragment_size { - let mut builder = OutgoingPacketBuilder::new(payload).with_default_header( - packet_type, - delivery_guarantee, - ordering_guarantee, - ); - - builder = builder.with_acknowledgment_header( - self.acknowledge_handler.local_sequence_num(), - self.acknowledge_handler.remote_sequence_num(), - self.acknowledge_handler.ack_bitfield(), - ); - - if let OrderingGuarantee::Ordered(stream_id) = ordering_guarantee { - let item_identifier = - if let Some(item_identifier) = last_item_identifier { - item_identifier - } else { - self.ordering_system - .get_or_create_stream( - stream_id.unwrap_or(DEFAULT_ORDERING_STREAM), - ) - .new_item_identifier() - }; - - item_identifier_value = Some(item_identifier); - - builder = builder.with_ordering_header(item_identifier, stream_id); - }; - - if let OrderingGuarantee::Sequenced(stream_id) = ordering_guarantee { - let item_identifier = - if let Some(item_identifier) = last_item_identifier { - item_identifier - } else { - self.sequencing_system - .get_or_create_stream( - stream_id.unwrap_or(DEFAULT_SEQUENCING_STREAM), - ) - .new_item_identifier() - }; - - item_identifier_value = Some(item_identifier); - - builder = builder.with_sequencing_header(item_identifier, stream_id); - }; - - Outgoing::Packet(builder.build()) - } else { - if packet_type != PacketType::Packet { - return Err(ErrorKind::PacketError( - PacketErrorKind::PacketTypeCannotBeFragmented, - )); - } - Outgoing::Fragments( - Fragmentation::spit_into_fragments(payload, &self.config)? - .into_iter() - .enumerate() - .map(|(fragment_id, fragment)| { - let fragments_needed = Fragmentation::fragments_needed( - payload_length, - self.config.fragment_size, - ) - as u8; - - let mut builder = OutgoingPacketBuilder::new(fragment) - .with_default_header( - PacketType::Fragment, - delivery_guarantee, - ordering_guarantee, - ); - - builder = builder.with_fragment_header( - self.acknowledge_handler.local_sequence_num(), - fragment_id as u8, - fragments_needed, - ); - - if fragment_id == 0 { - builder = builder.with_acknowledgment_header( - self.acknowledge_handler.local_sequence_num(), - self.acknowledge_handler.remote_sequence_num(), - self.acknowledge_handler.ack_bitfield(), - ); - } - - builder.build() - }) - .collect(), - ) - } - }; - - self.last_sent = time; - self.congestion_handler - .process_outgoing(self.acknowledge_handler.local_sequence_num(), time); - self.acknowledge_handler.process_outgoing( - packet_type, - payload, - ordering_guarantee, - item_identifier_value, - ); - - Ok(outgoing) - } + heartbeat_interval: Duration, + socket: &mut SocketWithConditioner, + metrics: &mut MetricsCollector, + ) { + if self.reliability_system.last_sent(time) >= heartbeat_interval { + send_packets( + &self.remote_address, + self.reliability_system + .process_outgoing(GenericPacket::heartbeat_packet(&[]), time), + self.connection_manager.as_mut(), + socket, + metrics, + "sending heartbeat packet", + ); } } - // TODO would be super nice to return iterator of new packets, instead of passing `sender`, and `socket_manager` inside. - // maybe callback: Fn(packet) would also be enough? - pub fn process_incoming( + /// Resends all dropped packets. + pub fn resend_dropped_packets( &mut self, - received_data: &[u8], - sender: &Sender>, - socket_manager: &mut dyn SocketManager, time: Instant, - ) -> crate::Result<()> { - self.last_heard = time; - - let mut packet_reader = PacketReader::new(received_data); - - let header = packet_reader.read_standard_header()?; - - if !header.is_current_protocol() { - return Err(ErrorKind::ProtocolVersionMismatch); - } - - if header.is_heartbeat() { - // Heartbeat packets are unreliable, unordered and empty packets. - // We already updated our `self.last_heard` time, nothing else to be done. - return Ok(()); + socket: &mut SocketWithConditioner, + metrics: &mut MetricsCollector, + ) { + for dropped in self.reliability_system.gather_dropped_packets() { + send_packets( + &self.remote_address, + self.reliability_system.process_dropped(&dropped, time), + self.connection_manager.as_mut(), + socket, + metrics, + "sending dropped packet", + ); } + } - match header.delivery_guarantee() { - DeliveryGuarantee::Unreliable => { - if let OrderingGuarantee::Sequenced(_id) = header.ordering_guarantee() { - let arranging_header = - packet_reader.read_arranging_header(u16::from(STANDARD_HEADER_SIZE))?; - - let payload = packet_reader.read_payload(); + /// Processes outgoing packet, and sends it. + pub fn process_outgoing( + &mut self, + packet: &Packet, + time: Instant, + socket: &mut SocketWithConditioner, + metrics: &mut MetricsCollector, + ) { + let packets = self.reliability_system.process_outgoing( + GenericPacket { + packet_type: PacketType::Packet, + payload: packet.payload(), + delivery: packet.delivery_guarantee(), + ordering: packet.order_guarantee(), + }, + time, + ); - let stream = self - .sequencing_system - .get_or_create_stream(arranging_header.stream_id()); + send_packets( + &self.remote_address, + packets, + self.connection_manager.as_mut(), + socket, + metrics, + "sending outgoing packet", + ); + } - if let Some(packet) = stream.arrange(arranging_header.arranging_id(), payload) { - self.process_packet_and_log_errors( - header.is_connection_manager_packet(), - sender, - socket_manager, - packet, - header.delivery_guarantee(), - OrderingGuarantee::Sequenced(Some(arranging_header.stream_id())), + /// Processes incoming bytes by converting to packets and process them. + pub fn process_incoming( + &mut self, + received_payload: &[u8], + mut tmp_buffer: &mut [u8], + time: Instant, + event_sender: &Sender>, + metrics: &mut MetricsCollector, + ) -> Result<()> { + let received_payload = self + .connection_manager + .preprocess_incoming(received_payload, &mut tmp_buffer)?; + let packets = self.reliability_system.process_incoming( + self.remote_address, + received_payload, + time, + )?; + for (packet, packet_type) in packets { + if packet_type != PacketType::Connection { + if let ConnectionState::Connected(_) = self.current_state { + if let Err(err) = event_sender.send(ConnectionEvent( + self.remote_address, + ReceiveEvent::Packet(packet), + )) { + metrics.track_connection_error( + &self.remote_address, + &ErrorKind::from(err), + "sending incoming packet", ); } - - return Ok(()); } - self.process_packet_and_log_errors( - header.is_connection_manager_packet(), - sender, - socket_manager, - packet_reader.read_payload(), - header.delivery_guarantee(), - header.ordering_guarantee(), + } else if let Err(err) = self + .connection_manager + .process_protocol_data(packet.payload()) + { + metrics.track_connection_error( + &self.remote_address, + &ErrorKind::from(err), + "processing connection manager data", ); } - DeliveryGuarantee::Reliable => { - if header.is_fragment() { - if let Ok((fragment_header, acked_header)) = packet_reader.read_fragment() { - let payload = packet_reader.read_payload(); + } + Ok(()) + } - match self - .fragmentation - .handle_fragment(fragment_header, &payload) - { - Ok(Some(payload)) => { - self.process_packet_and_log_errors( - false, // connection manager packet cannot be fragmented - sender, - socket_manager, - payload.into_boxed_slice(), - header.delivery_guarantee(), - OrderingGuarantee::None, + /// Calls `update` method for `ConnectionManager`, in the loop, until it returns None + /// These updates returns either new packets to be sent, or connection state changes. + pub fn update_connection_manager( + &mut self, + sender: &Sender>, + metrics: &mut MetricsCollector, + socket: &mut SocketWithConditioner, + time: Instant, + buffer: &mut [u8], + ) { + while let Some(changes) = self.connection_manager.update(buffer, time) { + match changes { + ConnectionManagerEvent::NewPacket(packet) => { + send_packets( + &self.remote_address, + self.reliability_system.process_outgoing(packet, time), + self.connection_manager.as_mut(), + socket, + metrics, + "sending packet from connection manager", + ); + } + ConnectionManagerEvent::NewState(state) => { + if let Some(old) = self.current_state.try_change(&state) { + if let Err(err) = match &self.current_state { + ConnectionState::Connected(data) => sender.send(ConnectionEvent( + self.remote_address, + ReceiveEvent::Connected(data.clone()), + )), + ConnectionState::Disconnected(closed_by) => { + sender.send(ConnectionEvent( + self.remote_address, + ReceiveEvent::Disconnected(DisconnectReason::ClosedBy( + closed_by.clone(), + )), + )) + } + _ => { + metrics.track_connection_error( + &self.remote_address, + &ErrorKind::ConnectionError(ConnectionManagerError::Fatal( + format!( + "Invalid state transition: {:?} -> {:?}", + old, self.current_state + ), + )), + "changing connection manager state", ); + Ok(()) } - Ok(None) => return Ok(()), - Err(e) => return Err(e), - }; - - if let Some(acked_header) = acked_header { - self.congestion_handler - .process_incoming(acked_header.sequence()); - self.acknowledge_handler.process_incoming( - acked_header.sequence(), - acked_header.ack_seq(), - acked_header.ack_field(), - ); - } - } - } else { - let acked_header = packet_reader.read_acknowledge_header()?; - - if let OrderingGuarantee::Sequenced(_) = header.ordering_guarantee() { - let arranging_header = packet_reader.read_arranging_header(u16::from( - STANDARD_HEADER_SIZE + ACKED_PACKET_HEADER, - ))?; - - let payload = packet_reader.read_payload(); - - let stream = self - .sequencing_system - .get_or_create_stream(arranging_header.stream_id()); - - if let Some(packet) = - stream.arrange(arranging_header.arranging_id(), payload) - { - self.process_packet_and_log_errors( - header.is_connection_manager_packet(), - sender, - socket_manager, - packet, - header.delivery_guarantee(), - OrderingGuarantee::Sequenced(Some(arranging_header.stream_id())), - ); - } - } else if let OrderingGuarantee::Ordered(_id) = header.ordering_guarantee() { - let arranging_header = packet_reader.read_arranging_header(u16::from( - STANDARD_HEADER_SIZE + ACKED_PACKET_HEADER, - ))?; - - let payload = packet_reader.read_payload(); - - let stream = self - .ordering_system - .get_or_create_stream(arranging_header.stream_id()); - let arranged_packet = - stream.arrange(arranging_header.arranging_id(), payload); - let packets = arranged_packet - .into_iter() - .chain(stream.iter_mut()) - .collect::>(); - for packet in packets { - self.process_packet_and_log_errors( - header.is_connection_manager_packet(), - sender, - socket_manager, - packet, - header.delivery_guarantee(), - OrderingGuarantee::Ordered(Some(arranging_header.stream_id())), + } { + metrics.track_connection_error( + &self.remote_address, + &ErrorKind::from(err), + "sending connection state update", ); } } else { - let payload = packet_reader.read_payload(); - - self.process_packet_and_log_errors( - header.is_connection_manager_packet(), - sender, - socket_manager, - payload, - header.delivery_guarantee(), - header.ordering_guarantee(), + metrics.track_connection_error( + &self.remote_address, + &ErrorKind::ConnectionError(ConnectionManagerError::Fatal(format!( + "Invalid state transition: {:?} -> {:?}", + self.current_state, state + ))), + "changing connection manager state", ); } - - self.congestion_handler - .process_incoming(acked_header.sequence()); - self.acknowledge_handler.process_incoming( - acked_header.sequence(), - acked_header.ack_seq(), - acked_header.ack_field(), + } + ConnectionManagerEvent::Error(err) => { + metrics.track_connection_error( + &self.remote_address, + &ErrorKind::ConnectionError(err), + "recieved connection manager error", ); } - } - } - - Ok(()) - } - - /// Pass packet to ConnectionManager, check for state changes, and return any newly generated packets - fn process_packet( - &mut self, - is_connection_manager_packet: bool, - sender: &Sender>, - payload: Box<[u8]>, - delivery: DeliveryGuarantee, - ordering: OrderingGuarantee, - ) -> Result<()> { - if !is_connection_manager_packet { - if let ConnectionState::Connected(_) = self.current_state { - sender.send(ConnectionEvent( - self.remote_address, - ReceiveEvent::Packet(Packet::new( - self.remote_address, - payload, - delivery, - ordering, - )), - ))?; - } - } else { - self.state_manager.process_protocol_data(payload.as_ref())?; - } - Ok(()) - } - - fn process_packet_and_log_errors( - &mut self, - is_connection_manager_packet: bool, - sender: &Sender>, - socket_manager: &mut dyn SocketManager, - payload: Box<[u8]>, - delivery: DeliveryGuarantee, - ordering: OrderingGuarantee, - ) { - if let Err(ref error) = self.process_packet( - is_connection_manager_packet, - sender, - payload, - delivery, - ordering, - ) { - socket_manager.track_connection_error( - &self.remote_address, - error, - "processing incomming packet", - ); + }; } } - - /// This will gather dropped packets from the acknowledgment handler. - /// - /// Note that after requesting dropped packets the dropped packets will be removed from this client. - pub fn gather_dropped_packets(&mut self) -> Vec { - self.acknowledge_handler.dropped_packets() - } } impl fmt::Debug for VirtualConnection { @@ -504,465 +279,27 @@ impl fmt::Debug for VirtualConnection { } } -#[cfg(test)] -mod tests { - use super::VirtualConnection; - use crate::config::Config; - use crate::net::constants; - use crate::packet::header::{AckedPacketHeader, ArrangingHeader, HeaderWriter, StandardHeader}; - use crate::packet::{DeliveryGuarantee, OrderingGuarantee, Outgoing, Packet, PacketType}; - use crate::protocol_version::ProtocolVersion; - use crate::SocketEvent; - use byteorder::{BigEndian, WriteBytesExt}; - use crossbeam_channel::{unbounded, TryRecvError}; - use std::io::Write; - use std::time::Instant; - - const PAYLOAD: [u8; 4] = [1, 2, 3, 4]; - - #[test] - fn assure_right_fragmentation() { - let mut protocol_version = Vec::new(); - protocol_version - .write_u16::(ProtocolVersion::get_crc16()) - .unwrap(); - - let standard_header = [protocol_version, vec![1, 1, 2]].concat(); - - let acked_header = vec![1, 0, 0, 2, 0, 0, 0, 3]; - let first_fragment = vec![0, 1, 1, 3]; - let second_fragment = vec![0, 1, 2, 3]; - let third_fragment = vec![0, 1, 3, 3]; - - let (tx, rx) = unbounded::(); - - let mut connection = create_virtual_connection(); - connection - .process_incoming( - [standard_header.as_slice(), acked_header.as_slice()] - .concat() - .as_slice(), - &tx, - Instant::now(), - ) - .unwrap(); - assert!(rx.try_recv().is_err()); - connection - .process_incoming( - [ - standard_header.as_slice(), - first_fragment.as_slice(), - &PAYLOAD, - ] - .concat() - .as_slice(), - &tx, - Instant::now(), - ) - .unwrap(); - assert!(rx.try_recv().is_err()); - connection - .process_incoming( - [ - standard_header.as_slice(), - second_fragment.as_slice(), - &PAYLOAD, - ] - .concat() - .as_slice(), - &tx, - Instant::now(), - ) - .unwrap(); - assert!(rx.try_recv().is_err()); - connection - .process_incoming( - [ - standard_header.as_slice(), - third_fragment.as_slice(), - &PAYLOAD, - ] - .concat() - .as_slice(), - &tx, - Instant::now(), - ) - .unwrap(); - - let complete_fragment = rx.try_recv().unwrap(); - - match complete_fragment { - SocketEvent::Packet(fragment) => assert_eq!( - fragment.payload(), - &*[PAYLOAD, PAYLOAD, PAYLOAD].concat().into_boxed_slice() - ), - _ => { - panic!("Expected fragment other result."); - } - } - } - - #[test] - fn expect_fragmentation() { - let mut connection = create_virtual_connection(); - - let buffer = vec![1; 4000]; - - let outgoing = connection - .process_outgoing( - &buffer, - DeliveryGuarantee::Reliable, - OrderingGuarantee::Ordered(None), - None, - Instant::now(), - ) - .unwrap(); - - match outgoing { - Outgoing::Packet(_) => panic!("Expected fragment got packet"), - Outgoing::Fragments(fragments) => { - assert_eq!(fragments.len(), 4); - } - } - } - - #[test] - fn assure_correct_outgoing_processing() { - let mut connection = create_virtual_connection(); - - let buffer = vec![1; 1000]; - - connection - .process_outgoing( - &buffer, - DeliveryGuarantee::Unreliable, - OrderingGuarantee::None, - None, - Instant::now(), - ) - .unwrap(); - - connection - .process_outgoing( - &buffer, - DeliveryGuarantee::Unreliable, - OrderingGuarantee::Sequenced(None), - None, - Instant::now(), - ) - .unwrap(); - - connection - .process_outgoing( - &buffer, - DeliveryGuarantee::Reliable, - OrderingGuarantee::Ordered(None), - None, - Instant::now(), - ) - .unwrap(); - - connection - .process_outgoing( - &buffer, - DeliveryGuarantee::Reliable, - OrderingGuarantee::Sequenced(None), - None, - Instant::now(), - ) - .unwrap(); - } - - #[test] - fn assure_right_sequencing() { - let mut connection = create_virtual_connection(); - - assert_incoming_with_order( - DeliveryGuarantee::Unreliable, - OrderingGuarantee::Sequenced(Some(1)), - &mut connection, - Ok(SocketEvent::Packet(Packet::unreliable_sequenced( - get_fake_addr(), - PAYLOAD.to_vec(), - Some(1), - ))), - 1, - ); - - assert_incoming_with_order( - DeliveryGuarantee::Unreliable, - OrderingGuarantee::Sequenced(Some(1)), - &mut connection, - Ok(SocketEvent::Packet(Packet::unreliable_sequenced( - get_fake_addr(), - PAYLOAD.to_vec(), - Some(1), - ))), - 3, - ); - - assert_incoming_with_order( - DeliveryGuarantee::Unreliable, - OrderingGuarantee::Sequenced(Some(1)), - &mut connection, - Err(TryRecvError::Empty), - 2, - ); - - assert_incoming_with_order( - DeliveryGuarantee::Unreliable, - OrderingGuarantee::Sequenced(Some(1)), - &mut connection, - Ok(SocketEvent::Packet(Packet::unreliable_sequenced( - get_fake_addr(), - PAYLOAD.to_vec(), - Some(1), - ))), - 4, - ); - - assert_incoming_with_order( - DeliveryGuarantee::Reliable, - OrderingGuarantee::Sequenced(Some(1)), - &mut connection, - Ok(SocketEvent::Packet(Packet::reliable_sequenced( - get_fake_addr(), - PAYLOAD.to_vec(), - Some(1), - ))), - 5, - ); - } - - #[test] - fn assure_right_ordering() { - let mut connection = create_virtual_connection(); - - assert_incoming_with_order( - DeliveryGuarantee::Reliable, - OrderingGuarantee::Ordered(Some(1)), - &mut connection, - Ok(SocketEvent::Packet(Packet::reliable_ordered( - get_fake_addr(), - PAYLOAD.to_vec(), - Some(1), - ))), - 0, - ); - - assert_incoming_with_order( - DeliveryGuarantee::Reliable, - OrderingGuarantee::Ordered(Some(1)), - &mut connection, - Err(TryRecvError::Empty), - 2, - ); - - assert_incoming_with_order( - DeliveryGuarantee::Reliable, - OrderingGuarantee::Ordered(Some(1)), - &mut connection, - Err(TryRecvError::Empty), - 3, - ); - - assert_incoming_with_order( - DeliveryGuarantee::Reliable, - OrderingGuarantee::Ordered(Some(1)), - &mut connection, - Ok(SocketEvent::Packet(Packet::reliable_ordered( - get_fake_addr(), - PAYLOAD.to_vec(), - Some(1), - ))), - 1, - ); - } - - #[test] - fn assure_correct_processing_of_incoming() { - let mut connection = create_virtual_connection(); - - assert_incoming_without_order( - DeliveryGuarantee::Unreliable, - &mut connection, - SocketEvent::Packet(Packet::unreliable(get_fake_addr(), PAYLOAD.to_vec())), - ); - - assert_incoming_without_order( - DeliveryGuarantee::Reliable, - &mut connection, - SocketEvent::Packet(Packet::reliable_unordered( - get_fake_addr(), - PAYLOAD.to_vec(), - )), - ); - - assert_incoming_with_order( - DeliveryGuarantee::Unreliable, - OrderingGuarantee::Sequenced(Some(1)), - &mut connection, - Ok(SocketEvent::Packet(Packet::unreliable_sequenced( - get_fake_addr(), - PAYLOAD.to_vec(), - Some(1), - ))), - 1, - ); - - assert_incoming_with_order( - DeliveryGuarantee::Reliable, - OrderingGuarantee::Ordered(Some(1)), - &mut connection, - Ok(SocketEvent::Packet(Packet::reliable_ordered( - get_fake_addr(), - PAYLOAD.to_vec(), - Some(1), - ))), - 0, - ); - } - - #[test] - fn assure_right_header_size() { - assert_right_header_size( - DeliveryGuarantee::Unreliable, - OrderingGuarantee::None, - (constants::STANDARD_HEADER_SIZE) as usize, - ); - assert_right_header_size( - DeliveryGuarantee::Unreliable, - OrderingGuarantee::Sequenced(None), - (constants::STANDARD_HEADER_SIZE + constants::ARRANGING_PACKET_HEADER) as usize, - ); - assert_right_header_size( - DeliveryGuarantee::Reliable, - OrderingGuarantee::None, - (constants::STANDARD_HEADER_SIZE + constants::ACKED_PACKET_HEADER) as usize, - ); - assert_right_header_size( - DeliveryGuarantee::Reliable, - OrderingGuarantee::Ordered(None), - (constants::STANDARD_HEADER_SIZE - + constants::ACKED_PACKET_HEADER - + constants::ARRANGING_PACKET_HEADER) as usize, - ); - } - - /// ======= helper functions ========= - fn create_virtual_connection() -> VirtualConnection { - VirtualConnection::new(get_fake_addr(), &Config::default(), Instant::now()) - } - - fn get_fake_addr() -> std::net::SocketAddr { - "127.0.0.1:0".parse().unwrap() - } - - // assert that the processing of the given `DeliveryGuarantee` and `OrderingGuarantee` results into the given `result_event` - fn assert_incoming_with_order( - delivery: DeliveryGuarantee, - ordering: OrderingGuarantee, - connection: &mut VirtualConnection, - result_event: Result, - order_id: u16, - ) { - let mut packet = Vec::new(); - - // configure the right header based on specified guarantees. - let header = StandardHeader::new(delivery, ordering, PacketType::Packet); - header.parse(&mut packet).unwrap(); - - if let OrderingGuarantee::Sequenced(val) = ordering { - if delivery == DeliveryGuarantee::Reliable { - let ack_header = AckedPacketHeader::new(1, 2, 3); - ack_header.parse(&mut packet).unwrap(); - } - - let order_header = ArrangingHeader::new(order_id, val.unwrap()); - order_header.parse(&mut packet).unwrap(); - } - - if let OrderingGuarantee::Ordered(val) = ordering { - if delivery == DeliveryGuarantee::Reliable { - let ack_header = AckedPacketHeader::new(1, 2, 3); - let order_header = ArrangingHeader::new(order_id, val.unwrap()); - ack_header.parse(&mut packet).unwrap(); - order_header.parse(&mut packet).unwrap(); - } - } - - if let OrderingGuarantee::None = ordering { - if delivery == DeliveryGuarantee::Reliable { - let ack_header = AckedPacketHeader::new(1, 2, 3); - ack_header.parse(&mut packet).unwrap(); - } - } - - packet.write_all(&PAYLOAD).unwrap(); - - let (tx, rx) = unbounded::(); - - connection - .process_incoming(packet.as_slice(), &tx, Instant::now()) - .unwrap(); - - let event = rx.try_recv(); - - match event { - Ok(val) => assert_eq!(val, result_event.unwrap()), - Err(e) => assert_eq!(e, result_event.err().unwrap()), - } - } - - // assert that the given `DeliveryGuarantee` results into the given `SocketEvent` after processing. - fn assert_incoming_without_order( - delivery: DeliveryGuarantee, - connection: &mut VirtualConnection, - result_event: SocketEvent, - ) { - let mut packet = Vec::new(); - - // configure the right header based on specified guarantees. - let header = StandardHeader::new(delivery, OrderingGuarantee::None, PacketType::Packet); - header.parse(&mut packet).unwrap(); - - if delivery == DeliveryGuarantee::Reliable { - let ack_header = AckedPacketHeader::new(1, 2, 3); - ack_header.parse(&mut packet).unwrap(); - } - - packet.write_all(&PAYLOAD).unwrap(); - - let (tx, rx) = unbounded::(); - - connection - .process_incoming(packet.as_slice(), &tx, Instant::now()) - .unwrap(); - - let event = rx.try_recv(); - - assert_eq!(event, Ok(result_event)); - } - - // assert that the size of the processed header is the same as the given one. - fn assert_right_header_size( - delivery: DeliveryGuarantee, - ordering: OrderingGuarantee, - expected_header_size: usize, - ) { - let mut connection = create_virtual_connection(); - - let buffer = vec![1; 500]; - - let outgoing = connection - .process_outgoing(&buffer, delivery, ordering, None, Instant::now()) - .unwrap(); - - match outgoing { - Outgoing::Packet(packet) => { - assert_eq!(packet.contents().len() - buffer.len(), expected_header_size); +// Helper method, that takes outgoing packets from reliability system and sends them. +fn send_packets( + address: &SocketAddr, + packets_result: Result, + connection_manager: &mut dyn ConnectionManager, + socket: &mut SocketWithConditioner, + metrics: &mut MetricsCollector, + err_context: &str, +) { + match packets_result { + Ok(packets) => { + for outgoing in packets { + socket.send_packet_and_log( + address, + connection_manager, + &outgoing.contents(), + metrics, + err_context, + ); } - Outgoing::Fragments(_) => panic!("Expected packet got fragment"), } + Err(err) => metrics.track_connection_error(address, &err, err_context), } } diff --git a/src/packet.rs b/src/packet.rs index 0f3fc185..525b1a1e 100644 --- a/src/packet.rs +++ b/src/packet.rs @@ -8,7 +8,7 @@ mod packet_reader; mod packet_structure; pub use self::enums::{DeliveryGuarantee, OrderingGuarantee, PacketType}; -pub use self::outgoing::{Outgoing, OutgoingPacket, OutgoingPacketBuilder}; +pub use self::outgoing::{OutgoingPacket, OutgoingPacketBuilder}; pub use self::packet_reader::PacketReader; pub use self::packet_structure::{GenericPacket, Packet}; diff --git a/src/packet/enums.rs b/src/packet/enums.rs index 8c5c9266..8deda140 100644 --- a/src/packet/enums.rs +++ b/src/packet/enums.rs @@ -84,7 +84,7 @@ impl TryFrom for OrderingGuarantee { #[derive(Copy, Clone, Debug, PartialOrd, PartialEq)] /// Id to identify a certain packet type. pub enum PacketType { - /// User full packet that is not fragmented, + /// User full packet that is not fragmented Packet = 0, /// User fragment of a full packet Fragment = 1, diff --git a/src/packet/header/standard_header.rs b/src/packet/header/standard_header.rs index 5d42425b..85ec7934 100644 --- a/src/packet/header/standard_header.rs +++ b/src/packet/header/standard_header.rs @@ -48,26 +48,10 @@ impl StandardHeader { } /// Returns the PacketType - #[cfg(test)] pub fn packet_type(&self) -> PacketType { self.packet_type } - /// Returns true if the packet is a heartbeat packet, false otherwise - pub fn is_heartbeat(&self) -> bool { - self.packet_type == PacketType::Heartbeat - } - - /// Returns true if the packet is a fragment, false if not - pub fn is_fragment(&self) -> bool { - self.packet_type == PacketType::Fragment - } - - /// Returns true if packet is only used by `ConnectionManager` - pub fn is_connection_manager_packet(&self) -> bool { - self.packet_type == PacketType::Connection - } - /// Checks if the protocol version in the packet is a valid version pub fn is_current_protocol(&self) -> bool { ProtocolVersion::valid_version(self.protocol_version) diff --git a/src/packet/outgoing.rs b/src/packet/outgoing.rs index 94f4499d..9af8e3f2 100644 --- a/src/packet/outgoing.rs +++ b/src/packet/outgoing.rs @@ -122,14 +122,6 @@ impl<'p> OutgoingPacket<'p> { } } -/// Enum for storing different kinds of outgoing types with data. -pub enum Outgoing<'a> { - /// Represents a single packet. - Packet(OutgoingPacket<'a>), - /// Represents a packet that is fragmented and thus contains more than one `OutgoingPacket`. - Fragments(Vec>), -} - #[cfg(test)] mod tests { use crate::packet::PacketType; diff --git a/src/packet/packet_structure.rs b/src/packet/packet_structure.rs index 435bf00d..695a97df 100644 --- a/src/packet/packet_structure.rs +++ b/src/packet/packet_structure.rs @@ -191,7 +191,7 @@ pub struct GenericPacket<'a> { impl<'a> GenericPacket<'a> { /// Creates a connection manager specific packet - pub fn manager_packet( + pub fn connection_packet( payload: &'a [u8], delivery: DeliveryGuarantee, ordering: OrderingGuarantee, @@ -217,16 +217,12 @@ impl<'a> GenericPacket<'a> { } } /// Creates a heart beat packet. - pub fn heartbeat_packet( - payload: &'a [u8], - delivery: DeliveryGuarantee, - ordering: OrderingGuarantee, - ) -> Self { + pub fn heartbeat_packet(payload: &'a [u8]) -> Self { Self { packet_type: PacketType::Heartbeat, payload, - delivery, - ordering, + delivery: DeliveryGuarantee::Unreliable, + ordering: OrderingGuarantee::None, } } }