From 1dcac7bee144cf9e0ce1c31a454431c108f221de Mon Sep 17 00:00:00 2001 From: Le Horizon Date: Wed, 4 May 2022 11:54:05 -0700 Subject: [PATCH 1/7] main change for lower bounding value target --- alf/algorithms/data_transformer.py | 181 +++++++++++--- alf/algorithms/ddpg_algorithm.py | 12 +- alf/algorithms/sac_algorithm.py | 77 +++++- alf/algorithms/td_loss.py | 227 +++++++++++++++++- alf/algorithms/td_loss_test.py | 64 +++++ alf/bin/train_play_test.py | 5 +- alf/environments/suite_robotics.py | 94 +++++++- alf/environments/suite_socialbot.py | 55 +++++ alf/examples/ac_target_navigation_states.gin | 53 ++++ alf/examples/ddpg_push_states.gin | 4 + .../ddpg_target_navigation_states.gin | 61 +++++ alf/examples/her_push_states.gin | 2 + alf/examples/her_target_navigation_states.gin | 2 +- alf/examples/sac_breakout_conf.py | 11 +- alf/examples/sacbreakout-lbtq-Qbert.png | Bin 0 -> 85159 bytes alf/experience_replayers/replay_buffer.py | 18 +- alf/utils/data_buffer_test.py | 27 ++- alf/utils/math_ops.py | 34 +++ alf/utils/value_ops.py | 167 +++++++++++-- alf/utils/value_ops_test.py | 135 ++++++++++- 20 files changed, 1127 insertions(+), 102 deletions(-) create mode 100644 alf/algorithms/td_loss_test.py create mode 100644 alf/examples/ac_target_navigation_states.gin create mode 100644 alf/examples/ddpg_push_states.gin create mode 100644 alf/examples/ddpg_target_navigation_states.gin create mode 100644 alf/examples/her_push_states.gin create mode 100644 alf/examples/sacbreakout-lbtq-Qbert.png diff --git a/alf/algorithms/data_transformer.py b/alf/algorithms/data_transformer.py index b7e783abd..cf2082f06 100644 --- a/alf/algorithms/data_transformer.py +++ b/alf/algorithms/data_transformer.py @@ -22,10 +22,12 @@ import alf from alf.data_structures import AlgStep, Experience, namedtuple, StepType, TimeStep +from alf.environments import suite_socialbot from alf.experience_replayers.replay_buffer import ReplayBuffer, BatchInfo from alf.nest.utils import convert_device from alf.utils.normalizers import WindowNormalizer, EMNormalizer, AdaptiveNormalizer from alf.utils import common +from alf.utils.math_ops import l2_dist_close_reward_fn from alf.utils.normalizers import ScalarAdaptiveNormalizer FrameStackState = namedtuple('FrameStackState', ['steps', 'prev_frames']) @@ -180,6 +182,7 @@ def __init__(self, observation_spec, stack_size=4, stack_axis=0, + convert_only_minibatch_to_device=False, fields=None): """Create a FrameStacker object. @@ -198,6 +201,7 @@ def __init__(self, self._frames = dict() self._fields = fields if (fields is not None) else [None] self._exp_fields = [] + self._convert_only_minibatch_to_device = convert_only_minibatch_to_device prev_frames_spec = [] stacked_observation_spec = observation_spec for field in self._fields: @@ -350,9 +354,14 @@ def transform_experience(self, experience: Experience): B = torch.arange(batch_size) obs_index = (B.unsqueeze(-1).unsqueeze(-1), obs_index.unsqueeze(0)) + if self._convert_only_minibatch_to_device: + obs_index = convert_device(obs_index, device=replay_buffer.device) + def _stack_frame(obs, i): prev_obs = replay_buffer.get_field(self._exp_fields[i], env_ids, prev_positions) + if not self._convert_only_minibatch_to_device: + prev_obs = convert_device(prev_obs) stacked_shape = alf.nest.get_field( self._transformed_observation_spec, self._fields[i]).shape # [batch_size, mini_batch_length + stack_size - 1, ...] @@ -702,27 +711,6 @@ def forward(self, reward): return reward * self._scale -@alf.configurable -def l2_dist_close_reward_fn(achieved_goal, goal, threshold=.05): - """Giving -1/0 reward based on how close the achieved state is to the goal state. - - Args: - achieved_goal (Tensor): achieved state, of shape ``[batch_size, batch_length, ...]`` - goal (Tensor): goal state, of shape ``[batch_size, batch_length, ...]`` - threshold (float): L2 distance threshold for the reward. - - Returns: - Tensor for -1/0 reward of shape ``[batch_size, batch_length]``. - """ - - if goal.dim() == 2: # when goals are 1-dimensional - assert achieved_goal.dim() == goal.dim() - achieved_goal = achieved_goal.unsqueeze(2) - goal = goal.unsqueeze(2) - return -(torch.norm(achieved_goal - goal, dim=2) >= threshold).to( - torch.float32) - - @alf.configurable class HindsightExperienceTransformer(DataTransformer): """Randomly transform her_proportion of `batch_size` trajectories with hindsight relabel. @@ -751,6 +739,9 @@ def __init__(self, her_proportion=0.8, achieved_goal_field="time_step.observation.achieved_goal", desired_goal_field="time_step.observation.desired_goal", + sparse_reward=False, + add_noise_to_goals=False, + threshold=.05, reward_fn=l2_dist_close_reward_fn): """ Args: @@ -759,6 +750,9 @@ def __init__(self, exp nest. desired_goal_field (str): path to the desired_goal field in the exp nest. + sparse_reward (bool): Whether to transform reward from -1/0 to 0/1. + add_noise_to_goals (bool): Whether to add noise around relabeled goal. + threshold (float): noise added to relabeled goals. reward_fn (Callable): function to recompute reward based on achieve_goal and desired_goal. Default gives reward 0 when L2 distance less than 0.05 and -1 otherwise, same as is done in @@ -770,6 +764,9 @@ def __init__(self, self._her_proportion = her_proportion self._achieved_goal_field = achieved_goal_field self._desired_goal_field = desired_goal_field + self._sparse_reward = sparse_reward + self._add_noise_to_goals = add_noise_to_goals + self._threshold = threshold self._reward_fn = reward_fn def transform_timestep(self, timestep: TimeStep, state): @@ -819,32 +816,96 @@ def transform_experience(self, experience: Experience): her_cond = torch.rand(batch_size) < her_proportion (her_indices, ) = torch.where(her_cond) - last_step_pos = start_pos[her_indices] + batch_length - 1 - last_env_ids = env_ids[her_indices] - # Get x, y indices of LAST steps + has_her = torch.any(her_cond) + last_step_pos = start_pos + batch_length - 1 + last_env_ids = env_ids + # Get x, y indices of LAST steps for the whole batch, not just the HER part. dist = buffer.steps_to_episode_end(last_step_pos, last_env_ids) if alf.summary.should_record_summaries(): alf.summary.scalar( "replayer/" + buffer._name + ".mean_steps_to_episode_end", torch.mean(dist.type(torch.float32))) + def _add_noise(t): + if not self._add_noise_to_goals: + return t + bs, bl, dim = t.shape + # rejection sample from unit ball + assert dim < 20, "Cannot rejection sample from high dim ball yet." + n_samples, i = 0, 0 + while n_samples == 0: + _sample = torch.rand((bs * 2, dim)) + in_ball = torch.norm(_sample, dim=1) < 1. + if torch.any(in_ball): + sample = _sample[in_ball] + nsample = sample.shape[0] + if nsample < bs: + sample = sample.expand(bs // nsample + 1, nsample, + dim).reshape(-1, dim) + if sample.shape[0] > bs: + sample = sample[:bs, :] + break + assert i < 10, "shouldn't take 10 iterations" + i += 1 + return t + self._threshold * sample.reshape(bs, 1, dim) + # get random future state - future_idx = last_step_pos + (torch.rand(*dist.shape) * - (dist + 1)).to(torch.int64) - future_ag = buffer.get_field(self._achieved_goal_field, - last_env_ids, future_idx).unsqueeze(1) + future_dist = (torch.rand(*dist.shape) * (dist + 1)).to( + torch.int64) + future_idx = last_step_pos + future_dist + future_ag = _add_noise( + buffer.get_field(self._achieved_goal_field, last_env_ids, + future_idx).unsqueeze(1)) # relabel desired goal result_desired_goal = alf.nest.get_field(result, self._desired_goal_field) - relabed_goal = result_desired_goal.clone() + relabeled_goal = result_desired_goal.clone() her_batch_index_tuple = (her_indices.unsqueeze(1), torch.arange(batch_length).unsqueeze(0)) - relabed_goal[her_batch_index_tuple] = future_ag + if has_her: + relabeled_goal[her_batch_index_tuple] = future_ag[her_indices] # recompute rewards result_ag = alf.nest.get_field(result, self._achieved_goal_field) - relabeled_rewards = self._reward_fn(result_ag, relabed_goal) + relabeled_rewards = self._reward_fn( + result_ag, relabeled_goal, threshold=self._threshold) + if alf.summary.should_record_summaries(): + alf.summary.scalar( + "replayer/" + buffer._name + + ".discount_mean_before_relabel", + torch.mean(result.discount[:, 1:])) + if self._sparse_reward: + reward_achieved = relabeled_rewards >= 0 + # Cut off episode for any goal reached. + end = reward_achieved + discount = torch.where(end, torch.tensor(0.), result.discount) + step_type = torch.where(end, torch.tensor(StepType.LAST), + result.step_type) + # Also relabel ``LAST``` steps to ``MID``` where aux goals were not + # achieved but env ended episode due to position goal achieved. + # -1/0 reward doesn't end episode on achieving position goal, and + # doesn't need to do this relabeling. + goal_reward = result.reward + if len(result.reward.shape) > 2: + goal_reward = result.reward[..., 0] + mid = ( + result.step_type == StepType.LAST) & ~reward_achieved & ( + goal_reward > 0) # assumes no multi dim goal reward. + discount = torch.where(mid, torch.tensor(1.), discount) + step_type = torch.where(mid, torch.tensor(StepType.MID), + step_type) + + if alf.summary.should_record_summaries(): + alf.summary.scalar( + "replayer/" + buffer._name + + ".discount_mean_after_relabel", + torch.mean(discount[:, 1:])) + + result = result._replace(discount=discount) + result = result._replace(step_type=step_type) + relabeled_rewards = suite_socialbot.transform_reward_tensor( + relabeled_rewards) non_her_or_fst = ~her_cond.unsqueeze(1) & (result.step_type != StepType.FIRST) @@ -874,21 +935,60 @@ def transform_experience(self, experience: Experience): alf.summary.scalar( "replayer/" + buffer._name + ".reward_mean_before_relabel", torch.mean(result.reward[her_indices][:-1])) - alf.summary.scalar( - "replayer/" + buffer._name + ".reward_mean_after_relabel", - torch.mean(relabeled_rewards[her_indices][:-1])) + if has_her: + alf.summary.scalar( + "replayer/" + buffer._name + ".reward_mean_after_relabel", + torch.mean(relabeled_rewards[her_indices][:-1])) + alf.summary.scalar("replayer/" + buffer._name + ".future_distance", + torch.mean(future_dist.float())) + + goal_rewards = result.reward + if result.reward.ndim > 2: + goal_rewards = result.reward[:, :, 0] + + # assert reward function is the same as used by the environment. + if not torch.allclose(relabeled_rewards[non_her_or_fst], + goal_rewards[non_her_or_fst]): + not_close = torch.abs(relabeled_rewards[non_her_or_fst] - + goal_rewards[non_her_or_fst]) > 0.01 + msg = ("hindsight_relabel:\nrelabeled_reward\n{}\n!=\n" + + "env_reward\n{}\nag:\n{}\ndg:\n{}\nenv_ids:\n{}\nstart_pos:" + + "\n{}").format( + relabeled_rewards[non_her_or_fst][not_close], + goal_rewards[non_her_or_fst][not_close], + result_ag[non_her_or_fst][not_close], + result_desired_goal[non_her_or_fst][not_close], + env_ids.unsqueeze(1).expand( + shape[:2])[non_her_or_fst][not_close], + start_pos.unsqueeze(1).expand( + shape[:2])[non_her_or_fst][not_close]) + logging.warning(msg) + # assert False, msg + # relabeled_rewards[non_her_or_fst] = goal_rewards[non_her_or_fst] + + final_relabeled_rewards = relabeled_rewards + if result.reward.ndim > 2: + # multi dimensional env reward, first dim is goal related reward. + final_relabeled_rewards = result.reward.clone() + final_relabeled_rewards[:, :, 0] = relabeled_rewards + result = result.update_time_step_field('reward', + final_relabeled_rewards) result = alf.nest.transform_nest( - result, self._desired_goal_field, lambda _: relabed_goal) - - result = result.update_time_step_field('reward', relabeled_rewards) + result, self._desired_goal_field, lambda _: relabeled_goal) + info = info._replace(her=her_cond, future_distance=future_dist) if alf.get_default_device() != buffer.device: for f in accessed_fields: result = alf.nest.transform_nest( result, f, lambda t: convert_device(t)) - result = alf.nest.transform_nest( - result, "batch_info.replay_buffer", lambda _: buffer) + info = convert_device(info) + info = info._replace( + her=info.her.unsqueeze(1).expand(exp.reward.shape[:2]), + future_distance=info.future_distance.unsqueeze(1).expand( + exp.reward.shape[:2]), + replay_buffer=buffer) + result = alf.data_structures.add_batch_info(result, info) return result @@ -922,4 +1022,7 @@ def create_data_transformer(data_transformer_ctor, observation_spec): if len(data_transformer_ctor) == 1: return data_transformer_ctor[0](observation_spec) + if HindsightExperienceTransformer in data_transformer_ctor: + assert HindsightExperienceTransformer == data_transformer_ctor[0], \ + "Hindsight relabeling should happen before all other transforms." return SequentialDataTransformer(data_transformer_ctor, observation_spec) diff --git a/alf/algorithms/ddpg_algorithm.py b/alf/algorithms/ddpg_algorithm.py index 7c0678998..25c3a73f2 100644 --- a/alf/algorithms/ddpg_algorithm.py +++ b/alf/algorithms/ddpg_algorithm.py @@ -237,10 +237,12 @@ def _sample(a, ou): noisy_action, self._action_spec) state = empty_state._replace( actor=DdpgActorState(actor=state, critics=())) + # action_distribution is not supported for continuous actions for now. + # Returns empty action_distribution to fail early. return AlgStep( output=noisy_action, state=state, - info=DdpgInfo(action=noisy_action, action_distribution=action)) + info=DdpgInfo(action=noisy_action, action_distribution=())) def rollout_step(self, time_step: TimeStep, state=None): if self.need_full_rollout_state(): @@ -330,7 +332,8 @@ def train_step(self, inputs: TimeStep, state: DdpgState, reward=inputs.reward, step_type=inputs.step_type, discount=inputs.discount, - action_distribution=policy_step.output, + action=policy_step.output, + action_distribution=(), critic=critic_info, actor_loss=policy_step.info, discounted_return=rollout_info.discounted_return)) @@ -355,6 +358,11 @@ def calc_loss(self, info: DdpgInfo): actor_loss = info.actor_loss + if self._critic_losses[0]._improve_w_nstep_bootstrap: + # Ignore 2nd - nth step actor losses. + actor_loss.loss[1:] = 0 + actor_loss.extra[1:] = 0 + return LossInfo( loss=critic_loss + actor_loss.loss, priority=priority, diff --git a/alf/algorithms/sac_algorithm.py b/alf/algorithms/sac_algorithm.py index 91dd59134..f8a798704 100644 --- a/alf/algorithms/sac_algorithm.py +++ b/alf/algorithms/sac_algorithm.py @@ -34,6 +34,7 @@ import alf.nest.utils as nest_utils from alf.networks import ActorDistributionNetwork, CriticNetwork from alf.networks import QNetwork, QRNNNetwork +from alf.summary import render from alf.tensor_specs import TensorSpec, BoundedTensorSpec from alf.utils import losses, common, dist_utils, math_ops from alf.utils.normalizers import ScalarAdaptiveNormalizer @@ -56,7 +57,8 @@ SacInfo = namedtuple( "SacInfo", [ "reward", "step_type", "discount", "action", "action_distribution", - "actor", "critic", "alpha", "log_pi", "discounted_return" + "actor", "critic", "alpha", "log_pi", "discounted_return", + "future_distance", "her" ], default_value=()) @@ -152,6 +154,9 @@ def __init__(self, q_network_cls=QNetwork, reward_weights=None, epsilon_greedy=None, + rollout_epsilon_greedy=1.0, + use_epsilon_schedule=0, + max_target_action=False, use_entropy_reward=True, normalize_entropy_reward=False, calculate_priority=False, @@ -203,6 +208,14 @@ def __init__(self, Breakout. Only used for evaluation. If None, its value is taken from ``config.epsilon_greedy`` and then ``alf.get_config_value(TrainerConfig.epsilon_greedy)``. + rollout_epsilon_greedy (float): epsilon greedy policy for rollout. + Together with the following three parameters, the Sac algorithm + can be converted to a DQN algorithm. + use_epsilon_schedule (float): training schedule for + rollout_epsilon_greedy. + max_target_action (bool): whether to use the action with the highest + target value as the target action for computing bootstrapped value. + To mimic the DQN algorithm, set this to True. use_entropy_reward (bool): whether to include entropy as reward normalize_entropy_reward (bool): if True, normalize entropy reward to reduce bias in episodic cases. Only used if @@ -261,6 +274,9 @@ def __init__(self, if epsilon_greedy is None: epsilon_greedy = alf.utils.common.get_epsilon_greedy(config) self._epsilon_greedy = epsilon_greedy + self._rollout_epsilon_greedy = rollout_epsilon_greedy + self._use_epsilon_schedule = use_epsilon_schedule + self._max_target_action = max_target_action critic_networks, actor_network, self._act_type = self._make_networks( observation_spec, action_spec, reward_spec, actor_network_cls, @@ -274,7 +290,10 @@ def __init__(self, ) def _init_log_alpha(): - return nn.Parameter(torch.tensor(float(initial_log_alpha))) + alpha = torch.tensor(float(initial_log_alpha)) + if alpha_optimizer is None: + return alpha + return nn.Parameter(alpha) if self._act_type == ActionType.Mixed: # separate alphas for discrete and continuous actions @@ -314,7 +333,7 @@ def _init_log_alpha(): self.add_optimizer(alpha_optimizer, nest.flatten(log_alpha)) self._log_alpha = log_alpha - if self._act_type == ActionType.Mixed: + if self._act_type == ActionType.Mixed and alpha_optimizer is not None: self._log_alpha_paralist = nn.ParameterList( nest.flatten(log_alpha)) @@ -376,6 +395,8 @@ def _init_log_alpha(): target_models=[self._target_critic_networks], tau=target_update_tau, period=target_update_period) + # initial q value range for rendering; adjusted as playing progresses + self._q_range = (0, 3) def _make_networks(self, observation_spec, action_spec, reward_spec, continuous_actor_network_cls, critic_network_cls, @@ -531,15 +552,36 @@ def _predict_action(self, return action_dist, action, q_values, new_state def predict_step(self, inputs: TimeStep, state: SacState): - action_dist, action, _, action_state = self._predict_action( + action_dist, action, q_values, action_state = self._predict_action( inputs.observation, state=state.action, epsilon_greedy=self._epsilon_greedy, eps_greedy_sampling=True) + info = SacInfo(action_distribution=action_dist) + if (alf.summary.render.is_rendering_enabled() + and self._act_type == ActionType.Discrete): + num_acts = q_values.shape[-1] + self._q_range = (min(self._q_range[0], int(q_values.min())), + max(self._q_range[1], + int(q_values.max()) + 1)) + info = dict( + sac=info, + action_img=render.render_action("action", action, + self._action_spec), + action_dist_img=render.render_bar( + "action_dist", + action_dist.probs, + y_range=(0, 1), + annotate_format="%.2f", + x_ticks=range(num_acts)), + q_img=render.render_bar( + "q_values", + q_values, + y_range=self._q_range, + annotate_format="%.2f", + x_ticks=range(num_acts))) return AlgStep( - output=action, - state=SacState(action=action_state), - info=SacInfo(action_distribution=action_dist)) + output=action, state=SacState(action=action_state), info=info) def rollout_step(self, inputs: TimeStep, state: SacState): """``rollout_step()`` basically predicts actions like what is done by @@ -547,10 +589,15 @@ def rollout_step(self, inputs: TimeStep, state: SacState): buffer, then this function also call ``_critic_networks`` and ``_target_critic_networks`` to maintain their states. """ + eps = self._rollout_epsilon_greedy + if self._use_epsilon_schedule > 0: + progress = alf.trainers.policy_trainer.Trainer.progress() + if progress < self._use_epsilon_schedule: + eps = 1.0 - (1.0 - eps) * progress / self._use_epsilon_schedule action_dist, action, _, action_state = self._predict_action( inputs.observation, state=state.action, - epsilon_greedy=1.0, + epsilon_greedy=eps, eps_greedy_sampling=True, rollout=True) @@ -717,7 +764,10 @@ def _critic_train_step(self, inputs: TimeStep, state: SacCriticState, probs = common.expand_dims_as(action_distribution.probs, target_critics) # [B, reward_dim] - target_critics = torch.sum(probs * target_critics, dim=1) + if self._max_target_action: + target_critics = torch.max(target_critics, dim=1)[0] + else: + target_critics = torch.sum(probs * target_critics, dim=1) elif self._act_type == ActionType.Mixed: critics = self._select_q_value(rollout_info.action[0], critics) discrete_act_dist = action_distribution[0] @@ -797,6 +847,12 @@ def calc_loss(self, info: SacInfo): alpha_loss = info.alpha actor_loss = info.actor + if self._critic_losses[0]._improve_w_nstep_bootstrap: + # Ignore 2nd - n-th step losses in this mode. + alpha_loss[1:] = 0 + if actor_loss.loss != (): + actor_loss.loss[1:] = 0 + if self._debug_summaries and alf.summary.should_record_summaries(): with alf.summary.scope(self._name): if self._act_type == ActionType.Mixed: @@ -850,6 +906,9 @@ def _calc_critic_loss(self, info: SacInfo): if self._use_entropy_reward: with torch.no_grad(): log_pi = info.log_pi + if self._critic_losses[0]._improve_w_nstep_bootstrap: + # Ignore 2nd - n-th step entropy in this mode. + log_pi[1:] = 0 if self._entropy_normalizer is not None: log_pi = self._entropy_normalizer.normalize(log_pi) entropy_reward = nest.map_structure( diff --git a/alf/algorithms/td_loss.py b/alf/algorithms/td_loss.py index 80c2c0a93..7e602c7dd 100644 --- a/alf/algorithms/td_loss.py +++ b/alf/algorithms/td_loss.py @@ -31,9 +31,20 @@ class TDLoss(nn.Module): def __init__(self, gamma: Union[float, List[float]] = 0.99, td_error_loss_fn: Callable = element_wise_squared_loss, + clip: float = 0., td_lambda: float = 0.95, normalize_target: bool = False, debug_summaries: bool = False, + lb_target_q: float = 0., + default_return: float = -1000., + improve_w_goal_return: bool = False, + improve_w_nstep_bootstrap: bool = False, + improve_w_nstep_only: bool = False, + lower_bound_constraint: float = 0., + lb_loss_scale: bool = False, + reward_multiplier: float = 1., + positive_reward: bool = True, + use_retrace: bool = False, name: str = "TDLoss"): r""" Let :math:`G_{t:T}` be the bootstraped return from t to T: @@ -80,10 +91,35 @@ def __init__(self, td_error_loss_fn: A function for computing the TD errors loss. This function takes as input the target and the estimated Q values and returns the loss for each element of the batch. + clip: When positive, loss clipping to the range [-clip, clip]. td_lambda: Lambda parameter for TD-lambda computation. normalize_target (bool): whether to normalize target. Note that the effect of this is to change the loss. The critic value itself is not normalized. + use_retrace: turn on retrace loss + :math:`\mathcal{R} Q(x, a):=Q(x, a)+\mathbb{E}_{\mu}\left[\sum_{t \geq 0} \gamma^{t}\left(\prod_{s=1}^{t} c_{s}\right)\left(r_{t}+\gamma \mathbb{E}_{\pi} Q\left(x_{t+1}, \cdot\right)-Q\left(x_{t}, a_{t}\right)\right)\right]` + copied from PR #695. + lb_target_q: between 0 and 1. When not zero, use this mixing rate for the + lower bounded value target. Only supports batch_length == 2, one step td. + default_return: Keep it the same as replay_buffer.default_return to plot to + tensorboard episodic_discounted_return only for the timesteps whose + episode already ended. + improve_w_goal_return: Use return calculated from the distance to hindsight + goals. Only supports batch_length == 2, one step td. + improve_w_nstep_bootstrap: Look ahead 2 to n steps, and take the largest + bootstrapped return to lower bound the value target of the 1st step. + improve_w_nstep_only: Only use the n-th step bootstrapped return as + value target lower bound. + lower_bound_constraint: Use n-step bootstrapped return as lower bound + constraints of the value. See reference: + He, F. S., Liu, Y., Schwing, A. G., and Peng, J. + Learning to play in a day: Faster deep reinforcement learning + by optimality tightening. In 5th International Conference + on Learning Representations, ICLR 2017, Toulon, France, + April 24-26, 2017. https://openreview.net/forum?id=rJ8Je4clg + lb_loss_scale: Parameter for lower_bound_constraint. + reward_multiplier: Weight on the hindsight goal return. + positive_reward: If True, assumes 0/1 goal reward, otherwise, -1/0. debug_summaries: True if debug summaries should be created. name: The name of this loss. """ @@ -92,7 +128,18 @@ def __init__(self, self._name = name self._gamma = torch.tensor(gamma) self._td_error_loss_fn = td_error_loss_fn + self._clip = clip self._lambda = td_lambda + self._lb_target_q = lb_target_q + self._default_return = default_return + self._improve_w_goal_return = improve_w_goal_return + self._improve_w_nstep_bootstrap = improve_w_nstep_bootstrap + self._improve_w_nstep_only = improve_w_nstep_only + self._lower_bound_constraint = lower_bound_constraint + self._lb_loss_scale = lb_loss_scale + self._reward_multiplier = reward_multiplier + self._positive_reward = positive_reward + self._use_retrace = use_retrace self._debug_summaries = debug_summaries self._normalize_target = normalize_target self._target_normalizer = None @@ -106,7 +153,11 @@ def gamma(self): """ return self._gamma.clone() - def compute_td_target(self, info: namedtuple, target_value: torch.Tensor): + def compute_td_target(self, + info: namedtuple, + value: torch.Tensor, + target_value: torch.Tensor, + qr: bool = False): """Calculate the td target. The first dimension of all the tensors is time dimension and the second @@ -119,32 +170,118 @@ def compute_td_target(self, info: namedtuple, target_value: torch.Tensor): - reward: - step_type: - discount: + value (torch.Tensor): the time-major tensor for the value at + each time step. Some of its value can be overwritten and passed + back to the caller. target_value (torch.Tensor): the time-major tensor for the value at each time step. This is used to calculate return. ``target_value`` - can be same as ``value``. + can be same as ``value``, except for Retrace. Returns: - td_target + td_target, updated value, optional constraint_loss """ + if not qr and info.reward.ndim == 3: + # Multi-dim reward, not quantile regression. + # [T, B, D] or [T, B, 1] + discounts = info.discount.unsqueeze(-1) * self._gamma + else: + # [T, B] + discounts = info.discount * self._gamma + if self._lambda == 1.0: returns = value_ops.discounted_return( rewards=info.reward, values=target_value, step_types=info.step_type, - discounts=info.discount * self._gamma) + discounts=discounts) elif self._lambda == 0.0: returns = value_ops.one_step_discounted_return( rewards=info.reward, values=target_value, step_types=info.step_type, - discounts=info.discount * self._gamma) - else: + discounts=discounts) + elif not self._use_retrace: advantages = value_ops.generalized_advantage_estimation( rewards=info.reward, values=target_value, step_types=info.step_type, - discounts=info.discount * self._gamma, + discounts=discounts, td_lambda=self._lambda) returns = advantages + target_value[:-1] + else: # Retrace + scope = alf.summary.scope(self.__class__.__name__) + assert info.rollout_info.action_distribution != (), \ + "Algorithm does not provide rollout action_distribution" + importance_ratio, importance_ratio_clipped = value_ops. \ + action_importance_ratio( + action_distribution=info.action_distribution, + rollout_action_distribution=info.rollout_info.action_distribution, + action=info.action, + clipping_mode='capping', + importance_ratio_clipping=0.0, + log_prob_clipping=0.0, + scope=scope, + check_numerics=False, + debug_summaries=self._debug_summaries) + advantages = value_ops.generalized_advantage_estimation_retrace( + importance_ratio=importance_ratio_clipped, + rewards=info.reward, + values=value, + target_value=target_value, + step_types=info.step_type, + discounts=discounts, + use_retrace=True, + time_major=True, + td_lambda=self._lambda) + + returns = advantages + value[:-1] + returns = returns.detach() + + constraint_loss = None + if self._improve_w_nstep_bootstrap: + assert self._lambda == 1.0, "td lambda does not work with this" + future_returns = value_ops.first_step_future_discounted_returns( + rewards=info.reward, + values=target_value, + step_types=info.step_type, + discounts=discounts) + returns = value_ops.one_step_discounted_return( + rewards=info.reward, + values=target_value, + step_types=info.step_type, + discounts=discounts) + assert torch.all((returns[0] == future_returns[0]) | ( + info.step_type[0] == alf.data_structures.StepType.LAST)), \ + str(returns[0]) + " ne\n" + str(future_returns[0]) + \ + '\nrwd: ' + str(info.reward[0:2]) + \ + '\nlast: ' + str(info.step_type[0:2]) + \ + '\ndisct: ' + str(discounts[0:2]) + \ + '\nv: ' + str(target_value[0:2]) + if self._improve_w_nstep_only: + future_returns = future_returns[ + -1] # last is the n-step return + else: + future_returns = torch.max(future_returns, dim=0)[0] + + with alf.summary.scope(self._name): + alf.summary.scalar( + "max_1_to_n_future_return_gt_td", + torch.mean((returns[0] < future_returns).float())) + if self._lower_bound_constraint > 0: + alf.summary.scalar( + "max_1_to_n_future_return_gt_value", + torch.mean((value[0] < future_returns).float())) + alf.summary.scalar("first_step_discounted_return", + torch.mean(returns[0])) + + if self._lower_bound_constraint > 0: + constraint_loss = self._lower_bound_constraint * torch.max( + torch.zeros_like(future_returns), + future_returns.detach() - value[0])**2 + else: + returns[0] = torch.max(future_returns, returns[0]).detach() + returns[1:] = 0 + value = value.clone() + value[1:] = 0 disc_ret = () if hasattr(info, "discounted_return"): @@ -158,7 +295,57 @@ def compute_td_target(self, info: namedtuple, target_value: torch.Tensor): "value_episode_ended_all", torch.mean(value[:-1][:, episode_ended[0, :]])) - return returns + if self._lb_target_q > 0 and disc_ret != (): + her_cond = info.her + mask = torch.ones(returns.shape, dtype=torch.bool) + if her_cond != () and torch.any(~her_cond): + mask = ~her_cond[:-1] + disc_ret = disc_ret[ + 1:] # it's expanded in ddpg_algorithm, need to revert back. + assert returns.shape == disc_ret.shape, "%s %s" % (returns.shape, + disc_ret.shape) + with alf.summary.scope(self._name): + alf.summary.scalar( + "episodic_return_gt_td", + torch.mean((returns < disc_ret).float()[mask])) + alf.summary.scalar( + "episodic_discounted_return", + torch.mean( + disc_ret[mask & (disc_ret > self._default_return)])) + returns[mask] = (1 - self._lb_target_q) * returns[mask] + \ + self._lb_target_q * torch.max(returns, disc_ret)[mask] + + if self._improve_w_goal_return: + batch_length, batch_size = returns.shape[:2] + her_cond = info.her + if her_cond != () and torch.any(her_cond): + dist = info.future_distance + if self._positive_reward: + goal_return = torch.pow( + self._gamma * torch.ones(her_cond.shape), dist) + else: + goal_return = -(1. - torch.pow(self._gamma, dist)) / ( + 1. - self._gamma) + goal_return *= self._reward_multiplier + goal_return = goal_return[:-1] + returns_0 = returns + # Multi-dim reward: + if len(returns.shape) > 2: + returns_0 = returns[:, :, 0] + returns_0 = torch.where(her_cond[:-1], + torch.max(returns_0, goal_return), + returns_0) + with alf.summary.scope(self._name): + alf.summary.scalar( + "goal_return_gt_td", + torch.mean((returns_0 < goal_return).float())) + alf.summary.scalar("goal_return", torch.mean(goal_return)) + if len(returns.shape) > 2: + returns[:, :, 0] = returns_0 + else: + returns = returns_0 + + return returns, value, constraint_loss def forward(self, info: namedtuple, value: torch.Tensor, target_value: torch.Tensor): @@ -182,7 +369,8 @@ def forward(self, info: namedtuple, value: torch.Tensor, Returns: LossInfo: with the ``extra`` field same as ``loss``. """ - returns = self.compute_td_target(info, target_value) + returns, value, constraint_loss = self.compute_td_target( + info, value, target_value) value = value[:-1] if self._normalize_target: @@ -219,11 +407,29 @@ def _summarize(v, r, td, suffix): suffix) loss = self._td_error_loss_fn(returns.detach(), value) + if self._clip > 0: + loss = torch.clamp(loss, min=-self._clip, max=self._clip) if loss.ndim == 3: # Multidimensional reward. Average over the critic loss for all dimensions loss = loss.mean(dim=2) + if self._improve_w_nstep_bootstrap: + # Ignore 2nd to n-th step losses. + loss[1:] = 0 + if self._lower_bound_constraint > 0: + assert constraint_loss.shape == loss.shape[1:], \ + f"{constraint_loss.shape} != {loss.shape}[1:]" + c_loss = constraint_loss.clone().unsqueeze(0).repeat( + (loss.shape[0], 1)) + c_loss[1:] = 0 + if self._lb_loss_scale: + scale = ( + torch.sum(loss) / torch.sum(c_loss + loss)).detach() + else: + scale = 1 + loss = (c_loss + loss) * scale + # The shape of the loss expected by Algorith.update_with_gradient is # [T, B], so we need to augment it with additional zeros. loss = tensor_utils.tensor_extend_zero(loss) @@ -301,7 +507,8 @@ def forward(self, info: namedtuple, value: torch.Tensor, assert target_value.shape[-1] == self._num_quantiles, ( "The input target_value should have same num_quantiles as pre-defiend." ) - returns = self.compute_td_target(info, target_value) + returns, value, constraint_loss = self.compute_td_target( + info, value, target_value, qr=True) value = value[:-1] # for quantile regression TD, the value and target both have shape diff --git a/alf/algorithms/td_loss_test.py b/alf/algorithms/td_loss_test.py new file mode 100644 index 000000000..ac7c9ef0a --- /dev/null +++ b/alf/algorithms/td_loss_test.py @@ -0,0 +1,64 @@ +# Copyright (c) 2019 Horizon Robotics. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch + +import alf +from alf.algorithms.td_loss import TDLoss +from alf.data_structures import TimeStep, StepType, namedtuple + +DataItem = namedtuple( + "DataItem", ["reward", "step_type", "discount"], default_value=()) + + +class TDLossTest(unittest.TestCase): + """Tests for alf.algorithms.td_loss.TDLoss + """ + + def _check(self, res, expected): + np.testing.assert_array_almost_equal(res, expected) + + def test_compute_td_target_nstep_bootstrap_lowerbound(self): + loss = TDLoss(gamma=1., improve_w_nstep_bootstrap=True, td_lambda=1) + # Tensors are transposed to be time_major [T, B, ...] + step_types = torch.tensor([[StepType.MID] * 5], + dtype=torch.int64).transpose(0, 1) + rewards = torch.tensor([[2.] * 5], dtype=torch.float32).transpose(0, 1) + discounts = torch.tensor([[0.9] * 5], dtype=torch.float32).transpose( + 0, 1) + values = torch.tensor([[1.] * 5], dtype=torch.float32).transpose(0, 1) + info = DataItem( + reward=rewards, step_type=step_types, discount=discounts) + returns, value, _ = loss.compute_td_target(info, values, values) + expected_return = torch.tensor( + [[2 + 0.9 * (2 + 0.9 * (2 + 0.9 * (2 + 0.9))), 0, 0, 0]], + dtype=torch.float32).transpose(0, 1) + self._check(res=returns, expected=expected_return) + + expected_value = torch.tensor([[1, 0, 0, 0, 0]], + dtype=torch.float32).transpose(0, 1) + self._check(res=value, expected=expected_value) + + # n-step return is below 1-step + values[2:] = -10 + expected_return[0] = 2 + 0.9 + returns, value, _ = loss.compute_td_target(info, values, values) + self._check(res=returns, expected=expected_return) + + +if __name__ == '__main__': + alf.test.main() diff --git a/alf/bin/train_play_test.py b/alf/bin/train_play_test.py index 9264be0ed..400640d20 100644 --- a/alf/bin/train_play_test.py +++ b/alf/bin/train_play_test.py @@ -141,7 +141,10 @@ def _to_conf_params(parameters): 'TrainerConfig.num_updates_per_train_iter=1', 'TrainerConfig.mini_batch_length=2', 'TrainerConfig.mini_batch_size=4', - 'TrainerConfig.replay_buffer_length=64', + # If replay_buffer_length is above 6 (3 iters * 2 unroll length), whole replay buffer training + # algorithms (e.g. mbrl_pendulum) will not get enough training data, and will not start training. + # This fails during playing, when the config file isn't generated in root_dir. + 'TrainerConfig.replay_buffer_length=4', 'FrameStacker.stack_size=1' ] OFF_POLICY_TRAIN_PARAMS = _to_conf_params(OFF_POLICY_TRAIN_CONF) diff --git a/alf/environments/suite_robotics.py b/alf/environments/suite_robotics.py index 3bcd2b70c..780ddc864 100644 --- a/alf/environments/suite_robotics.py +++ b/alf/environments/suite_robotics.py @@ -43,19 +43,42 @@ def is_available(): return mujoco_py is not None +@alf.configurable class SparseReward(gym.Wrapper): """Convert the original :math:`-1/0` rewards to :math:`0/1`. """ - def __init__(self, env): + def __init__(self, + env, + reward_cap=1., + positive_reward=True, + append_reward_dim=False): gym.Wrapper.__init__(self, env) + self._reward_cap = reward_cap + self._positive_reward = positive_reward + self._append_reward_dim = append_reward_dim def step(self, action): # openai Robotics env will always return ``done=False`` ob, reward, done, info = self.env.step(action) if reward == 0: done = True - return ob, reward + 1, done, info + if self._positive_reward: + return_reward = reward + 1 + else: + return_reward = reward + return_reward *= self._reward_cap + if self._append_reward_dim: + return_reward = np.array([return_reward, 0]) + return ob, return_reward, done, info + + @property + def reward_space(self): + if self._append_reward_dim: + return gym.spaces.Box( + low=-np.inf, high=np.inf, shape=(2, ), dtype=np.float32) + else: + return self.env.reward_space() @alf.configurable @@ -84,6 +107,41 @@ def step(self, action): return obs, reward, done, info +@alf.configurable +class TransformGoals(gym.Wrapper): + """Convert the original achieved_goal and desired_goal to first two dims, and produce sparse reward. + + It ignores original reward which is a multi dimensional negative distance to goal. + """ + + def __init__(self, env): + super().__init__(env) + goal_space = gym.spaces.Box( + env.observation_space["achieved_goal"].low[:2], + env.observation_space["achieved_goal"].high[:2]) + self.observation_space = gym.spaces.Dict({ + "achieved_goal": goal_space, + "desired_goal": goal_space, + "observation": env.observation_space["observation"] + }) + + def reset(self): + ob = self.env.reset() + ob["achieved_goal"] = ob["achieved_goal"][:2] + ob["desired_goal"] = ob["desired_goal"][:2] + return ob + + def step(self, action): + # openai Robotics env will always return ``done=False`` + ob, reward, done, info = self.env.step(action) + ob["achieved_goal"] = ob["achieved_goal"][:2] + ob["desired_goal"] = ob["desired_goal"][:2] + return_reward = alf.utils.math_ops.l2_dist_close_reward_fn_np( + ob["achieved_goal"], ob["desired_goal"]) + return_reward = return_reward[0] + return ob, return_reward, done, info + + @alf.configurable class ObservationClipWrapper(gym.ObservationWrapper): """Clip observation values according to OpenAI's baselines. @@ -105,6 +163,16 @@ def observation(self, observation): return np.clip(observation, self.min_v, self.max_v) +@alf.configurable +class RemoveInfoWrapper(gym.Wrapper): + """Remove all the info from environment return. + """ + + def step(self, action): + obs, reward, done, info = self.env.step(action) + return obs, reward, done, {} + + @alf.configurable def load(environment_name, env_id=None, @@ -141,14 +209,28 @@ def load(environment_name, Returns: An AlfEnvironment instance. """ - assert (environment_name.startswith("Fetch") - or environment_name.startswith("HandManipulate")), ( - "This suite only supports OpenAI's Fetch and ShadowHand envs!") + assert ( + environment_name.startswith("Fetch") + or environment_name.startswith("HandManipulate") + or environment_name.startswith("Ant") + ), ("This suite only supports OpenAI's Fetch, ShadowHand and multiworld Ant envs!" + ) _unwrapped_env_checker_.check_and_update(wrap_with_process) + kwargs = {} + if environment_name.startswith("Ant"): + from multiworld.envs.mujoco import register_custom_envs + register_custom_envs() + gym_spec = gym.spec(environment_name) - env = gym_spec.make() + env = gym_spec.make(**kwargs) + if environment_name.startswith("Ant"): + from gym.wrappers import FilterObservation + env = RemoveInfoWrapper( + FilterObservation( + env, ["desired_goal", "achieved_goal", "observation"])) + env = TransformGoals(env) if max_episode_steps is None: if gym_spec.max_episode_steps is not None: diff --git a/alf/environments/suite_socialbot.py b/alf/environments/suite_socialbot.py index 5cae9d688..e1a7e749e 100644 --- a/alf/environments/suite_socialbot.py +++ b/alf/environments/suite_socialbot.py @@ -23,7 +23,9 @@ from fasteners.process_lock import InterProcessLock import functools import gym +import numpy as np import socket +import torch import alf from alf.environments import suite_gym, alf_wrappers, process_environment @@ -38,11 +40,61 @@ def is_available(): return social_bot is not None +@alf.configurable +def transform_reward(reward, reward_cap=1., positive_reward=True): + goal_reward = reward + if isinstance(reward, (np.ndarray, list)): + goal_reward = reward[0] + if positive_reward: + goal_reward = goal_reward >= 0 + goal_reward = goal_reward * reward_cap + if isinstance(reward, (np.ndarray, list)): + reward[0] = goal_reward + else: + reward = goal_reward + return reward + + +@alf.configurable +def transform_reward_tensor(reward, reward_cap=1., positive_reward=True): + goal_reward = reward + if reward.ndim > 2: + goal_reward = reward[:, :, 0] + if positive_reward: + goal_reward = torch.where(goal_reward >= 0, torch.ones(()), + torch.zeros(())) + goal_reward *= reward_cap + if reward.ndim > 2: + reward[:, :, 0] = goal_reward + else: + reward = goal_reward + return reward + + +class SparseReward(gym.Wrapper): + """Convert the original :math:`-1/0` rewards to :math:`0/1`. + """ + + def __init__(self, env): + gym.Wrapper.__init__(self, env) + + def step(self, action): + ob, reward, done, info = self.env.step(action) + goal_reward = reward + if isinstance(reward, (np.ndarray, list)): + goal_reward = reward[0] + if goal_reward == 0: + done = True + reward = transform_reward(reward) + return ob, reward, done, info + + @alf.configurable def load(environment_name, env_id=None, port=None, wrap_with_process=False, + sparse_reward=False, discount=1.0, max_episode_steps=None, gym_env_wrappers=(), @@ -57,6 +109,7 @@ def load(environment_name, env_id (int): (optional) ID of the environment. port (int): Port used for the environment wrap_with_process (bool): Whether wrap environment in a new process + sparse_reward (bool): Whether to use 0/1 instead of -1/0 reward. discount (float): Discount to use for the environment. max_episode_steps (int): If None the max_episode_steps will be set to the default step limit defined in the environment's spec. No limit is applied if set @@ -84,6 +137,8 @@ def load(environment_name, def env_ctor(port, env_id=None): gym_env = gym_spec.make(port=port) + if sparse_reward: + gym_env = SparseReward(gym_env) return suite_gym.wrap_env( gym_env, env_id=env_id, diff --git a/alf/examples/ac_target_navigation_states.gin b/alf/examples/ac_target_navigation_states.gin new file mode 100644 index 000000000..fb3b0fcd8 --- /dev/null +++ b/alf/examples/ac_target_navigation_states.gin @@ -0,0 +1,53 @@ +include 'ac_simple_navigation.gin' + +batch_size=30 +create_environment.num_parallel_environments=%batch_size + +create_environment.env_name='SocialBot-PlayGround-v0' + +suite_socialbot.load.gym_env_wrappers=[] +import alf.utils.math_ops + +conv_layer_params=None +actor/ActorDistributionNetwork.continuous_projection_net_ctor=@NormalProjectionNetwork +actor/ActorDistributionNetwork.conv_layer_params=%conv_layer_params +value/ValueNetwork.conv_layer_params=%conv_layer_params + +ac/AdamTF.lr=2e-4 +ac/AdamTF.gradient_clipping=0.5 + +ActorCriticLoss.entropy_regularization=0.0005 + +# Episodic limits +suite_socialbot.load.max_episode_steps=100 +GoalTask.max_steps=1000 +PlayGround.max_steps=1000 +GoalTask.fail_distance_thresh=1000 +GoalTask.end_episode_after_success=True +GoalTask.end_on_hitting_distraction=True + +# Goal & distraction object setup +GoalTask.random_goal=False +GoalTask.goal_name='ball' +GoalTask.distraction_list=['coke_can', 'car_wheel'] +GoalTask.distraction_penalty_distance_thresh=0.4 + +# Curriculum +GoalTask.use_curriculum_training=True +GoalTask.start_range=1 +GoalTask.max_reward_q_length=100 +GoalTask.reward_thresh_to_increase_range=0.9 +GoalTask.increase_range_by_percent=0.1 +GoalTask.percent_full_range_in_curriculum=0.2 + +# Observation +GoalTask.polar_coord=True +PlayGround.use_image_observation=False +PlayGround.with_language=False +TrainerConfig.data_transformer_ctor=None + +TrainerConfig.summary_interval=20 +TrainerConfig.num_checkpoints=10 +TrainerConfig.evaluate=True +TrainerConfig.eval_interval=100 +TrainerConfig.num_eval_episodes=50 diff --git a/alf/examples/ddpg_push_states.gin b/alf/examples/ddpg_push_states.gin new file mode 100644 index 000000000..46cb62f0e --- /dev/null +++ b/alf/examples/ddpg_push_states.gin @@ -0,0 +1,4 @@ +include 'ddpg_target_navigation_states.gin' + +import social_bot.tasks +PlayGround.tasks=[@PushReachTask] diff --git a/alf/examples/ddpg_target_navigation_states.gin b/alf/examples/ddpg_target_navigation_states.gin new file mode 100644 index 000000000..24f75b919 --- /dev/null +++ b/alf/examples/ddpg_target_navigation_states.gin @@ -0,0 +1,61 @@ +include 'ac_target_navigation_states.gin' +include 'ddpg.gin' + +# Goal conditioned task setup +GoalTask.success_with_angle_requirement=False +GazeboAgent.goal_conditioned=True +GoalTask.goal_conditioned=True +GoalTask.distraction_penalty_distance_thresh=0.4 +GoalTask.end_episode_after_success=0 +GoalTask.end_on_hitting_distraction=0 +GoalTask.goal_name="target_ball" +GoalTask.max_steps=50 +GoalTask.move_goal_during_episode=0 +GoalTask.multi_dim_reward=True +GoalTask.reset_time_limit_on_success=0 +GoalTask.use_aux_achieved=True +GoalTask.use_curriculum_training=0 +PlayGround.max_steps=50 + +suite_gym.wrap_env.image_channel_first=False + +# Networks +import alf.nest.utils +actor/ActorNetwork.preprocessing_combiner=@NestConcat() +critic/CriticNetwork.observation_preprocessing_combiner=@NestConcat() +critic/CriticNetwork.action_preprocessing_combiner=@NestConcat() +critic/CriticNetwork.output_tensor_spec=@get_reward_spec() + +hidden_layers=(256,256,256) + +value/ValueNetwork.preprocessing_combiner=@NestConcat() +value/ValueNetwork.output_tensor_spec=@get_reward_spec() +value/ValueNetwork.fc_layer_params=%hidden_layers + +actor/ActorNetwork.fc_layer_params=%hidden_layers +critic/CriticNetwork.joint_fc_layer_params=%hidden_layers + +Agent.rl_algorithm_cls=@DdpgAlgorithm +TrainerConfig.algorithm_ctor=@Agent +DdpgAlgorithm.actor_network_ctor=@actor/ActorNetwork +DdpgAlgorithm.critic_network_ctor=@critic/CriticNetwork +DdpgAlgorithm.use_parallel_network=False +DdpgAlgorithm.rollout_random_action=0.3 +DdpgAlgorithm.target_update_period=8 + +# GoalGenerator +observation_spec=@get_observation_spec() + +# TrainerConfig +TrainerConfig.initial_collect_steps=10000 +TrainerConfig.mini_batch_length=2 +TrainerConfig.mini_batch_size=5000 +TrainerConfig.num_updates_per_train_iter=40 +TrainerConfig.replay_buffer_length=200000 + +TrainerConfig.summary_interval=20 +TrainerConfig.num_iterations=1500 +TrainerConfig.num_checkpoints=10 +TrainerConfig.evaluate=True +TrainerConfig.eval_interval=500 +TrainerConfig.num_eval_episodes=50 diff --git a/alf/examples/her_push_states.gin b/alf/examples/her_push_states.gin new file mode 100644 index 000000000..4a56e3aa2 --- /dev/null +++ b/alf/examples/her_push_states.gin @@ -0,0 +1,2 @@ +include 'her_target_navigation_states.gin' +include 'ddpg_push_states.gin' diff --git a/alf/examples/her_target_navigation_states.gin b/alf/examples/her_target_navigation_states.gin index 22aea2506..469f19942 100644 --- a/alf/examples/her_target_navigation_states.gin +++ b/alf/examples/her_target_navigation_states.gin @@ -90,7 +90,7 @@ TrainerConfig.num_eval_episodes=50 # HER ReplayBuffer.keep_episodic_info=True HindsightExperienceTransformer.her_proportion=0.8 -l2_dist_close_reward_fn.threshold=0.5 +HindsightExperienceTransformer.threshold=0.5 TrainerConfig.data_transformer_ctor=@HindsightExperienceTransformer # Finer grain tensorboard summaries plus local action distribution diff --git a/alf/examples/sac_breakout_conf.py b/alf/examples/sac_breakout_conf.py index e6b163393..dc753da49 100644 --- a/alf/examples/sac_breakout_conf.py +++ b/alf/examples/sac_breakout_conf.py @@ -12,6 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +# NOTE: to use this on a different atari game, add this flag: +# --conf_param='create_environment.env_name="QbertNoFrameskip-v4"' + +# NOTE: for lower bound value target improvement, add these flags: +# --conf_param='ReplayBuffer.keep_episodic_info=True' +# --conf_param='ReplayBuffer.record_episodic_return=True' +# --conf_param='TDLoss.lb_target_q=True' + import functools import alf @@ -82,7 +90,8 @@ def define_config(name, default_value): num_env_steps=12000000, evaluate=True, num_eval_episodes=100, - num_evals=10, + num_evals=50, + num_eval_environments=20, num_checkpoints=5, num_summaries=100, debug_summaries=True, diff --git a/alf/examples/sacbreakout-lbtq-Qbert.png b/alf/examples/sacbreakout-lbtq-Qbert.png new file mode 100644 index 0000000000000000000000000000000000000000..f839c95cb0263cb61da2b032e41d76aafcf2d38f GIT binary patch literal 85159 zcmZ^L1ymi&wk;46+=IKjySoKVAhVm8PQ_)YsS38yushhP8M35*ix$MbD?dbGWmYXq>22;SQk7GO}mI}jAg zNY+wR?(XbNg)pK`N%`ozfG<=pg);eB=oSx|7r%QA%z#pOL9+9kh<}FPF|JU(ki7)S zBz!-nP`x~AyekhX?l(s0^Tx(M_f(J}SlGBY$hHgzIb?NtkYc?1n93N&MW%^VTo9tv zU%OZt)fFVW20I5s{lGu&A(U>ha{fS^+!5;6wMF~x-4$e@+O+J4;FEHAc+>`D z0~$Citb0u?tolqwDY{RAyuKH>GInXzD1l?!y`7!zXRogl-{9eo6R@#f zUl~56`Gojw3K4)6N9eko?;9TjGizY1CT$`o2Sx+jLxVwpqk_Ey?!bXxun==F$iMEv zz$kz#7}&e`U@$1)iVFNG=0N;UDa3ZpyZ^Zd*LkZbtRyNe4P2EB?TwAC9n5ST9c!2t zffeO9S5|XWlau8!w6S8)H?lD>W^lE#eQN^7=gI@zS{Xa)6S-PhT08K#@{{~kg9o^O z`Tgl(e z{ux$ogl5>jmGeH@8+m%mhX zE|$2V2%Q<*Eh@W;Wo6qyyxL?3d9qlPfpAoRMQ^ehn?V$M00btw*hfY8CMn59m-dO% zUoN*Rt8L2+4g3@PzdXjjv$C4*xr~3mN;l1NGs#TFG&O+6f(Z5{*89)J0^g6q;yK(| zShN$F2@)RsKNn=k$jIe@e^mXe;VdeIM85@$YfITbdwDA|TFLrH>%aQSNk(R3(UIv~ zHb(fbBGTmbLIYxr!QwyS{VgnZgXw=uS3(Ym2SPn}R{xL0f6G7E$p7wN zV>Cx`y_r*8ZM55-&iW+;R4mbie!8AfAZMIhf4Sx4v{~aD;#_}Ht<=%Jykwh&FO9t@ zt!!VycbpWy$!jmpun#|)DUNyifJAUy(YOW6X)>CcG5oUr`U0ZYZRaa)#25nR><|+o zst~{QpyhIub)peT@TsbZ(eMEmB3Q_`(|Oq{Wrt(fZk&5RgGl1fswfiwEdSk#!+`<@ zmiyHhXUlh1)8;~%l;-mKrD3nTRhPNF>*HS&&7?7W(^f4f_162DF3v!G1(vbvgPB!K zUC%haaTiK7m&cn*pOcD?mp`7rN|Wy0eW2j6HoWgz*1383sEs^#hm%eTU++U6Zb~vj_&hs-$~7Cxh7MYF9jC-k zyWtoe(I{kXNhPBb*Jl(2=?{L>JpqD_rAgK&95gN$XM59lMq51HZ^taggTfr#eFPtN z&|EJ1Fk3KnT)WCWk}&C4J6>)W@1F10TiR}yMm{&`K5b#zAHF=ErRax`aV*P5AAA>D z^11z$Eo~E~gS2tk*?AArwBgtPbL~ywx&aOzMld}<=yUvXT`UM=r&ISWY1w{+ewF8_ zjMPowq8H8n6zJL|<8i|0W!rhhPLf-4p}b-3P$CLv*Okcq=8$}IWtM>7y)n~yNxC~s z0V!Zj^K7kcCfVwArRlV0guy}A`=QX0hsWt~r~P3M_ikE-u}L`KgLFIm$d~=>*Qfnx z3x`aXH8;un*)t@c`v8lXqNtYJC5u%_LN7Ao-;=ok1$BaNil_~VBg!j*E5kec%Txi> z?&ZH%jtaWA;_+WjxGi03~#Sza2&#&u#P#Ap$ z(ss$LeO%#QT7n|kgx>dExtYbpemByNpyw4HqqhY^aAec4T%zk(kBx-a;; z*mk$(;h(F6S>ZlH-ypU8=LV!J^Z2Qy`JgC$_=d$!wXI{d#l?jxu3^z6U3u(YNr9g{ zhtTu9%l_T$T5(yKtUE`1%d5xrl;ke>hDnB9KlT@2pw3se=@l`fR*A;x16}#w|KIQ?u?bF6>!~_OpH3m#rFB zcMR>;-1eKWSxsWg!jTb6VlQNkxBYkkUEJN=1J4&dkD}b>BctJqOoaK)VU_0FW;BF) zV8^1|gA9*E%+JQO6=3pqYCFC2>z|DFa}*aBuR-wIj^f-Q4atM-n%5jVH)A#Tnc%`(npZ~+sgFdW7nDUy=d@uEd#VHJIC#EO)M~w%*F*)q06SgpgoMfG#gn-I9L3uf zC-4q!m(mEokaQkgjg`ozLv=4rhnq}yEUWhjs^ucmd_%u$UNaAg7gmyfi*pMTpJp{pFhTHod zk=W9gVQlyy`wY^U&`A~DhB?flS3sK&Xi*NYhKjmTVeS^Lx?c`a!eHc%#`-`$4Z=ad z*C#>=xISGB@D6ee>yQm=Z@N=uG;vUHQf4~F7f>>YWw3m?L*QIdZ#lTZQxAkZM{uMst z;{F+6I$TBt+3b(mZ|xEI55hbwft8GY(8+@QYl-K5|er7(UDmg5lu)iZE*d*bC`=_4scc zxk0wZGwn$}8G1VkJY30xpLv zlc^xCOJU^eoUbGu7GF z07rOgCNV#z#Gl&mUL-vXfwHFrUVla>SI&TNsvcgB>%`0~9<{y? zb@8v=SWe`tj%`wXASS266|xJCH%v>ZlxKh%iO#~YEE)t zoJL z`++nXR67#$NwmUiLdZa$Cf+=l0pzQ96e~fd37)WtbJ!V6b1Ts960f>g6;*~L@jBJK zTn+e<$|RFm%gmO955IFAH}Z!iVQ{6OodBBzi6;xij1X8t`aKb4L4(CoMsxJMAq;3R z>jnASsAp9qm%hNN6B0oZeU#9zK|eaeKzU3J>pdgj+Ge!aFc2m8AecT}7t^`W;ewGJC6D`LfvSS4tchPEq zhe-ci;CuhJ34zDPj;CG5IJ{B<1JxKk*mo4EbwssET5#}8QPH2AR@fsIprn;2SBThJ zWS4UFL4>Ce--&#gzQ?V-PqU)&I|uJf#=*6B$1v;rf%o{i%QQf?9V7J_=1jqiF8QLTLd zCr!}@*(~xTm6J9Oi9X>Jg-x$^J&?54bPZ2@?|$+&&vJ9S^vaLy%T^2$rqzY0cRbqd zo_C4rQXfvaopT*|*{plRYk}9p z;UjU9Sx|LuQJc{A_@JgT(Y4vh#XUrf;WP=UlkARIMeF5o!J|kS5z**jP@&awy)^H& z=Mp8o=-?{VKBLdW2$^Z&Nxpztc@u>QW#ANe;xNwzs!kXMz9cy`jK>;rnFe(^2P>YRPR|Kv?OTl`|U8+noz>K4R~&_Z?%|I z-|qzpq_zwSbhQ^JYgxe)npx4lH-ppdve7q@eA5)oc_Dh!#wBsFhFCiR_)q=j5f|S& z2kef@8!#$Ad%9wOlcSU!G)1K%w5K3vzBGTc&K88=nK?sL=z<2TsLoG zs76o;ZFnaJ)W1v6UqYyz9`Hb5*%e_OOCE}ZwnBqU9Ljj8S13?D4Nk`O{+W!EAzA|E zsZzILR=%08&xhx)n3W&RL&Hst1{dWh()K%1ZyF-{T--|V2c-(3trFYQ`MOpU?Hy>M z>0=BtmD2o0Teey=#13w7J#J0;`=P`v9gGTBk!75dYGMCs1=D%K&J6EA2Sb*>y{2$C zJ=^1dSeji<&0Fk)N5vtY?0FzW*Z`xyA@q~A60bFJ`hg#jOux!J<|%Nc(dma2@z++R zhg1yRGC}&1Il>$D06jcjl9N=3-;+T?W6N?W@4|qCN${Q#jytn|Q>&RT?_qygK)fkk zhjE=AD;9I5jz?M*lgVGypH!1a!ck8|^HrrkPr9vwglD4N*eU2!zhII$`9~_*27_1G zTvj;r8j+=BeIiD$XZtH^oc#zfA3gW6Cb<$gXMDjl>|`0vaSXn+KIArp)UhYbTwP z?Pfr{oGpLf4pPE2iQ?h@m4~DdMf0?>VOq17WT} z9Ux_AJwTuKpYQ9al6x`?kBk_T(_kR`d2UT4pd-aG=};ea{}O_dJgKs+nqckirf?EJ zABtR8%>rLBj>cw{SQ#>=CA3bl1_CR~&&OKe(20MzYn|yD()CW%oHKB~@g}#;JJ%~h zep-!2*FcnMxJ6b2@#1#CB5p9`LV zXt8)hM#pS!ssuHrFY!DmOi*TpcZ)ZH4~UYj%C5$cGKFc|B}abXR#CHBi9#yT2sn_-)vXNv*={Bb+fDw` zbd7mCe;U!d9}#iPvk0ffvgs+H0?;y{*^c;h>BM4B)lGL7HEr6yR`mRm#J*rnV^FAo zX_Cl)$KY7uIq_&GQ)koy6%Scg{Ia0-;C06*#UB1( zB*Ig*-eAI)pBMWpScMcz)%~G{ZKb~`C7CNxV&2jx%&!+Q4=FTznLPu1Frg3V)P}`b z?%zfeVDkE&;67e0b)O_wbWs8YwxL6}}khz5=-b^s7L!y}AWv1u61{O4pB9 zNE)LFiK%xPB>Ei>V>SlfNfW6q2836%qJ6CxYe3kH0yVNtI}gCr46&qJgME)FY_m zQp*>{)m;g`DOkDpRXjUe7W8}Zm+2_y8uQt??rFNN(^uOe$BxckMriWgZH*O8h2CBD zZT0I+VQT-gvO(4n@QJR%6djt)DbolKIPbM7?_*$dgg(Sb_nWTz5Rv(;!E_)etw^%S zrBI%|TOdtbbU`+3PoNoSpZU1$c8;H)dPXe>SH6$YB=e4Ns0b`RZ*hUmIf< zj#9fEO37r9)%i^Vioi7|@M7TbQ<*C^`mq7Cx$ANDz|O~!sG7qB`Eeno!B1ZZ7_s*x z9m2}7G~|c7RDwL!v{^s0&6t^c+#XZ!Uf_oq@$+?IrfaQ`n@khbh|y`-vh|kb_0mXP z?@T$mf1zx1U|U%1?chR002iG8QT6F#ng&~~S>q3+QjrG<3AVgQ60zr!{9S$^n;@<^ zVqdorJ`YY0ZrpbarC-p^sgq|c?vaC|qT(R39EpJ^$6G3s`@{IF0NXY6(pOs(80}%- zVEoOcUP|~ehLK*0Y&exuQUQ>}yIox6eHiy7*Y6cw!#~Xw)Z~sEx1-TilmDy~w&>Qtrrm}vOdfY)%lrG9@bn>PrVWa~uZPpv zXpTx|8M!*cDD0U(igyKLu4-z@4W_9-l#1o4DF{-@3P@M=muv~+kLp%Mgoq+j<*Gzb zim;$dF@&Y=kkX>rjf6EJNb_+; z>@KD?2(s&hKtnkX)^hO46Y;Eyg1w7HoZk!~WZ1*?X6su8Qk7F?{w_|S&Xu5GLSQ>V z?`zQD85U2lG_T;wCNS|5GxCriYX@^NR}bARe!445;P9-t{VnmFCiFfJsSQx;q0qQGb+n3(EnW zY=)PL63r*o;*y?4I!1^B{>o}#dfz(N>AfErwxq=m9E)1F*bh=jJ2FEA&ZEePkRF;z zmOK8QZR%mMUu)fSg<7j=w?%D%WK9GI0X>O%&F&Ak94D1AhWgfp-vACVkm+&SWIy(J z+M*zZy>#aYq5qZ%+^uN;b68OHDp&bk5?9`h;F9CQ%uR-T!8D&=vL`bdG6a-sy>=`< zjMQZKQHU2;zt!`u33H)dNsyvj#WV#|0CS_u{~gT=Z(T9cwsG@^oAaL7_>9MXuL;MF z#%#U9`YkrTSStx8qSmL@Cb=)X@{$ATkV8_CV(c@&3XMTl&8GMsR%`EVSjeZY)hEam zYH>UhtJ);75H{6six~3EO&y`ExCFiWr*4nxS{K9hNWxASZ6Le%nO57#MJ)y{gW1Xb z-fWdQZ-2|q7=pHfAWswooH(?@ccKHSA{Qpl9IV0}&5#-h@{gpkQQF^r;#(Xy&hP%A z9O4pvPW9a22a{t=fF?qg-4M%<+MtF&LSB>CtR09Y2n-8}?UVI4hkYox^6zGv-%ZhF z*l#Vbj3uIwGsyEFlHkbw<%^LL=4qyQ(do}Y$PzRJHURHtJX4>5QBIEIVYJcdXOP!b zfI7%uaaaShlW1{(l#pl%!EH#zkZ=%LBrYnfg~NVXRGnYeH#|g{O;?cI+8E=lBQ#If zv+w^XIuQI}VOf5pK!Qo6+jX%*m@yj#vUV$#$GBP7)w~N{B1k&cF$1ejag$(5-JfeF zXB&~GmumG_p}AA~f%-s3FwbYq+Ot1p`|*@#FeU;|t`buPy zVz^Nfgl;r6B$oH0h3@f8nj7Y$zxK_%>1UpTIKv4RX}zb-X98lX?`HYtGqJK~IT^A~ z%s68uTt(HMHeVkw`^_yM*%zbPIp~gLyp&WnjHK|#2t<(3KYM83P8o>AAAskt_91@9 zo`fGNm4>{4MPc(No+62<;#<57jQm^_+pFm9ea(b27&{+k*lM=%gET>JT-a?8?FzaM zeE#11vdGN&rm?90+hxuT)tuAxW+wI7Z?7C#wp2ZWPi#7_2vujx4V7m7Q_zw#5@(oM zvggJGbGOA`nOBNh@Xkph#6!v)Y8UtB^)LQO)RxW=oCz!^>Y+QBdGGM&#Q!x$|^%LDKM~sGel4AY9Vn zo6Y`RMG1=md$6Ir0&yTds;yGfqbCCVYI+oj341;j1p*wn#Q@IyvsQXtwX>yFo)u^~*tgDbaq@@m{w2lsO{u$z29R(3CwiE3 zN)nsVsqPYCM$Y>0DClnp5QhQ)5OMds>NJ38?O#x&5Wgr`rCwulw7W@1_$3Ob3XhUkRqRX(ziF3}@?oBeF4(1~%iF4I!y*|UT#i%1 zlgRNiHg5RuFs?f(t9@k7Y&rHE{+wW^PK>9z0o{woIoWt_L@Yhe&?SSLcUSrYZ!!D- z!as$|NjpVs%cQ&9@tO}G0B9{u3n>U z{&-TI7=>&q(NR6$(|M0|r!RQT@0G{1jx#O*Si3Psz*eCjI5`5mog)f`EIJFL4GV$H z6#$H>`2g!y%|&=*Q-yQ)7J|IoHgv!|7-eE&+>Vd|*_ZMy=)m^{Y8x>;ih@tGHf=Za zHL=^-FV~dRcd5*Vz2zM33+nV<=~ne9QCBLOR^$V3aGm>#-2lGzbb<7j4=ZHbMQQJe ziJu*v)s6yg9AmG>c@^f--L}Fg3uMbwMeyyqMm0zi!S$85-Y`ZwYp2=Il0$hRW-aP` z7l4b0`uYcmIf_)H{OZks0Dq=Fz&eIgTQKKeLQA3K_m7u>K7$&k8`4 zt#;?Rl-c^Z| z$BxNUv`8Q`8{hC2WHV?+I1b@5b^ezY2?`PGRL_&nKh8Vw?=E60iBc4Z;HJ4(iL7-f$3RBL<6a2S_v0`&I1$>Tq_O3p1x zXPF=R6!CVf6!cLS6k$e82 zmy8v4x$3e`PpdA9I#k@+x{|vXG+B&RJ01IJI5-(wjwSGWQXe%fj((_5eNz9AZU6Uh z@q%rLZc2S-{gGki5(_W$767GyS9rDb)gU9mIe~cZCeM-Ql zaWmLD+v|2pY0jEB4%f0un1z6>UM`*E4V6WkiB;8c>pN-M&obDHi~M+u+~D*2A|h2R z`<=uDH>2pvw+U_!6WMGyRl0t`0Imt6h^i+Tp%fK?{#U&A;L-iXcHuU&*(4Eh&sS73 z2&n-6rcdhX-_rpElz^f^|7A$11Fa%)OaLD-Ff92vfHM;z)X6cA4j(m&F$~pbVQ?aY zTZsqE(Vpo@_>mwy?gANM`9HZI09r>R@(t?&jQHsrz`Vmphl*nZeRDBD$cTnLwL~(O z&RHpf+T(bfuP%|_{%wwiIA%fF=k@sx(ZiIjJ2fPSxZj90L8$&MiVWw#LJjA@VMQPA z+ff>%Eutz!t{q5?)bhHSJMDajG@3@PCOnwrn?JRtAA(;~G_*^W=03=opclh?$~D!U z|2$5Xp;Qx14Y#AsUs7wczIGvc?cF!P%(6S*+3mI<@}r~#Ii1!$CM!D*4OMIh`T1?G33o|XZP~`nGj~Hd!$9N&DT@D)IEM6}U`z*@g^Q9lVc|*&|a54=FKU(=G z=YF(G-0KUuJ*wvOy+k-x`SC1zuWvpp3OQ=!HsTp0P?fp> z|FMtJn-~%iEwV)KC@Pah!yzk)wxUraFK4R#q8~SCM||ct^8{~}RH{+ft$PgrEpkna zb0#7f$6{)eSY1O}7QJy#JuVu0pp8ik0e|4{tLHvvF91PKbU1ina)23=YqaLrZk`2d$h*>1?s6mcw{Z0FTY$te^3(Zu-O zJ8*7bGvM*h{eR0Y%I28BnfB9*Qt`P4LH9;scl3EKT*wSZn&D?>+Cr z6@%ow)k#|Ym_a7C7atkozV-lpY=EXh%#qz!3zJ4@^|7bL6@yCU(}j1Cv0f^Eg0IN> z9aH6(TER43>NWhah;n{hVw|QE%tw0Y$}dVMSYmujaXX8C=U+k8#fBq~fmRMwsrn^w zViX2HPwM--UewuJaEdf9w?q0+z?-Ql6z?2#HNU7``6tb(Yps7Y&oPF>_b#iZhg zaFx^7&kRGOYxscd3I>_q$_iTPBm$DRMU|Er%+|= zkmsj?oQTOe179})_Rn^f>sDA9Xn>q}q$_TO2>kz3j?cwROf=a^P@Y8wosG(qT_MG{$>(KM0jh>Y08kg zHOQaNNq{)AA+*DG>y3Pa{<=V?;m`>V)d-xsBVIs=hUn5?M(@w7KP|cwHqkBUvhLLY zL}&YW7S*z;Ov1xV2EEV?+tIwuKtR8^DVlD)_se{}&a!?C0emKMKRBU}1%N!=!@;~= zE1ULk9&FE1V0Yj6Iq0VhZP#Bo?RN&x6W{P9U*f8HlvH+$WZwM@hm{}2S%-++JIDiD z0)xvo9rR*29jkN~<4Yu;yPkG}FmU;8p-n^ndKQ%ctf4TOu2$6Q)liCEj5AT^( zh0Xn{q(1g$_qnRoM!k0-T1WP(J)EjnMqSkHg2uvT#Nv@LdqX0?!Jp1+B3WqrgSyYH z*W0~}gE&;ai9U*@{%ih%z(!_Wju$>-yfYG}dGhsn*iDH>rv@q6Ffup|^k^Ez9RB9;{Flsyiwv4eKkrexh0gvEFqo_gG+PuUFlV%4^CjinyABV{kdg`EM5tnLX5RJ+Qz4xvtC`Mm?DKf_E>~Fc6+mt8vl_Fns1R?11-hcUW54{KoVF ztR6i#Gn%HoN+cQKbj^wvbs!=HtxQ@Xq90Bjwkfr8uxzu_(O!{!CJn1+zPKqBLn1>0GmzLcegd2q<*;-cFdSM6 zlOz(x3c02#6uE|8r*`A*00r#>&yTblP;MpGYJ4e2lea0bJ{1jmT@chDV8K&Hc7OE~ zGZuV&$?nR0N;Oi2tu`g~`Ph{mZS^B7DvRGkMH5<9Ia=_=m3aG{F!UZ6B^?=_^;N0v z^QWaUS(!FNrD*OUVJLhu=goyEfFhTWC)D_smHmVK2iK88m z%`HmFXO=7(C#KI`=txm)I9L=6llM%UhkuSLT>yF>#$&O7*B>cHW~D6TYXBnBbr1T! zrPfG18H=3ieHUf@XO&Yr$r+oogZQge`K&Pzhgi>+d>*op%i>d~pe28Y%IgU)+Esy9 zvs|iL$<&zR1&vDB@nuIGe6r}Wi!sO~<2Ln{vsRA2Bwk=DPdq}571ZW&C-O_GNR)Lv zROAO!z=}i6gZD?;PZ_*Uhsu^J(VE#{UwuP1pY`Nx)GeUUX2^!L|p+G z)sp^_$Xczj?D0?axt4OFyfgyS-k6!N<8r(8X7s=s%_w_X~9!+K5iVtyI$h{KY74GF<1XxC~&Z+9v{`W_d z@cL#Jdt>SQ(2UAd16zF&aN_~)c|!)yj=9}85%}I>tdax=!R_|+m^T~S+Egexak+SG zY(cO~d}NtT!WC{mlWQjJXLtCuuLrfiM2W|&Wq!#F&`#5TTA<|b5#@s-ID zuNOL+=UK6ctrXlMmwaWBuk6t6@Rq);2dHaO6G*re$D@lRmMxA4TRe`xHxT*Gg}#^g zau@=vMEjR$-c#h)RI>$Qkus^9HzF%QRaG88A_Q$0cQX+od0gktUNu=y9b%?I?b6r7 zUz-tU2PONtfBB%ZU~om`m20Ghn{6D1Kv+3G`6G-bw#z`B&1bP^lu%~Uc0R;63`gm_1w4a6u zn8?~P+NCAVB?Q~=I%QmdbsUT0r$N3QtLBXi`0~ly6R7?#00&ULJ`pdq?IS-oe$AV@^(vcK9p7s9XCxpb0`$a-M}GcC(r=>xPavzuPaFtix6=(I+R z$PgDP`BV6h5Y$qG%T7lyJXrGU!q#G$tVT}r@b0!#yFY!FPq(Ox{dbJXT)hZAN5lNa zwS}pVxp!>po2X}4FpSJO$(h88bP&kL(k~%GgAJ{u?b)Xt%~uE0%4qf1BQVx$Bl%i= znA@v|(b_8?cu)3tPF-Bte;n5mD{?{9iS({OTAx^t@&0V)SE)16;OW0M~R9x4IqNtIEc zA@4Y&QU+7|-D`{-YpUkc6t6V^z&XOkQ(F!o4-)zT@?8X?x{K;lRKg}8q;8f{km9rh@g}dg*<9YQ z;9ur8*lDI)#bLyX>&SW{6nP0OixU#v+?67&T*Mty&`tgZnt+KJgaPMcp(nViuxRP2 z7`ivM;YXp|YKp%w7!#i^?3hegSMvN$w0$u8HL=!|MD7 zS>7x(*}sU|B+!96-!mdSOn*jhVNSL@k(}|=#TPWJ>SNkzRIS7ay>9Gd!}w(jEH)Ke zyBI4xRkD~^t0v}QicoXjK^LW(CC{Z`27hLI+SFZBfwL&&9ynwI!u|3K5HpFx~TlbsoYdh z$Mxn4+#^FT=hW2GYcJj`kcksQ1qGvyAc$NS8Rb1R)P9vS42D^_=gcf^jC z>AR-scx6&Kwt9_y)ty+=G5#r1MHb#~npDrI^*U7}I#@^rvKxsAg?P#=6IQVg38uo8V%r!VR$)=@%{RE$wQ`~R~(?91| z8N4q~*V6e*xlAhc4*IQHw!#q3j!&^WQA%sAx=sgWC63Z-j zu;$ILTE9?P2M(iZKYW9hWWcGH_u?7|s5<#6iEtUrhnsi$h-*4_GC3t&!g9VDW8!m?&(JYwcINp$ zYHFL!*1zg>lj;JF_Iz;fe*gaSyvJ#s&>?_3HWiKF7Z1EL(rTorArUNO0(q{Iqw@up zV)Oei1?~W6JoTp%Li;FK!W=fj+(fg*LHAF+8(>x#f|oCQ4*OXb1TX@B6pRBe&Uo*3 zvNH`W-Y8h?A-lOlmA&efNY_-~`Tq;xUB;A+k>J97&vTAG5Y8KG) z`(NJgEJQxi(h2qX>&VJ7sHl0lE!CRWEZJAP#k~OTcY?j~TXe$s zxO(b}+gUzs#WN0=&EGt{n4k4zT-|QhDny(PIp7kM=} zR9{udB;P$YMXao_D?NYfPde)`QtKp~kSLbVgq3Q#jSoD*Q`K&Dt>zo?`g2YF6S?-g z0^da-E}v`F5kO@4>{hKvdIsvpwb4=*Afq6}@S-*O%tPM`P|L|n`!JVoaj;d%tSq2s zks-|bzIO)plUGWoH6=p*LQC>3xLYvsO@CQ*9oP~+8Sh{rASD|#>&Un6W+SX;beBV} zd0@D(s(&?WPdEN9_MW)Xbh^$$RnTaaUH#i2E?J$APst~{ZkkS|B3Wu_oy5R78;`qF z`sJ-pdn=LHtbw7Xnjm73g-|!~4)7Yo1iN&}`*1Ka$ zU$S-v9T0L)**Bx^*{6~{?AwGx5Gyh%%}!2zl)E+Y4eVUQJ53 zGkvmM`Xm_}FI9D21Y2+Oj;bG2DjsbV#$Fe5BHLFFI*jvnW{B4VrUeV3!nX@F1MbL(quCJFJ)dB;L!jEjTIi`)ui@R5?j3h_5DX|!3lKbJ+) zS6%Uu8hhF>GJ4RxC`>HY3jV~30nq2nvOo|VkGy;++s!R2W{*xjIdw14dR#I?rp#9w zxPL<`p;1xHvGwVvhs}|o8l_ZtG@&~q%){^>9bQm#*|(n*NA51>yvhg-$bg4MMc^l; zsp#EFM4Z-0x3asCzhbgm*+S8fsU$61y*OV_G&6U@XQVOYH?eiH12io++e?wDisNV=*2yMjA_$QpyP-~g=>s7qA)Lj2a4dC%B)HBoUQSt zQH$@70AAVa_c$xxvj7Lrto=S4e#A~nD35>8l1o=F`pOD|bUdcZ{^PBDNjwIt=!$ z`KVK+c(o!;6ceM7WF}tWrWiEh~_De-#C+Zpl)z?`BQ7CBs&E)|R9r|Dq|Dgx%e5jwNjNDC z+J}Rsin;LR8g=&Q=viXuR6A4Mj4@ijAwN)kS0&^#1ck26 z7=yBChn8K_2gNg+ThMJfj(I3qv@S-qt#6rC3yjvg89j6NF}<0V#?05{PJaYa3K;N} zX7bA?T7E9psmMgi=J!a}oazpVc@*kawX8|9xH#Fj9-=(sy5W6!AgAP|v*hM%>HEMS zAf@H@u-I;SME}zkD|M9N=uqYUsCL0l7qqzFak+?MJ%08=6h}ScE^ro}O&BlR8kS5y zs9NDLTWGyIO>X^C_Pyeff`2t_uYLA?tHsL9mjx&aI7{AOA%3Bp#StXlhB1Pslvy-u z@-4Eh)w8Kpmr%68hh4J&dtZVO64(z*Ti|^-U zHRrpZ$Db`}xh1o-Tz0c0TZ(jJ9L8RsOLnT)Mk~p@MoyfarrB5YRPTC@(pxxY+HKRd zTxtu{?@H}Zi!(b;f40Rq)86q&L=P!kk?Y+YaPqAEL@!@%L~oB-sF2x(l#-4e$X=V_ zi&s+?&G9DP7_yjAC9uPF6^~dx?#?)5S$#aqI%VIIiXrxBCKxdx3**usX|_3-zg}B+ zGpz0wt6LFZH)?gNXc8LDH{v5Y!dKvuTzZB*3$N>kc2iV&DkGKXeWz&IFppxUY{a(| zA(=i3?=EG=>@QO$a^>4NB}~P}ze|C{cHOt*_sHVix08EAl^DqhymH9H(U&O9zSXzW z1r4tD@lad%bU-F%h_r49J4MCd(9|kIFJ^3Ka7uw|K*6%MO+7U0E>6Q;Rjb^`+l6{)|yw4dBkN9*(8MYh$D)y)ERHRo&Tw6kM9g~9i4JzXMd=1Cr*mc z;O-$VrIDWU8N~juFDf6(MQAj9G=dbSpkq<$cTq!NTLUk5g!^^agC?m)P&9{`C>G1= z+w1ziC*>d2q&))cLk2YVwmUL1^ViW{@i`MysT3t!vC<571WsKPnl~eE@os_p&agDt9A+zku>Tsg> zRQtj!@aHRt6jc({{@D^$E`MOIa3t9jpX&~wyEXwH)mMq+U_|`G`Zf0>9z_~jk$efF z3;FUzPh8chrWCXju0X!1!DapGh^Edc2MG^q`MpzAz)nvG#AUC?cE z-dj%#9U0HDK+YpV93nE=vEz8LZ66W38x#hAl0iiU1va2!x^oZ{kfmpTBJ)9zw|*!I zdax%M3#k~Wyp1P2TqXZ5fFs!I{HcHq3g#}vp5mvS0$N+edwu!94+lUx5{OScDm>1+ zpk(TQ8fur8CKbOBSM^Wys8t;aU-*j}J&;3Akbdf|Mmlp*_B@q{KmOL`h^_uV{|lhE zxn_<7CaMGV65cp~F2AjxyllzEG&D`gR;u?g%2S}2?S>fjM0J)d@%_-B+Y?$ei|;)> z{+|PZ!6DsP(iyX{tf`&W&iyh$R=h&Y<|ULOZ=pxL0^YtB6a3`P;4bUxzwm#EI_K~@ z`Y+lyXqz;R)!4Re+qP{RjcvOz8#`%i+c-gEHP)THzk8qif1aGooXL0gUVE+2PU*a< z9E=DpimYHL2;xTU>Xn;Cxhl+!W$hg*BSu2Hv^Jp{8Ia6{mMV8US)4FfiaFyNqe2Ek zsPpXL7@KOXw!OmpB{WUp=t3;J;mm^lojNR-4p(Wt!V~_bOYSQ$t^6#I+!-ciP_0M~ z*LLE8 z-y6SkaCFW2STb`%wsTm8b6R>{ny>4Kc?w}=&%>@Q0l&$xIN|qm3VpYyi((Q~BA$iH z&+137tv2e3>@-G7kL!R;-Y1qL`oixE1=;VjYkr4RNlesU#INP8_3N_~iR+^^-Hv=O z^jnQ9uJXj9XSjFA3sTx1ynxWEh|uvD0l3t?pidV%yE+pTU800IdmM>4Elr;>^+sCc zm|oY3_HhoB*If+6qEWZd6m#aWEe}fl9cM%VT`R|Ym|%gzMis6|nq-%5FRs8>q;)3* zOB*N?3a8n&yz8F8&5>$TmafDg^u59N_kzx; z2M48unn&ekRHn?)9jEB7zuh(M>2kc=XkCvHkTe&w6E^Js@RuH^LE4egYzyX%ZTk>C z^0ce=9f&(AOdHd5n5CCQBEQC-AoG{f1*rnHl&X1Lj^_dSUqVb{<{JI@gl{~GbGO!P zr#INZUf_8<^jZ1k9-V5rS8+iDOt4h1K(k zcFAz{5aYt5CdkzLy0FZ4r{KL6kIdDvVo=UiMt#`KVZG`|)J*Td)B7KM^KfP@^}W?s z>wkaWTT!v>_DxMaVk~!8h7WPfFUD5Sb@UEoc{Y0UvQ~1UQ`BBoXu4mNE$Y!|wD&aM z?~;fYk6iz)m|v^3W?rmu6!y8?w|vHOb%6=ADwfKmjWUC9Yk%a$I=ZI3^@QG8g&BAA z0McOVz%c3pNV#lm_iVPOpszLsKztp&$)@Gjlsy+|r~}$5N3K>g+9%(>WJ-gf(h7$4U?7I*C8i`S-S=7GN zx>xQHbo2AJ^c6HO->^7$@k1koCS6#GP3^>Fb$agK)&d=_rPD3HsR)TGKCIBJ16AVD zHHuOt7od}We%mR|atM0f3HRFfIORU=z;`*Oof_dtkG=t2)jh#KvqqEuI>8b;h}$iFj9dFe_LT8%eeDl7L?_9{q6} zE4Skt4kZaZ>b2mraztGK&1$XU-oSIw?R*_t{eoxRgj7cV>w{2Wrbj_|sp!j4crIzZ zCij63Lvx(J37@;x1?0*dBvUqji%w|ty4)F$$woo@V24+fiq!4-{N=!RjY z&hFvzlU@^6Mr}Rz-HI%(#*^)&o7+ zXr8q+GtebhYP1p>saBwqYO|O|)!xZo=6eEZdhELtD7-487zy_xb+3WnR108`^r)j+ zgk68A+N_Vd+Fv!B*WE74=aykg@kj0twvVzkj2y8;x}{3jW;yGsJ;nP<~;`_m1hq5y+!5?YiFL zVA~r6eO~%2hLXIKgc0&uz+Z(1kyI%;x%({Zy%rJzZ#HFa#*k!OGKpI1l<@7_LY^3H zx+)iLxYAz&ym9H-I<*Vw5=#>7<`B*x&2P~LRm#1zZFh*%_ZV~pIr6Dn?nP-i@^QD# z7{q&}XK?L>NH)xy)Ja9x6`hLEEFN9^ehbi%kH6bY80^7YJ3NGLxr`8f{C^JmQBCGgO+{I@a2CQV{;V)@D_pfPzC`C$-x;>ZI=a= zbKqn|&y>bq1`#QlL74?Y+JM!~h+-89=Z8K3ll`!Yx+G**!0zw~9O!;2D>M^W_EgCf zs#osW@2l|WH_OA95N@ti1a%=y;;83jPAd;IEgC@!g=Oc=iGmBy0sXfbrgoWe(u;

4$iIC3*{k-<_jvqLBq#} zL(JUWjbaovsVF501U=Lgi;>rAks5H$+=qt*stRL zHAb=`7j7Hk9Kr6zJ8$mdjpb5nHkG;SMa6@)r zYFU%DjaiB)LDj}E)J&Kzl)_#Oe*L-Z8B--83Rw`Ojr!@cN8PA9DwhFNr4D4rars@) zh~h__J@NCOM6hDYVU{Ir&?N7(wcAcGI_JwLrk|{t{L|^yJrPQzA^bg1nNfkm}!6=`R-ZmF;0u4ZsiHQ|$xK0@-H-D{c z`{LgnwZ3=Hg}D=a(#qfr{WRtMsDj?V<(HCwpM8b9*%uyn=3EpUb|W-DiPc-hdbqA!gbs7{<}(Gxwe})Et!fQTAB-%2){Igp z?N%2jq#w5Y48eCUn$`%Zchq4dzj2R~5JMJ+A=k(&rZRj8HBUeXb zPN#HWxV^!8lAI{P22R?I{F`FiN!zcjcx-E&BowqGZ6t!lx-P}1jcY!!l=r?>TXV%W z*{%YB)j!p1^(icI1IR@APzOJMVhDKC$ zH>D-5L2UyP@rPrlm3EO8?(WKLsjILJPdsb}cf=7CKGYpyJ)n?mSP>lTo9xbknOpM} zg~Z?EY%>$)CX#k7$ibH<8)h%WL~?!D35BsnFik1j1Wx4rG@4?nhd}PjKM)%L?uycs zlD$pr17$;S0idR0t;uAw{sO2d42N4HL?pi!Mi{mH-be02zGE#WX{8O`E}j3Q@#M|vu5sI4`wP?NWXN<=+W5RU6(Lv~ZzBW}A^I%#P?bUW&nYn>^Vb@|+*N|p) zXfHR_JsW*%j4jv|ycH|i=EZsPrnNEtY@1w)mF?Yi!g%^(Ubj~pW1hU5L(0^MBcxA5 z^kanI3GP44qe?)&L7q51E6il%^s8Cro8+NA$Q-Ci(@mlW$**-& zX@Pda>+fA*c=*Au`=4HztHDmXHhnC%#Z?FKN8Tc2|{4_8_(_ zHys%XZ_^znyk?0mcH0!!i2Ls;gH7!o;)51)N#mrh; zGX4)31h|G=e|~{=EXj7wco^;PhZOkf3Pc;*kma})UQFr=Q2OQpyS=Yt0ojqD(wDH* z1iCPay_O$yiBG;tRWF|jDs?^Jj5nrr_; zJZ%vm`n_%E*0vAgbv?=U5xGUCs>cr)2{j*ixV*l>WU>~WiAa5v(BfGoKD|+~$|0$F zvX^WG#o1;rOSGzSzxqF4ra$a4G5>s!g_~LmZPUPj4TypwW}d5fb-bH+dQ_m7izx!7 z|9C~Rd0>6jNoNZ!CERJxUAE(GT&E!+v?d94eNvFCx2R!2?s=#O z*R|R?S@Rbng)?>>_>?Q%9Q<91jGiw_C+n2G{TK4~BW?COfoXwtvGCCw*-EoI2}Nsdq_iRw{pnu#OQuaFm*%%@?$oTGm+QxD z(V_KlDa42u#OU0L zhvXpPK@WL_2zVMB6}ei=n{u%5*MwP*Xi?wLnaXm!l}}gXi5<5B`DCFb4Je14;uqh& z&j-q`(i&V0yh2AvVm_sfsYG2)60&a%9~cv*TIs)jN+Gw5M7`d5Lt|2t9rO2KKIjc@ z>U7D8vJwFXa>hdZ6Xfczwm*%g?4>$N;7PTQVjBdvl+RJ`j3l&rMZ?To0A+N(-=n0Pn?PHrT%b~szOF#xHSbMt0`j(TVW!LjNkj))Z zGul`3_+h};tc5f+dsj6Ek7!88b#23@ZPl$ICni%C3Ccl_b5EQjR>z$)Kv`%|Yw|&{ z3A;r4Ac}bQAzq)08PXCKY&@uM?R6BG3%VIb}w@dULwm9LlGVu$0 zJhlugW}KpVt36KmVi}5u0xL6*I})WS63Zh#KrLEJoS>%1t$*_4bvH+sQjp_41u~iO zOaJy|HjWcJRM(|O$uW_c|WjH5#FMHI(3@D1Pd5^LhvXC`}h4AG7kmDhAnn;ZnR2a@Z zU`EVW_cq!m)ha|mD9~vF0cI%Iwa)j5`)nRI`~$WTu{5ZSRM{HI@lMk$iOjk`vAMrB zSpWS_dcj%+TXhgV>jjKbHzBGb6|n@{l%hOI>H^BX419=1w+Q^XYca`s^y2R6@S zQ>xt%;Zug+$S~+@nKt$&kDZ!-e3)^7ARFW6A6s*yayVgan?8bggbVAYW)r@*ZS0^` zy2YKBA+$goPwCQ6+}-^yH*#IhHj@vXT7H)Za;u<3R*nHeyaiwVaP?Rb33s0AK-3L{ z;Rt^AghAs|Qn8$aHe%7)1ZW|`?CBV?siXe!k3-9yQnU^g`ah8u4iI0bz=$Ii*%1B3 zi8y3vy&a7A_a=m06dMlAM{Qq!BK%W9=5WC{XAvIKIl({{@C_p2 zN${3GpuqE{nN0XbSdHKF_4puKov21XWiD?R@Puc=z zi@T7Stoox#>z0T}*YYs9g&|>d7>+^iTF`+uOTf>452%D<9`Rh1#RPXj0OKu}NXVLe zw`*hUe$eLzV0YrZP7zYS80Rd8tb5`E~`+XiQk^lQ8e?btZlOESYcdrLo{znF{b1EK$C=UTGtS z!7)V+YVws!<}Lgq9#@`F>ZRXUa>~%{vsp7oa0kLQ|Dh)FssRdR>&$$+fSNMPa#zA! zXf7G4jRYaCY6n7K_JAOKx4osD=_ZsVIsJKDn}r3)DF1B*L484r0w4Dr~*iO zCu6b+fSdiw_%mXuNR3+6+lxbe+Xkri7V?`~XrIz~yqcZ?;@yp`6gb-NG*%nU9;woV zuTqxv{K5=leTrXIS1C=8_5rA%Dw;5bb5A@fmFjRW%6h82YHF-YWDTcKpeC+sLgs?2 z5Zf52bLY8+V{sy5D?e1Ldx&vqHt~icd_z$H5OY9TUQ|HzI2}z7;-@q z@@?g6Eq2CY)1A~7WuAk)fvP{yB3}w9r>i==*5^EK4p18*8Ko3cEq4+}6CO>5U!Cqg zAP+pZ^GYIg_t*zk&y7+1yBnrH{zrJ+*tz~VlQhudYL2vJ84W0c+fH3!BHC+E+P5_l z=f|mJt!$H$;#E&ry6;3Mt~{GLY@PPdNX$heyD?v??zC!u<~=VVso!|g^C8_o z%&`TG7j=gEjE1;eKw_&WFlw%h;@>NNi6LiLGD%q*8(`S>Pz4;)k{qo-qN?4XFGMBU zoZuD?j9{1PK%&*eR-z4akBKlvZrypQ?l))jpUlL=B`(Go)~pMXAgz6X0qde=)0m7q z%4;CrW={l?C|%r)p@hKS?-4YNj)-Cdn&{8=VeWIqGCIuMB#Llf_ms?W3xhk*GIDkm zLJdz8BDfoHgA*K5e9s?xV5&7PBzvPf4|f^fs~YaVUkoKVI5yaB0SCkq-)L9!NMqSp z8yz2Z?2*ktg}&5FP^H)Qh^@}uE7}E2KMfj8dr3q!ae&ER?DS z2xWT0ktnbH4A&^5fH;qn9R(nHLwY`+F^xwh5z<|2ba}V#sr=;<$EDm;kzOl){B1HX zJBAj5UMFY9rlnh9I{B~SU@+W07Zjq>)1=ZHCBi^PH#$Ua#%j@Z&tR)^zqwu+HRJ58 zcr*EfArv`@N{h#7n2;pmU?D9x8ioI*=P%u%`1J8YML9QF*0dQ!utsgclxXc^9PYpp zFw&xS(s{5gpJr{eT9Sp|*BbQLOn2TH2@A)m{MvgL^Hd+Fd$L&dOW}r7^<}u?hZH2O zBu=e8zlCaL9Q@YHKqM8abMOJQD>;h=Q@sQo%dZzeW_y=fys<2V*`12IN)%w>pL6x@RqR#*Hw-c%GEQwOhiBh&;LRWh zyr&E34UD-Qc7^Xh4DB>c%e~2|sqH;KzmAd62LPPhytsSkof${^Zx}JchK5jAbc@n) zb%aqJ=$YsjpC3^(;rDlVf}`wy_k6)dBk9mnmoPp#tv(Uk5?V38lD?liSQD4OqA^OV z2RznA{D0&sv)+y%*eePp@1;n%lb!*{GEo!;ex{?6RdjZN!N(|>_*u-{NnR& zRhrN_gOt&vbZtR@D~fQyBuoY?jajXUmGhLStbejpC7Q)J5eJwCXyp%3bIzFkTJ{!? z<-j4v*wJ}7$h2GMeLf}H$7Zoi3c6wQxHfhJFu7w6z+1_<8yU%=_el7r8b!;M1mtW9 z%&eMHpB1cm=VQIyha{8HAB}Ciw1{(v4)WY5+#@aY*qbM4Q3fzD(0`oP(Be-s8)z0F z*N`iT5>|hPYh25TW4p&&J5U0_TF)lu55hL1uOs5F_!0?!LgEI5Qn$0lrO~gjyRAUt znN5)RST9*?@-ng3?xxs8+F#PS--Nj8g(+0DV`yh+P;AIqT)m=nsHdNT$7F3V_>=1{ zDu30ZUsIgU}hjbLuKk|DAjgt9N#Xp~Q zhEh@aaplAfcszua>)8vEz0zBH4Z2r*mp!9zxc9fGOah~A$4;gkeO!}5fa2Hn46 z$~4=ye-2Im4okQxucaxBC!Cfa=4gfoA_EScMd`h_9X3`70i}feVuwgOfGm&THO9#Y z+4X!0A@K(s$AoW-DktqL%Z4FuhKuo|C#{k|)}vs@T=kOLSqtvT5Itt~q0JgOVrlH#0M>qVd1P|2Ouo!@w@gkK zhZnR8Vji)!5m|3^6cv6`l?r5<`#`8mU$)z=?RUB>Ar9y@oyLmU*`u#2(4M6`lBaYy zjxB>tRX}jAk_PgD{K`NA;UGazmGi(zXFRil=A^60xQ57LHeie4TtHBE8qIty&n5&- zyzh3o=`(@+wvF!dWVvPeqY720-^9p9p2_ju+WQjK?*> zC2(!j&8u5S<9F_{a`$6bZGHijRpmM9yf^%xgx9sR19jyLex4~sImQx56vAJn!5M1( zxq6R;)OmGM*lDRrA{nerrom2h@R$F}m8J;)8PAxf&B4>{MnBE{n_nW*s(;0O;xvE` zJTMM?n4BdL6BRt22FmV5Oz@o{sQTnf_FTa@U^ro9{g~!qqd5Mq3Md&B4-XqHhe;+( zicYXBvZsl%&^!!G4OnIA%L+#PKzr$;BI99H+PqN94{0ve+eAD=8o$<&d#2}8Y_KML zD*UW`b>(NZ)wmI;*)BsJO*s`| z8!W5(E{AVR0@o1@Bp%_*EHp))RzjbT484IlD70RX7(#Yu{qYc4!GbM3kw2<{%d&TG zz)jYWOZKxQ9~`a)VGzCTrSRSOQ?1QpHrO;eaQp2B<4MSfVfEe;{1g9Y^aH-}T{Mbc zd>D)2yXx&RLY@y|jcwW=6_;pkx5KR#I2_IvmxiiuHhpuAGTKwka;Uz+SJ#QYO3$0h zAF8L@IKxoO7;U?>YD@;Rg`!IwI}|rxGQjH|-o;VVaY_C!z6vD8)Duvrnf=xlzk@T0 z7Rq*LCaQ;1?psO@yJR}f{?|^(;v>8++gIX|=>0*f;mPn~V4>+xM`ZAYM&fy$yE|+H1(F4>?afa5{i>DN|imUbaiHdSKF?QyktT(xo%! zLDy(o94OksGnVVMVv?$ZU9nA2Z#v=F7Ygsjw(0&0i))eXG$0D-8GP#U`uY?CfSA9^O>_;FMO~CsD|x zLSBM)ns!z=V|z4L2fl~m=2y|5{s|6M^s86qX*u;(1aYWfDo_vNR(DdBb_F)^q~rYS zV|HzlhQH-Cs_34W?6>*ZKGMbjsJ(6GmsoowZJYQvw+WV|Fd{G2f#*KFw{1YrPt-43tWR_vfW`f{XMuYP)J0JLJCR~O;bD_B z$6dr-I>s1nD*3*t4?n=QtG%CF>c+cuyp+QfN`AOYXiiRUgj;x*c0>QNfW(sD`<0ZE zN9EV7_MpX?jdzbV$UkQYvYDch9iSXrBEYU1C0oyNEvGudLi(e zMR>LWIXt@ZMFl5k%(hi=bnZHW55_k8VB96!8wwsUfcucRNT_Ygp0z!?UnwMT`1;9W zj-%6P5=%sS?0AT}*Sh1v#1Fd5G41p#PTU+!IE|&!O`#O6+MipL$Z``j60ywKn#-7T z8P=-zoP{3zTGQSVh|aYcdmc_$JL2`MGL?4e^xowlva^?JzfF1lSd~%smVLUnhIGC- z|JLm+g?wS&es<@$G1NsF@X+}hOXbkN_8Y#$A)HtEdWKK!#L#(~S#CVsqO z*-n6wsJpLT0bn+%qpi`wfrYu>l2vv~>2NsWRb@HI zpzyf;6MjAS4W$`uZ{P=qW8s?DMW;@)?S`ZL9#52ojAC75lya=2)t4wV65il}^VD&r6=t^1{e7|r)WTU-(z-BGNwd0-YK5QkNIO5G=QmRHb=a_8M&W4HJ z!iczP^f{On+op|9?48+0@EWO0`>4&i1yPxDKXobzIYhZ{3pQ;{$0 zgKkoCa&RiSdF};zP)EC;XhbsQlxWCzKD^hiEDnPj9OL$jhJ$m^g^$=YpWAEaNQ7kV z7WhPH6)%>7y3QK>V|Ik;rkVF4Z5e~l0I^B%CJB(x zv^_?GVBJ@tvyy8S?+JyL%gD}#fr`KqC1CoC;soEkbTCZCqojP;Ksz2Ld?hKcS$MT6+P^8!)NWO)gW?4*DSDrQlFOe3gb_sfmDc4YutAWS=Ef9 z`#F?QltKw-iJoy%)zNY0&6X-maEM#B?Pu#iR2sh!+pC~{l2gi)k}Su)fC~t)i?m1k zOg|px(sfcEvzkxmq1aIXYFFr902`{OuDRpv?WLf$wI2W^-&88EOhM<}TZtgS)8P{k zR686;Tl$adydm^ycKjJP;mST2i7c!G$sw@c=^NJ(G5*DotX3L999)1iBz8g$Ew_#* zj8#7re}6r+wn#GEsGT>LNM^{N0LA{;;{{5L68f)r*m$G;ytEA~AL}}B_(F~g`a)+6 zl$cMYuZ@RO^Vxk#U0}c(cByCM75b7ZRL8hW=c}-+iY(Tlf-v*6w+y(eh0!4h@q*yf zrpS`Rx)JzNkNB2OOeO7hLioYKv>t|VgDru3)gc|<$OD4{J15qi;gK}eY|tn>*hmX# zwDe61gt$48=SI&)fY2*n^uCbGfeOS{Je}S!|UG-(LVyT*V_2GbWLVW zF(0C$O27sJB}4r!JKEl#@>(P$Fa3+P2S`nd{}=$Vx@p3gEOt! zw;G2$tlrPPcQR`{$EfFKrDmn5QwSn~(NShXLzAe%{uw1nw`+sRosQjlpZnph?2bZh z1!s!RJW3^kE0iD~mPWK4mF>0=$_PF`6$0N)gc$#X z(o^(5+YwWrcpf32KlMX(^&$Mx_#dbkw_a#6D1w@J80J7}CPe!M|DmS8WEIpY$@<9y zqMu4sW`F_6C#pe0Al>GGOeX7W-cPXiv6Dqf^~TiX0;s2ycS9Eq;(BU5DziGlO{Zp(D^{4ypl+`f2&JPV3>%}zRP;$79t*b4B^!LKQUW&n z)P3hV)Tj|@+{I>E4Q(zQFSL(#@vENm}RSrZmUT#=2KNYT8P!Yy&! zk$j9s7&2yNsTq5%;S%$4nx8L->YbjA($0pT^)dIPuYt#-{liMj<#lJhR4kJzE#U}f z=CY$s{lRg`aLTYoq@X&XVYZkh)@sO_R1jPAP;ap}kbH@L8L}Vkx6>K;iZ1Sc^9EtE zcFDOb+ZR{P}fA2TCrJbpjTfU4FA25ZPet=?ZLcF5ZE{ zxd2cq53)soIx}JtMi4&bu~Hzh>^D#wB4gJ#z8BhDK)Z8w>-+z!7 zYMeq-6b?Cg`n9F$WL$Pe3%p{Q2(XK zzokd@3e5VvKY=bzXwxOzUN*D~xR2EUdGJ5{ic}s4-;+6?*_hOSTHF5=!Vgob*X?mK z(%btA7yLt;3ZcBn*oAiWiz@o=H+C z0Rc2q5)QaAh}hW9jDy(E=hi#twtB1MV#T+~<;A1x{#a6MR<3P!W+p!A=%?ji54V=C zH##|X-M$bmxte=`j-sb&lg~y`Jp;U?|7VCo-A>PumuK6A1-VT#lRROsh@r>|U1*R% z)%Ra4oLS1h3f%v++KvvnNZsd1j?!f^o8ww$429J2#y946kX5;1Q|L^#kY|uluAmu3 z)IDx1%nn2C^1U`lU~<3v=e4h`+of64E=eZ5Y3qaCt?1`lK#l#PO)fJOFv1IQ)F?LC zl|wrKQUtMHrp+W%gqG{Minizfvkd z)R&~p3_&LNoI_m*CFD#^LPFwNPAu)-OJCI=8pDs6f<~>b?*R1;_vqS6;X1q0^JR0G zLNPS^#{k5=VV|+aGj5&hTkKue3=O*QiY!(GSB3m36VVvZGOfyT9TW6n^$MLUYlTps z9y>7;8Z;8P6iN3)w}p=gsqe%vno!bL(#v6zE1j9UPomEBSdxv+lLb1k+6Yw7CdYas z^e?-F8g5b~wk{es+UlmW8OoxGhH!-`;h<^a-5v z=LKg;+DNX~RP>@-l!-qU?}!g@sYlZrvN&G!oZEm|o-}~`v0_QjZ=>1}ZVaLa7FK-u zvM^>q0V;Q3Agpr-stO^nrG1zPS(XI-MH_QUhj3%EP1y5oHz*=+ORnx-8lrLS;?Q&; z2;+P01c5PSQ>I;U?howSqdBr`-hRJE8yHyu)82?LQ47bEr6x#d-M|%(z^sk~9dbYO zT}z2n62;ug3U%B(qYT?5{TuzSmh-lgUdSpggTLN+{Gq?R_OHtymK7AW8Pm1oa04`f z3S|&P!abuJdi8P;nK@bcc@Fe#=e%9`yc*9>FXoe0%QZ;8p4novlQK###Wd`j-Ac`M z+4la$F$Fl4U8lBW6u$0qTW}srq?BhbX(Y=96Zq3kHyQ`^t0Zn}iQ7LYcSy&dp8_AY zCc|H_V!u;6V}xU9AqKQ?)x z%h=e$=g}g#M!A1~Dk1cpi3la2iwYH(2n`p|KUa!fIfPrYp@Y@)?Kkhji}Jv`2g~n> zarx_TbA-bK-Uu7u2>Uqdw?PAZ=!C681b)t>Zfhe=1#c%VSggMoJ}o$>QYpAcpwJ}L z*y&DN30_?aSUUBpGw8OrB{HOY>PJArja*!^sabIErjN`NlGyHQ73u_)8OLQ zODJV5@alu1bvceRD2>M&ufl!T_8-v?H>iOvPg1iCDq0yn!ngt75#_xug)*4w&S^xL;d@ag3S2QGp?nFuI8ArX%xGACj=7S4%N|7b~?(M6R6 z4SSUaeZpz3rBk^W?;#@9$aH}a6Rdrp>Tx| zpJv!MIQLHA24{dGW>j$C@yOp0`IWP2ln~0Ho3Np*OqDZIFwx$vM^)nUd-hE`U>yc@ z+mI7H${OhY*c1fxS%2g2_rEu{G!?c5=kuL+VdMB+LG9szle~M(TlPz6T=70<;>3?1 z#fLL*uL|tzv3f^vgG(U(xEZ~_d(C-8%56$J)locnGwBrKZ}t{JkaL?M4P@^|JR^MW zCQ7Q{KM%h0R0{@{fJsE2r%3chpUq@gm`$RE@IjJ3sljKf1pOjsorIf?s18PtA>DZ4KXaWe;B z%lLI`;9hGv?%#Wf^qDuL%~HJ|O#8V>W|W&~cfaE$!XF_HV?iEF<>Do#07w7G_5K(M zdMKiUI(kXsK_GfKu}&7$cv)GQ`N}W1;X1L4Bvl+P@z^^W<38Ka;cE>&DJ&fs#c^sBx zs&hV_wgq>7^_oSyQ{L0>NS}|^U*2YV+?&_*T~Tydh_0;>Wd@mDs3%=|VJ#9|Z*;so z!ZWs9l12J~2BU>?2Tu(lOjoc)53kG1y;eD!J3*hgWDh4@r@Xo}!jQ9pd~K4-b(xd? z2*rwg=u&RtKHZBNYj*0xbW@5`ck1jp1myq>-e~jh-}DDMW3Bat%?=v##hKJkPgY0f ztL?L-^@R=81v8tM(tF#zRWZCGk{Rfd;a5yRiRpE1(b06Dc`LJj{@KO`Tg_FMbiz*? zzrC-!ls`v9LI1NOUDOatfS-7nKZ62Adb_$ROStlCo`<9~yVoDNM~ci%VUjPH@4$Cx z35jr?obh9)t@57r=||^e`(I;?`+=cZ?`2uWhDS?~Ss1=**;a9VahP>yNrTS?$BOeM zIac*{wx3wv(q43CcZRfBN6QU|O(9$A(HKdMyLyO&R8le4R<_xSPKQ`Cy|oh(sluvb zFSnRzcBy5Mww}{u{8%rl}+d9+R0oF)#V$i`=oVU$57q+*N-n(UQAl z@6tI-^jd`E-B#8^bf7;A^l9;`0yw&RfMC?tzE5s9@(2937*u_$XUqK}* zxev2QC-6284iJ=ZqCr|ij&sQ!h=Vf-08h}y5%2{IGo_A90Gw{Rny#mEoE>Z+QOoFl zxh);IgxB39h{2TbeVA5}UP&OUj-syrydkf>q~GLMWP3-rq_%s9(V3t>+$ag>LS0>5 zmU-*JWVThKZ0l@V*GNtSZ8xwLifSj@4oSw=r{{jM4R6 zKNP6%{KbY?TkF@1LB`@kz$*I-A+F+kNuSV7R;2fAZH;;@Az6nwhmTw+98MwNfhl83 zD2~{Znl)Tv)@!8>)SYML1N~B?iEs^GE&x}Ds)c&uleH?za{kNAqi|C zfV`q8d0pQ`hsUBEn3!;`UDZl?uBB^LbHh#w)$6d>xq7$61Gg+-Jct`D{?HRYc<{Lv`s}+Zf^&T+L*9kU%1DOZVw*4jC8La^X z`bE;H3bLg4mNDE>GZHpc3&Z8{Rm`LVCO*v5u>_G8;=H&iU#Lk?-8EL7;dpqkRKx_U-3k z(MqOV|F^?xJ--XYVJApu^yCV7S(>nt`ZI$Zv*t}=x#VE$fd~X$>f+8{I!x`E{HvQI z37-jzU+de2%M1xnCC|ZShia~+&1Q;em!(Oaa^bwB)%{#b`mwEyN#bDUs> zrT({rP6rEH;zyAA3|qfqWr!v#4AY?v^*LOQP7ZFlSSzc;XHQF)ekXk?SXnv1S#p^pPMMYml+ON*2snRh)GKD0D z2oqD|?4lA}CBg;;xUbP-_58|8HE^*1+4n542#sn3olGXd{ukZ;n!DmJ8V~X*&H&s{;)G()% z272u8!(UlFu4Z@KrQ9PCq&9k`5r>r^H?wV@7=(_#fgWx`-}WpFQpl_(67N^I#?OSKT5)XB zn-sJsoaUn-ZCVH_jI*ALldRWh2geI!<^o?tAn*? z3oua!$>egZgUxgxKN$EUK;DytNHi+WcB4%qm)FOdw8iy&eWlu+2k<=Z1N*0b3!Va; zRpYISFL7qR%eh9QfqCbO_C=Qf8SAu(6|nWWbecHF>Uciiosm$(f5EZ7MP}s=H}Mbn zt*xz%oq6YIgBIx40Ajh@-MHhGHl7gFFhnzE^#NIEJX}26w7`{QQB7p9IZ-+c5b)WE z>R30kL`LJ^g=2t^QYNFZuqqEAYbv98ShT8jNW9B_Wm%z)>KXO_e=LB!0_k)c31;L( zN){E(;xakOlj0(T-K@y z>PIW!3%YK=oM99$4J+++#|Km%WVsD0e-6lIMeNDYYwwFeY`>1Xn^;Jz(AS%!vP4p{ z1qX@lZ2Hw%C!!8Ij0qzQ`9*&N0mJFkuXxPhdK@y@oGNb@?{8DzjDfA$q2DnW`?~~| zWs7NW02Ot^?XDj=8Y7sXAtSk|yw>h`h3}sI=`Zt*MKO|Su@R)aoykU|bhj5RmHk2A>`3ZB51P>aj+bE<}AVUAeNI zcV}f~YmS1he1Y;hh@bOrH$z)?XP#ANUn%>T=Fc3#Z6SKDsm=^%#0Fil@R8; zrSgxVoa9`j%o<=;x5}I2d0gf=h+(FF#yU3-WFrm!Z2;RVQ0n!Lf$be+K)>4wV7&&P z1dxBM$4G-pQ`IQT5C_jj()UqeuuHY$PfQm74e>(07i1^#bPS|B2=6;CbMk+{muQ~%<#P*FhLT#hmzaNVNT%oegr}6_qL>;C3dIZ(%E{6 za_w6-x`@VlPv4~<_C{9}Uaohm!}uo%eGz5eHy?vIpvrIo9!bpx4p-fH*o5;ynf;oj zj&h3+#6hvt-MW+9v(XGm=43lcmaABvSv;B2ahiA#OQJof6#3q~5mtksBX~t7Kt%iC z@Y@arfF$^rl+zjd6z0EboZ!}kuvYP2-3a~#&AS`eQ|t)X0hU=PQ?(t#c~0_FiFqXm z4>!Di`HX>|_Cmy#{muGY`ZumO4t=;U@1SoF8r;D2#C%B<_Be@*BTPoLuIU6=IT2TD zG^DAdyULnefz&$WIC(*qy+Fsl6SeFrlPJR#Okcg~aBgw09HU^+xvuyLIc3u8lm>=0 zDw07j|BRY7^TOP_49&)n>}}$S)KRtjz=Cdhp7?~NCdff1$qV+|?s@yR52I2_-qzFQ zcvo+QYH|h$*v_&tAr4go|LKwY01|BB|B&^TQFUxhyLNB_!Gc?`9SH93?iSqLS-1xG z5IhTaC%C)2ySqbhcRS7Vyl0&Ke&1ilqGk5%>N%_GzV7h(x*&NPYm?guTu$$j7xhL5 zlf0Z5N67E2j2gVR1o%o$ME2=JtU_5Z_Pbg6Hf*qO<7Clq@PD$2(G7Ue+gXDIvg^Wx ze{q~EV8)ls8h{8e#?xxuYeSW#b?my5NXJ#b+2*(lEU5mP*Db7wjMjbiUTve0xQI^6 zU4TFTMyIH$*?cKZq$Z~t_QeZ+<*)y!(Nam=`n4^(Tgt1^8sheWS?`k*S_NCq`bedm zF1%~Z#a*L_K!IQ7;7nwB-9m&dP_R(^mgtug(5;myf3WJ-SI~+5s+h{ z;2gC|UAmMpnf|q(n}krDBc6H!L6k%9=y|UdB`{0uE5PTy z1XnwxEE>}Q^f2$PIF8TuCsK1`Huktg&`^z=4omEY09kF?@*tEI8pJOxiv%mIGE0$< zr30dHh-UC5^B~&h6i&5PAS7xpehdLgtvd5k`v2oOst@PAxCe$`;9^m`?~Zbc;k!4L zKKFBok-to&9+5zQme5-$HQUTmPC4v5_4>s`hvL%XPd=hf7m@<;gYGEUFtNCJ*=3Tg zP?g|yh$iwu+&OY;fg?g&;Rus|0q$K|E=K1XOCJ&7_!`&zTjLwTI+^0B_Z_!e=?6na?QZwhSo+Nk%-Mq8{o+l>$=NRRJAT2u*7ol_= zw4RI4Q1E;*#VL4p_vth~-8rZv`MInHAmT5p$78MoDM?L(Xk6iHb!MSmX#ytN<=3$Bpw`6pH^}$(1&feMli*2^sHCOk=~My@4A(X z!_6J8ZF6&+onLY5nXY76-Cr`kH{IFK@l=Aes<&h9u>&t0EREP8En2tJA8{OCoO(^k zeu^*ka@Q|QabdC6b3ZyWSRgi=GCsvKl83%#X`*hZd&o@?nCEA}uF-`NHK}0nJETN3 z+4a~=y|5=pZ3a?A$j+=ZEL~Yj=0xSL5pOK1A70yl7**U43kyn6%hJ?Zgb+_h0+Gm< zqJXB2{wrW#CB*x*6tdgf6iqY?V~>23q&G2loPwW32V%3HpYR-n#}9iMtd0h6^v+C+A9C>>*yJE0iprz5kz&>?Da@3!_@^&lRF#Q zV_s)yK7%&;M``+SHUCB|j#wfSd2yb*gr|WaO1%XUqh$q<1kv zt&r_x5=hyvFA4X(v(TGasoWQU7CYne*4wG$ofqdJ3;ueGon?9g#_xM6)p&TZXW2XT zWFt>oS06`6AZx_c2-WR$B?6( zhVo-PbJmtbJbu%ubq9lF3bXLgtlLfEf?IOur9)rcrGmQVOc%=U|-{_bGlpa6f-%XKObR$5e(?kmtL6KSg{54MCV1&WfSdg@>ux72aS z;^EAdp|xh7w!Rww>mk}71Zd?e4Eeuvt}1Bd`?!m5F1tIDE_psqY&7sPkfU-FDL%LG zGcM3h@u%|TvfySEbF#_A=4-t@w3+>mJW>mGf%?p8cW4j&-DqE_1p(CcTAbxbF$VmMXq! z-KGIX{p+aD;Gqub^lZF%BW{>oYAZ#Q047O?xh_6`guj`$LGAaUem!)uOhSw!VSj(O zu*P-K5#^N^7xd~&TOLSf1V7h;6qp(@9JTKxvKdM`X7oA<6T_}ybN_hm>nSrrniumk zUq)sj?E{u<3H2;qBLs>O4fxHs{9MGtHOLJu*SawFbqpf_BW4<}K8nEgv`iOA4XC6_ z%muOr5|t03R`FYF`?7nULWc;DOVJIt zTb(VNX8Wg&NkfR2W#!{kKfyj&mT-mLaP(rSLwD18lLcAtwVz@5zh!be7h{9NBew2e zMFqJY){7*&1J{ySZDP;Ud}2JHYf6RpDCv?2LOT}RZQ01Fi1?mNy5IpA$#MORW$LUF z8)PHG*7*;+c_980&mm-=yxTHY4- zs-{uHwK;E=^}8C^8?@gaJD00W57pQx)8u&de#vpKuW7yMJV}ymyH2x%^KP2umkCuD zv=Ye!Uc%TD<0cM1KW$YN3n#{P;Dzjw(Vz?kIDNw!Vh2-mvk5GgcvB#q8~DsIV9-o6 z@ciW0qA9A9(aZQ;S-Y%%s8P}4axgi>#+H|_Tdq;fdF1IUO6O7luY~-7B{q&a0h7Xb zzzpTivaJRCDFN10Aj&KsPWx*onSwZ~0@62sB!6Uoj#wGR=x;~d7}!!*`4Pse zeFJ>kW$nBX=R1Y4sQTk})s>;Vf;ggiizX5u{n#WL3b}9m^wbRqp6mkczm*V|ff#rs z5HaL)^nfL2AiB2{70=;X#bNqj+>7xhdOX~+Wn}{8X3FS8-zmz65-B#!=mJ+IM0MCtAK)B_zhd9kKn=Kra7tV<$ zx{}1jBoyp?77Gh`j1jVxWjDY;X=8&d)_q*&Sr@fY?}c;V^Ltiuh-ld!GZ_5f3=aHI zmdjm0lgy_^m&QpuLz8oQo6WQRajT_Rt;?9*BF+!(>)M;`wy%j0WHq$`VUKGhM42cu=BZNK=E^+m{@F_OaX0vi!*8}Zvguh z&~n3<>X_;2Aj1r@oX~xtYB9}`%#&$yI{2)K8cC_dZjZZSm87~vLIL$fMJ1obCe;bH zmztWd0R5F$DLLiELXsaU$L)1$^jEw^)9X<-f4()CZb1Z<9WS?b6>dm1fBQSg0MwC9 z1zo)qedZ})#I?|mPA<33mimBu!N{?LU?9ZWBJ8q96w_%cBK+7N@Tg_5lJ+`|vCk}i z8!a5{6f4O*Quxc!C$`kXDK@8WI7&yEX;d0?(|!n+WIHIuPssRn1izJB;1pZ;5Sgg&YZQE@D%#TT~>^tFEsyV9x6z2u`9_(Xg`I^){jRRBlQoI_o{2d_7TP78R z97k-vE)X2oOn>n3=^FmZROOm=tXZRB>cuD>>}m`0$r#u9`s_cGB^WgK+qKCm*y?wc z(!6P>J}$p2y(Kxt@q~(LorWz2^9eKZ8I*B|zzhKjE_`VN*|ZFU7ecN{B>6IC1*_?- z9SW}j>*Lk_t|9iqo0v zD2P9k(E4<0FFm;*zpRB0AZ~|77mD58$n*Yl#!r}JSSNIUo`vb5W7>o|U05@l2t%a8 zD=5IdZ%oLcaQ-Et=d#wMO56K4D?XcKr)pc{U`_HLVz~wq6h1x9281p1$PwF8&TF{u zAd=n8G>Nw4A`z|zi8=o?tS+MVOphK{@7MN=Rd>HJdXzv|w- zhiYd9NlzBVWUmzQIB#F6$x!CZtDjZrhtx8jYT~lI9*3LQkf4{>n4fIEnnjsYQ=BC% z#s?3L=~;b&SmOB^l#-3eo@OkYii_4~FSHR!ZyZ)}^|8kDg#5v9P0C_jADmj{=(547cKVnBVg%}1lr^w(##363Qz%fo=|71rSjY06Y+0T5v z+$;*gr^foZ%_q(ZbT;->^ps7DV0ck*~GfKb_LGIa!_; z%R6IPBgXng^qQQ2*mu9$re*xYJAg67n(Ko4!CF`3D1?Y*A%lh*>=91vA^$K~3O^e~ zD@bfZV~&}@8V@-hpTPy+9{J+$NZ+6e&%aVmIZrc|Qm-8BOdW37;=Vs6OGH;;71xBp z^1JtF^82q5-QYhq)-!2fFO^}986Hx74V2xt{PJbea?)%3Qf|lxM$nQLiEI$#e0jTk zWXJPE3|M?Np$tl^#yo~AsL8PDV<$Z({p>Gn0=otcR!$A6=DURN6t7cLKVhGa3Bz`> z%gLbX6<6y9DpX@~k*rF|>X6XjuHOEPOYOXH7Vq8#;7N^V58QRs|G1o!-B}waUGMJo za0<&uS~ELq0JpH4#f2X)E;YS91-4BcBq$oeA zv;&Kte~KOs5w?KJz6yYPjb26iO68fedF$`Sn(!-%4jW#ELCKlDjh8uiwpx!S8pX6c z`V^Fm$c;5ao*m%8P01AxgNPNVi@&+|BJK!D5Y_?shHh~!KgI2eud;JL&NJ1E#*DCf z-?o{>AN=#ncqCtPs&nCIf(sFG_0us;upg4+iB|Q_#Q~aX|1C|mr#Mc;sPVKS&*k1* z;FD9R9JGhm+;3y)5(@*_pL7+$>+DA@f4}Y=8GBSekM&uP6i(8PkF4aYQo6;KiTsgA#PP){r_*6+V zS5G8fA~h>;Fd4B7i)Cq6=ZIZ{mBNSFsrf4)IZa>-Ge^CS*h+hoU1Enn2#55!Jl{)- zEHuD=EVWO;s?=%X{@U(#@!4X&EKQ>VYf_+jQ^%*FV2X;_cTjYdm@ZJKj{angx2=zm zzfhvRhk8`tfu3j#j7+$sMa$X6$^UI+Fmxu=8tbb0Ku%5gFu3s4qdslAVR&e$>Wj!9 z(-!BWIZ1a}pKCe%lgzNUf4=iw&FBA|G<*;!W|Wq!%_N5#9QY|3;HWFr)ImXJy5?9aDgYxWqx^j`s+cMtF2EbRRPLGXZ+#oz zv|JDHK;eYd@JsI(`kEIrM$9RVV1I$wQ3ZUsvm;i1OpPg?d=jFETn8MzV?y}pVpdO| zw8_&rTck2oAX0jjn=++VNztZOMQV!SJ*r16Le^#Uh{f;dW0hPEY+yPrrtub9{?OgC zLusmmzuk9$(LYnrvnH#93kt=2Gb~5ZNmrg;DwJ>OZERjvUY9T$o%Y8lM+>W9AV(;% z1CO;vF6~!VOWWb>^QV6Zc*d158-y&s&$e-Y;rH@;u+-ctKY1)fDG}h?++f56)`L119S7pMq8i*u@;4B z+T3+pyRw2j4wgw{YE@2wTkIK*m3U~o3`K3|win4b;T4S*5-DC4z-D@vl`;ebZ%7&L z_QoNgG%iscZM@AQK-U~cu^-j-;oMh#8 zhSz5iP%VjLmQ}~G8!g@7 zUWMf471d$C00`tllo z_#;duSSc|)ap!FOm*OE;kt5Qc_ z+Lg0>E3xqb%~4tC&U~*+ZvL*C7-^fVG4Y#`^^E3?+3WhnWiv9Eje2&N8&CUK=*!7K zgwB?cG-=wAQD44&f1PZu6G+&C(J1}7!n1W!uCz9deu?xi5fQG3yd%pGHmFa--+J36 zO*?zn!*v#(9Qh&-MwFnh$_b%Sdum^4nYhtcK#>W-sF71yF6>mQbe?+teO zO`N;M5s!shK36OCUNDF6xd$`mvg8V-BY2y~Wu#-Fm3ZJ?MXsHY29O^uJ}W^em6OI2 zf+R&!BHDks@~A}HAO$B$mgI9+d41t?Jy9ekN~GMZEH7hfmr!4-CsrI^Aw={)yLlKB z9+jv}C3U_PF3#$E4w(L2fbZ@W%E+uhBJ)y2`DuJkMx;rknrl+y&!1OxofI4c_ueKZ zMVii}fF^Yb)fsXJs=#GC8SKt0JrHkPR9e zkLSlsxU&U~+B_WDB}Hw@^HGWZn)b!5F-*k6jaH z?ndSOGeY)gobud?G%eqSO?y%dtxc`@Cu(4HNni?}E4?+9qna4j)+a`nhzD_z&DP@E z=vtV*L5svVN3!x%MB1OvP@iuLL^s}MPBA7rT!!@T=2mJQF2{NgSvTX(&;7*LyMm`$ zdbJm`#FqnKZ(ufwyQpHKM8oPN3L;__A=WBuKU)~xLFVtox$U?06Y`sw;OF@$l2K}6 zZDzfKb#(a7IdbKdk~nQ;o$3NA?(HS7PMZAir%#W&&IiJO`lv}CJVv;#*9pg9+*Ej} zj~r!;_?I_owYX5iVqBd^X_W4^FLVrpSpQuCP_brJ_;6rii-G6yft$h$`ch|11w8u&$HI2PIh$+%XU;wRBc;;}pZ= zTb&&@ommG?*zYnMG_0E-4>SxXhBi1YuJP`;a@W?nbh7d9edP}Rdz^fP4Z#b36jL9C z3mEw6F13?FL3emHamfZ!s@`Kf!GdJvc8|qFKZZrZlPa{JC-X2!Nh2C!eK_COGOzMZ z1EsjDl9rx@FR2iXy2=Y%%V;B!r(a@2V75qRA?$IPPg^nEM2%@vv>(sPG}R8AMrRZ2 zy;-|3uoQrViZJWIUM`l0^ZD_Sv{4^0xy7mIH@mQ*NTYy zgA^Gw%huJhwF2B;I?NVr9R%C2VI?#xXo4V z;TZ&r!InD{E>$Y?Km+8+U?1|-tOR-oLUVp~5LaR^x4H4XGXr8;9jI3H^XjUvL_rXK9C&{^m@R>uoIU}+??qoe<<~u%RNbW)yZe_b zsUE)H4BKKiiuAX??9LzG@)uz~rAjpon&MoyRfyw-$zrAsQq0%rq($6 zMr7SaHa7WImS)~l8Mp9a5z%pWiqXL>GWge*c#HGZXLWh*d}6Tj?xe=i^Gs0<3EO!B zpBW=Gm^~n`tWPG>%J^+{XsQ0!U0@wIK_dN?Fn7n0aeR(H3J-aFg8Q*`NPhV<9>t5y zg|L2sTi?j9h5F4`uLchFrOo@9IEB(Zd9XY??#D(?$j$2P(%nu-rO2pOY9sI3f^bH) zf$40?3`zAeBH??*)wA=crwCQL`l?*i4b8iR;HBR6a#e zi^kdNszycdBXUlZ+a8ncHa)Ei83Y#a= zA3~ls`&aE`qU#H9*;&Komu%*$X};a@LsPx@H)=}C82;mFVdO7Mv`ZhG&sG_R=Dp;9 zAcl`V_(FiP{XaykB&yBzIt4pz1xEEl;SaJG&;!!1(VEJvOf2mBdMPTFC!XdduhV7r zWg5LQOMNU85m<7W(dM2HeP3%X5BqrAkxcM&XWX=EUn!1MSnq5&khYr|@A#8I^LTM7 z<_ygW%5B3C2Ooo-*SiBWe&+S{{n&k!otiK|(w?e;cHijV73LY9vtht0*&F{9sV&t9 z<4T|rgYo4`!xj_7|aL=lIlz(6$YP7~b ztCFbQ*z{Xn0v}nEG||;)Lx@%bWJ4WhU{cf1x4}?bd9w{=Vprims$7eaxP#CuZfE8> zMFCsAX-96`wK|QF-B-ol(D#w|tqiUKBU<fB)q$=Q z;H*a@rb+wNF@Ml6r*CNJ#fMTh{GsdQ?LlBoSM48aJg3-rk6W;}{r7;v1Qrp<4@e8E7mL{NjjQ(wi9 z&7M`nkLqRdBwlfD7aAJZX{#COp%lRL(6lxUlY6J4!x}Jdh(uTWaUkesV7s@qwZ*|R zWlx0xtA328ql{6hCrhC`F+Njx1-8@-cy&*&G6ZYC?BZ2N5Hd4Y)@b#sZ!+J4UOX&0 zeCh_Bf=nL|Gsp*#;L?Z_N>8aB*;EphGi!4|fQjjf3n|q7HTS;944Ww2@nGBAr|ffG zKxy&WPp}mKD@NZF4mjVFL*@9IC9i8$*O zA*{?t^^K)o&UT_|XFp2O@0^8j0KnzA5j0UC@k)^}&YRjW+SE#-iW4KXXMf7U%V6K! zTrvg2OBDIa$QL8$$@WdI>OTd*^DRXz`l~?ZzwEi#q$7f#TBQ0A$%HBN>Lp9Yko%`H zb*;Q{f274oq9-E2|K%z&m-81AW*XAfcA9fKv7oZ`dZetX7S2s*{ML`2? zSz%fWw1W!V(udO=qRsIquYXsS$Ne$-%pn>{X1G6+WVG7#yxd@)qj8SS38DvdR=%P@ z$Ln!=ih+CPXd2MD$*D{_S3|1f4IntGi10LbD@4d(?4!lig~&z?P-b!%c7|E}mq@fU zhqV`_R1-ikW21NjnR(R4OCBanw89FM(_~RROx+3x?&BfIMNt;~>cV~yam5hjQV=}6 zPA&9N0x!b)aLigSE9sye*6P~LyD(L$W>s3_g2++hH@b%$iA319b{pb$*mjdvckwZ@yR7Lm z2x@kHUUG&OfIh5%hJ~LrkdGqO4J5z(T%#z*E~_udRfYRNIw5QTW6|{;YeC8kjG~us znqjZ;PpNE@KSjSJv&^qgaRQuPQ%th_B0u=howa8fbFgF#oXtA|>>thE1x0+2t>K&q z?&bvmmJee?<${K$O< zx_cXq6?Ayox^6JB)w8Ctaxm26nO(p}fv24y3v?t^&ETu#uFVY*>Ph&*nL0XIr((sN zu4{AnJ&Eb6$Cs1FA}JUtau{C2@j!@^p|9CejX zyBEWo1WJ`qorOPBa{)1O-+<_!wY=e&h=I5sY0eY+&|tMIz~D3%f4I!QDRN$a9ZMX9sNv#`K&dqftt$ zjPc4j;ImZ5AN=;G)w3xS)GjkpY6#QR>B=veX^3^s0M3QI`bjeXZd!uE8AY?vA#b;K zqw^=j+QvRb@42_t6#LwIY}V7_(MLkIp>eIU$Vxf^9KAa0BwqQ>A46N~zrRYCxZBW~ z>Pe!9S#)b+&wMmUt>V}YXU^UuU*Pk&1T=9xkb|faGxjju6=X>KU+2=Vj~Ay;8**$6 z%CDMS;-=T01wJg@EdW}tkO=Fw9=~o4%IjBw^@@h;U*EQIK z2D97O{^{sf=Rgnu~Py&%p$b4&i7-`!|w>gspwVQKD=uh zi{Si3R3SrGiYZh~J9m^&7XSPb*Hfh&Y97T3CVWDQCC{@8XE;5UQ5!T{b%L)aaa!%^ z+8=s9P6+&j;H(pc3W~v<#w~Cp06ZMP?>)Q6R;Ier(%9T7@br2f;dqEfg{c8_0m5Lt zjsgF5{+}uHU)^Fqd<~rz{^ou5WaO12d?`l{Wy0KKI+Vk=a0s_FsLap?$f1d-8%^*N?-JH;s)2$kMk%YHDnHa;xF>=?+ zp;eu>0n*A(SK?42WF-mM`JtQo<9-vpj+%HyrN3)P}kytLFMx_s4 z04uo6QS2^_%lS8et_s$&;M}&a5Yu-Kv($vP6ae_E?Ekh$47q963eD> zHv^z_Aonn4o39Yj5FltLjE6_ekH8Ok;_*yUC6?DH({0t{=;EPK%x~n-<*|NN0nkvI z1Na~bHc<5C)(|^~*Q1lgM5f;n!C^XE77+V2YCqyt)+s)E6?r5c0otW18|GEFPV4Sq zq^x$=goHi%lw0wK-K}LsU2U&byGe;NzU8)#h`jt|Mt0iJQ;VaO+ejA=z?dxS$tfa) z?_!U3d0D(>3w0UiOZ7SH!wFpT;y!N#1E0wdhDXLR(fT0}tg3#y#`1e|7-d&fD5MGL zy(`ZFQ129=98=)wlElx-9vT-oS*iv5`}?56&4S!t0bWa>D#T{ z6gpGbZRuz=MZ}5>E~^}Nr7RY!O#TiH`1(5;gHbRat2TjtT}yib)Q`e#GSmt=GH5+j zlqj^*2xY83383aI&dkV*QR9H9_dyw8C#Vy)bKlLDRq*~aQ4Pw8RikU8-|b~xK3m<* zb7sZfO$xK^mrW_hr^7o~1$tP>i-9qK@ncypw3;vLlK&5Fi1$FZ2k+0J zc`;>De9RWCG}IPr3wwsUxmf0?d6DV#43LfT4)1LtJHfS^|m?m4Ft?r1#SV)(K4kc#e-oX9j;6yT$M-7Q_{qN`IU+)?;n%$_< zE%mQs=E#gOtSU&q6lLJiuDYf~xqg`1--;5XQTHAe@j}_4W0xJ?u;CbCl1saJt)8sFVz# z!K4O^PBg#XX>+y!>`F*jR5*JA#}US^LVLj`z`k(@=n+NcGJ6@BZ1j1Y&M>_6zUajs z26P-cq3`lPs`?cN>92RzgytKs0M}+YB0GAyCV3@k0tX6rG5;U4Mf2Nvc2s<8t zzr_QP!*Aq&OEjvjy?E|c9Cyo_Pk*%)Q)%%)|S(DM?Mf8vcu^hdC|V#j0G;j7vTF zl9{Yl3!8siSX=n*(UdfOX-u2-!{sg@e2N+XZm3MMcD^9}htxZtXt`b{8Dr68rvtJz(}^256s@EB{1q;e{$lXwCf=ppBM9qhE%kArlj`Tzp3 z_8Ayd?6T(Ls<4*(tuVHCB;pRBAPh_YRy2Y-!8z3gbQ419x$nmPg&S3DPyKZVTu-k^ z|AEVPe-t)eoAwmoCPVl3Fz!<_@ieUW$~2Vh^zyX+7Rh&nbZ>VJq6=lQi#?4~)3xA^ z1%$WjReAt8#}jz`iq^p~9WMBBXb85w3mq!rnwG@xBfjaww~Z|>YZ^kv0D?LQtbplE zF0+=X`?dp?KmM2=SRQ#st-^Jc^%9yOw&VeAH*^Wipqv&!QEf zGL2rQ|%_Sc(otaf0Xr`b`Ke?IMG;*r;8 z0|+A&*;cmqD1A`R*|tfB{U=``;0W}Et9^QE-%9V(~)i-(L@1fEHd9g^PcdO{!g zZ!*e>d`k_<$l=|%#~613tFBRR$VSHpejB&+X9GX^mU4Y7qpA2ne)o zd+5_G{F@j)F>o2P{~O3!ShM~x53!&@`_U*q?Fv{vAOw71ofHg+T!iiNpgYJ6K(6AM zY{-#a0VrlU$DP5HazFwg!(*>Cr0bi#)mKF4Q&gkQ;Q72(hXpiv_!$@gt5&^y9U z+xg;W9=FF9W)dor{?UH;rl8qHKC2&}briM#dtCH`#s9;8(&c-|y2VNg;Onmcm8oNJxK6fHj3&5kVTDECklv}+? zIN?Qd1z1wM^ga(^pourY??Hn(Z99s8E%LU0)kUi_9BL~kk3>LWoHa{DWFpv~5JCN2 z!9N$f|C21TLeqgoP@eC>xD1T88tPf0~xe*m*-wC3B?9J}BvU zCE@#=r?;i@odOxWsJ0=B=j>3at6*UdZ4F=Bqg=3D5ylc`kGtgCVu5@{D#2sG!QMx> zGqO|Pg71)1cIP3lxe(vIE~nake2S4~d6>J3q)Hq5y0c}Vn}BbGR$8_iOXh;K<7W&^ zUHqLtAp|S>zkLNFGowl=ArqSah7~g4(bmi^)J!+-|A_XNlxX*U$$r3Rvrep~bgmA8 z{eX@Hi)=pRmsw%FB zg>JCn*M{^JAz3cr5{92$_F#p1!GH~*m`-#tzh>@KwFaRkuz zT43D#82<=sx+%ZCz&?UaKk8(Au`@bGcpPTaa6%4-(b6T7dHK3KmYMLWxIa+l$8q0s z0Fu#?Afo1F5&0Gr3aoH$kf<&9NsW?B&8(j^-^_HP%>Je@mx+qbU|(!FD}%K=R#Z{W zSWj32Oc~ctu>oU7juSd&8wF<{z2rVm3m+EV(^{kcNJhl|$*mj?hr*&|Gxjo(yP^?8 z5G3Na196?A$hMJ)@m8~KsHud3R8pdI$LW+)UMQ9mxPX8xCwDg=l@MR-K4HVKy$9g2 zRVEwYff4B%i2o_L4yXhpAWGV_J?xni*9AFz^x$TZ04>$nr#ZUFi|2}nhfai~SXKGw z<1u5pT>qgm{LsPX7Yl``vWV366&6CKv9~(R@x{b3N{j#rA&d++5mn9Z+)wq3k2Rqh zVcypquk7lI<~D9P!%6autTxen81B;%hdKbsbw1iiUNza(%a>c zTsiiQYHIqK5tA851x@Yr5!^(;+Y{!|OW>!0EJH)gkjqlH zlOtGk*hYgxG(JLQWZVAs`hggU;(mulV)H^T^hyic=|6c89>XLTMwGuE41bs!NMioYqUIeUe~=rNaA) z(*1Jp8*0wKnE}UaJrRrXps8F*hut`}e1xO_-=PJeQRQ^L7k`8vwGQPS+f1OPXDuEJ zf(c2E%_JO;XWH#N*nQ?GO)e$FhvLJtm;cvQy(b;NK#2N+n5G$972ZP2%zHND%Jw+W zhyv}r8zs;^CFJSc7$+2-&O|Jt_Qyyf2yo0neVh=*d`bs>$K$+~i#38?!4vzd?^!GZ zXisj{ZkUanJNOv~=+9{rNdvs#<7`eb9C7{8;0e>KVbNq|oN*X>jOFt|rwYaI-$@C% zoTC5EEjpR+{9hJ8@S-nxxMI=eTM~RP;yey_{?u`O4apm!7F(Fu*_++vN`i+fHOXo|@A<7FY(IT&(c?D4;%`vvkt+I|Xa9M20?s7*K^IP> zTIx+e(lM6Q-zDe+z9+rEKuO!x+o8)q=S((l$nOVAS}W85`REWpl;s9(`2{%C?sDY< z)5VE8&6n3fU}OdeAsZ`(=Vpwp4zoQS+Kk5+`T) z=O0+D97(-;@d2VKG~gChzI{J5e0bz>2kx;^kQlpeCi!>1ER=sv+rT*hFQe9N+!DsB z+Kr20NSEL)IYrxK6*eY3GcRIb9ti@5b@w0sH+6{b zR}*+4M>|?W-i{%XlMz9V8-d}HwuowhwETeT(`0agn0LgV!t+q^et=u}dzTWXU|?j! zLtX#7FQ1JN(Cl&}hcqMi-ZoL?4sko1a*#HfK;6mFA~2LrKJ8L+N$5=HN$C`4AbeeX zpIBVbOcp6j7x?tOZG>SVzg+((O!&FM=Y9?EM^sN(QIBr=!=fGOuf-RDWyNs~J0g0hF0RFxRW&;2q)E3Kir6!$< z*Y9bqz0s8Z3T+_EFKY`p!yBy>A8!7lE{g`H*z9F)4sDk$*Xx$aFFRpwl{ekho|MAi zlt}qI%#21J?&)MVbvO3++E4v|?mhkg-Md4`2O$n+ZEc#=u12>DLqXzlIztp&SSVhP z+nNF+I_(ApHk%c5IOx=!M$d;!sjmAXmEtNsN|x1UAd|1GTCSUx&QSJrmH#>R;FzO^ zB4zqpZMO_NnV`?&Cuskc-@G9a6u9zl@3~=M$bfedR|}XyeLcZZTfz65HRfnHL0C((BVe|g?UUvDdhZ5tH@ymdK^7U6)rM+li=;wxp=%bY zbZ2?p(*w_nG&kH8od0e}&K7DukfoiDWr^GO5HfL=|CP)@TRtZoz6Dc@@bKC;^`6PW z-?pha$+(9j*K8^$AV3@Ofdsr0@IHSuNH>-%X&|mU|MR^PNQP&znj2X%ueSRvv{6b| z+K?i`vIV~mNY~4%Tl+!4mds@`zqq%2hfy3^-&d8ufPguDod1o7ml^Y}!DFf`n<(LX ztLsaClWVPN;d7sd{1Ovd>f~Vox*3=ER$hPe`zas%1d3d!W}jpnVn807%at+{x8A_n z{X9iK2x9vw1=-iGmW^E-QVHV$U}FOIrR;3dcP|Faxka74qS^=ZwWqr?BH3YJ_cV>( zrvr{PH{U0p_2{=6TF%VOU=rvM29S1|wl&5@l!z5We-&B(;=){}zn{QTm{^57?$C?; zV`-?1^+r z#3MA1$pB<734q^A^#-Ifx~c}7?VKLDZqi*thnWOvexv|J=2=tI!6CSEA?S(l92hZT zQ?od^G_E|%V#SXYI_FM!1r&;3S#i&ZjMj%yI>oA(*ucY+JxrwM?% zaFt;%0#f>9+*p04MR#~t&lQ1;BM<-D==*!Z53j72(h+-z#QrQOc3cClVevl?PpaOr ztzV%#lN-=RMNZj!PeIfHFkwvEg?F;MZR?GyY8S_J{ki*g1p5?L-}{D*^8zp*0{UeW ze4e+7CWl-Ax%IN#OBW)2Y!v@1$KmOk)5+3o-yN_bpZ{p^47eY}1D0{CHWdBW+)&9! z9aS^#KNW+x^PM-Pd=> zHW{N1CztM0$LHl{{*d6V32ZU#{j|I;B`GN>Wv~JY(7!GBc^O977zl@N%_aL%xtjgE zn3dwwx}-B2NbUBwIc7n8sTqz2<5ZTtqUBOep|h(v_9GpbnoyyxU1sa*ef9ydvkX8$?103`Z+P6M*(Q(sU_yOXAykAN?YU4T*<3t$!z`LxJ1YNJM zCc`OT`$#a*MSj@lz9)3$5V8fJta+@v7k9IXsG?k{La$C78I2(*5J*7}EMO0Zzm`DU zVgg`Vt^2Lof|l!;t?^Vmr?%1mp9^O%5 z9&@_f6IBKWnLII{ek9(F(~_gQz+)Il=3($nid{eBc%CAK*eXz%%oH((kzm0-oOnOG z7uHE!pq7TRHP5nEh-diVZnbSGE8hSRLvc71fOa8ChUESI{Y83QfX-q8YyG-V_pv{z zETf>)>z8nO!dPNfxe<2~`(=9E764b;3`Ju91%{KnP(M&c6$cL4YcxXO*i0UE(M4%k zX?QRs3*5B-w{R24-mJ<+-b@!11dwpM>93}9^`DyZMAZ48Za0hpL>uDI)$9u_DmmRd z%U6`54V_kqo#?OLX`mDD3?(q+oqPvBY0PLTAmN zp))P<&7Sx35qJ}^fRf{r+s!ffF2j5ofH>imk%d~}#MFO++1G_88j&H3hVMeYO9`ap zhf5`+od97?v)ugX)1@s>^(nU9L=f#fd3tMPXXFXQAC^Dtxg0iOs3vKv%Chl_$rBA) zhTlPEkT?3ySI-Ly?y-b#TQMzKVeBcm`Y(;KC+@x}#F5>gH;<}(xz>wxMc#4(GF)e$ zV!a54`28>{F)xd~0l?Antvk04Ja0ctvfeb^cSRx7V%V!+ZchMs{e>4C?fYh>eZ>38 zzTpEFk)JQEKhJgSGLV95B_yh}Q?QS-JNm~z9A1o+1pLEco*Le0pJ=-Wc}MKyRxYg>TI!$pSn+qbQVJ&skQ@CU>4Ti^&CdW>uzg$ zI}(R8oQ@(5m&W5Y+#d{sN>_L*R1)_$K{%Y9?w&pF{Mhi$|CTp^mu<$J zh3B6CfKP#3^-|JL^-knww%$$vp*25|+mg9ehSYcA|44$FZL=69;6kU*d0Ot|wC zZ*8O=j(UFDEdF%>MKq7P^1|!9>Lki%Z&em=bKH!UnHP@x>7bVs*?GtV$82Jj0o>Il zr^V##D50(VGU1{p)NnOl(C_bL6Q z+9<`zz!NW%?3G+=y<{8F^*EElC(vFkQJ??xA>I6WRPIlw9u6IV#dMPnRUCV*Hg{mn zOY_gTWFHknCgeEiEe(&Rb-IhB2^{s^umeT|IA|4iMzGcn>UPp9ixQ8rbgRVOhaW>*y_`QVy$go)b_Bl_ar)8vQd$xum){rKjQ z_6IkSK5Dq#EN4a7-_`VKK18ah;}l;ag6yjl;WQXR*gv@bQYP4fsu-|;;f<&L$W%*e z*HHzSDzBzB7m$g#IB6v2MEPB6u(ow8Y}6m$)N_jFk!1<#(P2x$IMRM?%EKtI{;S%~ zi}2B~y$Ugjh?a{_&L(5er1QYbXrF%S7h)i+>3oDaGe6zi0Mx^R_WK*)oCPkm{5Yci zKUN&U1hDf*W35U1 z<~2-&sZ!e&5=;fX&Af1G%Q1ec;DwFKFJJ1N0~I& zF_kus4^Nh{!so!ue=l|)JT8a8XA0&ATD|`pcU)8Ww8Ufr{IU!Nna*?&S)ACJ|7v+l zUG=8ATTTi_FG(Ive#wlIL6jfOitOw6b~feAHt|?-n(+7(>2QQEK#StbWuuwOXNe~j zf8v-qWc}~;$lq6TJVW!j2AW}XuPWi;3AlmKS5v* z&8PN&l+nmXT{zwS=3wd2;y6syk{}VJ!iF;J{si0ZA z2E(qyHqs9kfyI&R&jUa|RqK42x~EM4>s8{|&g+wbARb!$LtkXVRL+xq>0_3}j!!N< zUb{~4(l_?omd%(?hgE+ShD_Rf?*CQeO!({_k333<;qn`Vd@ z#7FTk>P2%|`5MW0D>Ia}!d-m=5s&Q>*rg|Q#yoN?>|?9T5A6`BE;$mB1(<@R35Mvi zzu6w+w9g_06xg2sK?-K?z!{+Y)*#`XA%_q!>sRU!cW7)-){@fK257pQ^W~fZ_Pd_H>1y1u})5V zOvO9U5ME$#Qg$={6k~IYb2{)tV=Zgv4YV#mN!@0esyTp&o#Dg2buknv@gG#V(%C2} zv4m5`smVYoLF?>aTn&0F*UTWtPtH=sF6i-61_IkmD)OjFxgMjYL6dx@IRLsfTz zA)9iKtH#ux_}>6)MF_5(`YGS<6(d(<;5^(ha-`PbLmD8Er!$ddUYGI_ zPyEZ&nJJ1$%)EP{a)E5-$-Goct8YO3`wIPf;l2M~%CnM!-&>g&{g=Z*qy=tWA>^}hy~|s|H5;=ayY#|DHT$J;cb_@@H%bt{N>aE)5r>L$lrx! z2L#(8f*F@MW~{S^nID=J{5Yj4c)!@pOF#MX-t}cSueS();cVk269f>fydho2(2qQM zdJP~KdPNfY(*Eh@Y9Bwq6J_9-t7M%VJ~Y49>!@vpPp3l&L0GO zp5!C9{|_E!a;HTnUcQYl{4OC6H)6{9!7`m2VXL4LYBP1@vbLM;&u%oE+v?;oIo)I$ z@Q*B7cpQqd+j&(*V(7wRs0|OzkG@*%x!MzZ-h{F( zT)x@Ro;Yb_*O{PQ-MIPRk$9OPQKrb7-x7!E-0$MztN9Yt)Uvbm7-zqkwQ))uU8QTmNzRW7!2D{!&Ox|RVb^_US!g50GwVfBC=8d^#Iw18Xi{gKB`Q#H&F2M@kw#(G|#dnPF(u+ z09I|uY}Ro!G8GL-5mh!TIzXqEkz(=#P|Llg{bjdf)YKJ9pzcP{?Hju`3>46HfP$d| zzI2@3g96a$ga^aPUFwlUc-hRB{Q=)hD*t8QFXh%z-eMsq-;A{>4d!c74CXck)MQW% z%hH$JxV?$Ip6qm}pv$S=9Z-a2xpsnq)bp2Oy3@lF{tyTZ1b^boZLH6?-x(fna=-aF zrc}D&dGSk-&NkyH!xMR3JPHttEMHq*?2IV3Q#Gjrtbe!x@rh3kWdcxrtA&JwSk(cB z1(Z z`Tp2xRqC#geYw(9QIkUWZWB*{VG%fSHvxf3m@K;EyM%k13+QPy zkabKJDE~BEEPcsS;js4xs>}r&M14A?M5Da-+n>CYb7+*7W7c3s0NsM3mOkEE0R(je zK-ug@#eg37XqsY9e<#g7pkA(m#pw?NF3aDR-40k>0sP~r1|AV$w6hxl!}a|oodR9t zH-HW!#J03Q5Rm8r!aoTse1Vh3O4X+fxxSz55!gXxSggrB>ZBGOpskm4=<)akTPdn= zegOd5$hRmGTTa{2&Vk89zbU&Qx0KVu{{r&FYg(6w5G|TLFJ4<;Zh2SVo6#^x#NEu6 z8x}`}ipoB_0NN)_k%J^iY=bp1jqbyVA>XPGT#>;BFuV#$m{7YT8nb~<8SgC3Rd{{n#= z^$-cxclvC+$hBC)=q&`kJ7YEPk7Wf~E!J}64mnQXTnkrSnh{tAg#9XuS@Tm6Rv@0t z$Y__D0(7Y##ks#>ib`~diPisbdLdvrEvK0e&~~KepOSWd%VA?5kHZk@O%U45R@St>Qx;tQ9eIL zLe+}Def2QY>P1Vk%m?(NgwYn{L4jED{gU;5Pv8vR=a;@1Cgcgdcbv4|hTUP&fP1y7 zFo55g1yMqvezp0NCGjLfNPs?1dSbYTh;7A;9s5m5)AcuF)xD2jW7zs9SO6)?OLfZe z)P{1{=F>>CX|Fin`V9fMA3L17mg57+@?1WZlC90IO{CHPIQVo_vd;0R0=D2PU^E2Q z^Ect0vTXu1y%pxF#V~7iGg)1XBI*};XGo=OZe2W^+==J`3`}LA>ba9b>u+>di(Ill zrYB?WGp%$3I1t;5!tTD`i2+c&m^fs(0kG5jiPpr22M-xyHedf0+ls2rLNpGP-by5X z6fQ2H^e~m{#}qhZzXP-oSyvRKcWU(VNk#(kF8T(9ljDKxE}zwb7Bs@?FUVF4VqUl0 z!$LQUX53~Ygqzfh#eh_pPBD!csC6PQ)8VSYTFk48d!74! z^_xX{06TSwlLuxO;8y7r9gIi+(Qv~1wetaGg?aQ5386-;?mot22$R1gQDU+rxCa37 z_T4Js$Pmju7C>FD684F|5t`1#mBpEB5aa(4dFx(jGIYogRco7PFc$QA*31uEl#q&J zT#~TkG5z_$^im?X{VXk7n1&@c?JkkflS*<_mTHniJQw%WmXdu1kSTFD1h1O=?GhU6-@UibBRBtUuBw6aBL@Pw`EXiwek}tAyo$_3&~<_j@yM{K_O!;J9`dM49G&P7?V7Di-p6(U0tY z0>HU4iYC;Jv$CHAcP80G_yo66^n)BP0gFr??H64Hx?#yS#6;hHZ)$Bf-qMB(#;A`= zEK#QS<-)g_1K4QwR=#eF z4&Z8Wg^J6%z16gnhhX|NWI*{Q_gM>L8_3Kv;9_2t_bg zQwa>~oC4_|P!iDd<2b|R&$xmd#}EcB9-5JK{S<)Yi3~U{{RY;1LFFpayY7>uoqG(w(p#0c9~m)! zIs7RUw5H9GRM-BnUi%8QPUS7?QV#vc7-;Zmsmfai95>{iCY3L;kB2&Nr$8;Al2ipms?aVCzJ~0ryDeNXLbr*9 z2D(vv5XNlfp*}()Yd`RCBWKb1>1GOV>sP$)r8xRbkSnUAZQJk5nyiMmlWo#sc=j4( zgbl9t<3(0fj~<^M|B~snxI6CG43jiMJ)?uy6Lj^v_TyWgn1oQaD4rV|zS&;|=2^P7 zM#5!;A68sL=7OrpuXLx5-j&MpD64r>z|%gTq*?g)lqx;0)o)X-NzxqOt3g`kua&)6safLFB=$ZHweLd|-X5Fy>woR(?099v zDlf*bIiZKNA&azrqvL+`KG{n}I{ElJORr0%O&Kk*$p(#X`o^u^Lvs(obwPF-9l585 z$jT+f216h)HZ0o9CduHKvz*U_P{Rm5y3`c_s>3Bj$OYNL^}?-xJ^R>Em}>-j?Xv@% z{3)L`o;3HV2bwESMy($)2xr4DT&Myf}zBgMqWNT*Tkqp?_Bs!(P!oIhk{;qhVZX`8{;`1I+6I`vjp9T45fDc0% z2x7-(u^2eR@r?8wl&^>(c34T^pJeefY`3ZCqr>y)pXjpb{<%iEAWDlsg5#5lG!UOj zSGN}KVQUr3w6)Io6x90JrT>^%Ii&wJligq)cJ;1>Zq=WWXt7{b^1VkyO26LschcaC z+F};%3hb>B8<}AWY*}uWmSn>#cXcZkLo>;KCC#+EVw;VsykeU=f1MqqyEY1Y{M$e8 z;8g~jeg+RN1koMLyqsW~_@pPhEYRNy+lhh)Uq?QQ{kTI3s|iPWT*?hSaNT8alP|-> z-tGjjzOKq}zTajQ7=DT=x`Md8127&sSf11$Xnf!#s&GhR2Iy0_NiD{m2ZUXI?{_EL z6sOf3G0SnGowCM2zM(97Txss8?L2MrFh1|JIGH$(N8hUgH(8gMGl1S>ob!aFV`5oU z;Pf|LbsrQCrPp~~)hv{Jy0*@F6<<8<)u}girTC8KV3g*b_Cu5nQMjCy+iXZjgdpUb zi5}D6{d8MbYM<3k;zFUK`D0^Em3wlRU8_iTBc@`?AM%ld06uImKP@>7=-wYuQb26C zx)e3sOE_D6F{Oa`(0yGcAw2!V`ZmsPbq>>quME=S3md4?`?sp7ru#$ zbvm|_MBD9THm1~w`q4^|mtnOQKZ}*i(?9W$6|0~5c)3~4xK93rDmeyLY zc@Yx)=)gt=_cI3hvDY7CVKMu~(gn+lWuLV+v~0he(xpi4NRAV|r%=Lk{JcsP+h02H zagqpV3O`MkkV0XPC>Kf6>QQ4hmmJqdqGNXLev*{#aJ|1D3eRY^%UMeKqJ*&pdqoZP z?2fbAMM_)MR=&>^@{H_`LPt>QY+;UL1O#R)J znZT!H){>g9y86L;a7{K$SsQ>3E4lSuM#%gnjOj82r8Qpr6|)631nC4me0M=vfiH*g zYf{3cQq&5pK)&n>OY?3~&yy6?1=x@yAX)3$?Gc%bT3 zs&|8>U*Px1sZvTYZb-92hXYcBOn0yB=Y;S`F3(6qe_OUo*%9m$P33rnWu;v}(fa`; zGl|Bor(uOxrUjpo<^|oG0(gw%kSgP!zQ+=xPMCBwk$uP+>E2Iwwr_pe_LBF*B;&PzDZG21|7R+KT@J^F>GF_rS>IK~&zqGB@W5GbR*C_9!#*GT%e5fHga14=P?^}B}H zUvi((DVnGe$NWq)yCqUfX+8|XDRm*mL%46-T~*)-_B0o;@v?vG1K-)&$z34Nv&3~K za1I$VoJpSN{Q5P>?PMhD$=F@4+m(9A&+TwulSCwieguYp zkH1i(!=cBop9Tia(?s3gAn#Ocl8orvL$Z3>~f*Yj~#Q69OU}3 z2}sSYTrzj@{Q8_C$!s!7p?c$P-PBjp)Zc3R#cuZGMTt8VOU5)$E*BV6B@IV9LJuu! zh4DHT1^YgaH^onO_d}|@ps{VpM#MrhXR+2dTj|5d;-ZFdw(Z&nAF#G(Lm7;{9gcse``Ryrz zTz1-Eu8G+Cb|3Xrc`(x_zj)Ze_7uUaPOLqguz%*YCZ83DXupHbE1DiOuIk&j3)YU@Fs)sJP^Ix~z9Pba4l%~`|u0MvVeG11h zv7R7H>UrHkOuZ)7Vdas1;7*#=R~d5cB5pHH8u(>7jc7^EABEZz5)HDC{~DEFB(x#; zc?NcK&fkc{C;62tM-PSfgCy98gzXh?HB!<}l8t;6c0M)pbzBeLXUSC}D1<)?wpA2f z<&ion{_2Qi#qFS28TXMgYB$AxSoZ$RWf-KwO!jq|vyK<%I&r5nSv@TWHyVU&d;|nz?^~ zq_5z4`k>sPx!{xRw5GK!a;T^+2s#xPuz)Mr2j^6m8>u->?&FjF`%3Pq!#G^&daSK4 z#5SW8EC$|{LfL;^L9U8JDID^9GP=VJtg(l$=2ajzI`If$(XZ_E9Qk$u{ZQ%`Y2sZ9>cBAUK-RTcQ&*_Bk2qNbikaW zMQRT}yF>k6)pC$;BP`~iJc+Y&zQOrv5cbA4qBy;|uUl5vrfN-j!(rUn;kHl*y1w7A zatEYz_R`~q-)6}>Gjyxb2;dqpRoZ)6H9B~jNBgp`+=r+)X4*jK#&Z&$!jO9j9*3k1 zn=WOl#(RDY$o?quVlB~JM$$+QGWaWlbg(x`UqQ6MwC<*!8lCI4d57E1@AwPIEg+)a zsf(H!AiHVl1DApoQV?T#gJgPt!-O#SX_bGN=+?YD52o0fiqw8A$-8Qi=S~`jq#h6l zLIkfaz~|ii8o^QDbp$^(g(qCq=j}h|gO3Dq<&O*#C0Ffm`_=~0v)lxh8b2s(cm)Z1 zF9;4#Y_{Zyzdk=dj$FA^Ied4dxx(uTjTb7N2%q4w8>cVhD7{zPPw(ob2B{oW!iS2_&5-*jMthW0pe06bZZ$qUaI2 zUmd@?@my)I+r4Tjel2ids11HQ^O_P9WZ?4-t9o?OmJ(@0S>1xM;CseDk6s078QJ*T z()&bdZ6R!*e34&P6-SJB1Rj{64Z{d-#0iy+Iy1vJAL8nJVeh{?_+B% zD;uU2L2Hv7^-?3#kXiRpxXG$hpjdS2^c;fp1er_IIeN}>`-2h$ir_gz{Ecj}JskNT&T|oX&^Ly&t z8>)Sk(O|9PGpCZI!~6Na9`-BJL|k^SiT#m+r>b?lZL_7NPb|83wb!E}Pr2WY0?@WU zC@cIsc>niL!OwhUv#Y9k@8~Y%HV()iGxymBO5ewMKf8?fPl=E>T2w!fsbk6u)Zn^& zO0Q8_0_1VZI$Z#)r>*U_mt((GoT$7!D~n$X-7n0W8FP0xf3ldZYtwXe#VDPDLvkI9 z;`xC|BH@~Iky~}kT(=oB1@sK#NDWmQCKtKT)?9}}=>SEw>#hVvl1}oW>I@nxk<3Hy zN_@lkM&6pK2qwmUE8tPy=kEp8);E7K);a`2O=^Ojl!b-I=?Jm&?KO zMA~)LCDg7obi`b2yT_3HBC)$>ocoOa}n3THXda3lA>ple|hH^A=MZ6wkLPOJw|7?n&Do%0%FuILip0rs4!0U!2BvuZk)F zSX6VU_quV_gs8VvejVS2I3_lSI`u0eEdoJxwifHVESZsRlUZ+ja(R4Z+UZEvj$*b0 zcv5$cLIgb{Z$C;`Md}bCyeot2W;x$>ygt%Q%R6wMSOj`FwT-WuAV-h|wOH5<>UB>a z7p~o7T8b}^?!m@?jh~@`ypAp8HM0hx{b;Jdxn9p$dh@>2Ro)R0$vL6At$bu01!ybv zmb)Pvs7amf_z}RVOaT(ReJ)04z|IIhZleoHxoyF%rRl9F0DuC(dk2Mr54ugB2LOGi z>v%3tnY;1;_Ji3TYF~|i_3r>(P`{ZBSEeFzEx{EPKu=lDwY>D&`zeAQKovCdM*Zc) zN=dSb1hfDbg@83q-8_K0`~}*y&Dwc682|=Pmveu_WBXmKU8%^;4U3KIC$}Z8Q&|(X z=G*mXC`wRi&9d7RpeQ|vJ|7D0L@%ogQWu>7G*F_@r|l^CrFJVT;XOp!QOb@{`R(C@97&+ElEx5BsrIXUtz`6|ZWT z|3Kx$b;LcUdoiBDx0)=6M=N|r(aKx`|2(G=pUxQo%IJLntVvD06(xj_06(a z^>Lk`g)o%9)TN6lhBS(nyM)ce<-~nOQ|K&Mu?T)*)t9NH)__`+=i^Gi`+Bgo!%(2U z@pThsSY(rY7x`IkF=NNu1k)6u%{Y*Lp?ep4xEQp9!UXB|6LY!bRj-UNCsCe9yEz40 z&zt9!XEH$&PtktWW|WJvKq0I3#tNzM{jo_1zp}nRTZn|p+xfp9;)md0YvJ$W@jqpA z{JE5XKP`{+vfZEKPk1YNj#ZFe$)>o|RUuwK?c>v*@< z!3xRpX@C)PRuObeV?W_+uqSE#uAQKnUDM8^-Cc0h8ON!m8l#lvuOT`Sp-0y0>9@}I z*@I7>TG{GOj~*l`5t7Td@-04iYA~4p4MZG(;L+`Uyy7eHaYZS4m*9XGqsH zYSmcNo=A)l3Q=o0<-6r5@^o$JEv!4+Q}pI1>UkhWcmia8HRn*h%RWUf&Y=slEa(u; zMSm?A_}qK-XEZdP-8%C@tkqZaHWQ5xEc=hCCmyB-i&-oneAioGtYZ{7@<+U`D+>WA zNPtMp1{>bOMTXw7IZ*wk;Q@5McK8A-6g_K!tw@0*sO5amzz{P_DGa3} zL{n@xpS|N*zJ}BQ5Ip#y4{r^r^`b!0^KI%Az$&Czp)O5!GmYtv4Ff{ycY7s?BB)XH zhl)CiGbaPA4Mpc-+vR)|l!6YQX1_cd=h-hUWrtosXI~jwF8cSnHKQJQ#Den=B%k-f z%7iWn=(`s?5&*faAYX?gJ?(yu;Vz?7giemG?rl4P8mS+vCqd!s{V&%u&DYf`o51Fy z2{U#`q0@9#Y$G4>Mb9sv=e*~cj^r%NR4@w)I6mmwX3s_f$Sn6zY{{-$Tmr=$hSC{f z)o zrqT9XkC7Pmsl>U7iLrv9aTNhiFFt`BQ&idQ5lkK3{?ZYEC!JowBPI_8ZOAvO#(RZI zKO8%^d{T%KR^n=OL`@sJ-{xug4I!Vrm+xB*t$+#O-^*&Yz?#Ee_MglPk) z8P;n30hSuMF-$`Ql~A2s8=%0wl+x*6w5iBveaxF}psuFw!(W;K{1;RN>!6+V!qGd_0pYkgqd;rzS<|W?+Lxt$N$67-n-@p8X?i(9G@MDK;Cb)gViBe1 zMq%km`w4lT)MCc?t3Kz49ydI`?(0{E-BB={+sa-nv^t>X?zO-tB%AY$UcY`U z`*xYt35DoZ_}d6&u_5%ok8^O_;B}u?D)^3&O4R z0l;sjIMwQ-kF8sLI35Tsb0f(L&fTjWh5n7_ph+ugolZ_m(K13g(H(aKFNMs;8#5fp zjMl54(m)wQsD{+=VmBzGwD;jeItsHiEd9Bg{X2FU@m+PzkTh@vrvGNpBp$Zd4Gl&U z^3oXM*5OMx@|RyNSybEXvXv{U-IV0{18GJnDo;NRzOx!2JJb{0>N)`?l_l(e`kG#D_ux$)zlMTOhgWnF!#TXtLFu;li9QX%{~aI=#bA0$2L2F7G4Qc2-p z^DuS1Ncv=KVK5viStDtg_268#qRAXa#qhSwDcD0m+WlKGpIBuV_!&qjf7v-;%$j|q zI8yj&q}wRn&r82Lk81=J!Hfm~!|}~cSwsqNzz0f5iHDtkqgjOz#N{eilH$VxN}>75 zTxf6M*Rt6iACfG`NiqkcE8tlDN>kTN>Dl*z|`tzn0$Trszdm0t82ycSE5k7;M3>Shuaj-NLso zwwdJdPL0$6Cl6=|5N6H$RUm9A60LulRUncj9q>;Lw#nI23Y!l5owIE7H1R?}E8wuc*FpObc3-ue}D^}=oT zwHUD;X@D6`#c!mjLDikl$<2l`yQ?Wh-_RC8Y1DPdin$oDVkwT?Jx<6<+in1ML#4r( zULn;zSEm&8SYd|^?!TzmbJWDCpwkA~w5-uT2p*PoJb;fh>v?ZqG%3oO;0^m{iB))+ z=LO)3+EZ>R>@auRyK=+JO&AIAZN&_VJRf!Rjsf;wZWZOir3J?9P#q$Iy2@SPqV!|r z2P&n3IUVAND=gJIOec(Q9A*^q|FRUb081gm_kN1VGnV``MCh+i^A-Uf1&r@J<{Ky1 zex)=TMUlo$z#wMBTs$$K4!PfZhi5)N@Q<+`41*}DcM)RexsEba#jJzCh8JH_>ovk_ zW3-U-NuoyMQK@-CFA5*cXMgIC6wk>KYHt|`@4K4M!foW@7?;#M*jCibg zN|K-RvrH%_HCp`#Q;|i=%jkkPoM|s;j@TQ2!5EmO~XO?Lm`gwXZ;>GSBtyET-hSmo1T=HEO%cyl!^zpOcg zx3K~h9EM$I`p*0mv>!nIgXY8acZSx9mEG?8TQV~)Y4*c>DESr1J_JJk6#1d*C6xk) z=uvO^v5?7aJ8^zJw%aFLnVL^@UDXd^Xc$06PZs)jk$(A%l-D9tmQ>s9r7LVlzoL<98SHVZa|;G4*D+eRrcChWN$Nvp=5}$MUJYXMJU69^HzdQPfQt?eCJ+rw+3BN}xdmYfo~=vTu8)#0cqWd~Ag#b3ed%KhHTciTD|J^Z!FQU|s$ZI=c+_QGqw7PUn*b=!QU&YywGhxD6v^ zb`IoeTTDL49?*<)IzB;aMuCZK`l0F2-Wsnun6#$ie+c7Rwb4 zeb{5jQSHARLVjBFp2!@|)efmtlx#hud96+5%Z$fv#aR|)P2_ksh_7kXREwCl)KlU9 zuu{?O3upu@@Eq=o6@k)$-E#THng&`!2G@eI1MRn7%x;Q zeC+C}->jI;AQqnIS{#YHc>|_E7Q_#~hw+RCxTynhz;vr((425{A2j6u^#YL32FHgo zfWYsRV3t`qgC_@M2@w?*`eAm9F+Y!f_)5J9^CP80G4A3us1>eX+TFE4N%~`or{fg~ zHbPt;7B!)m+DA!?Ff61VHSGHhX(}aHO?H<7bavvd??K-#Dyp@gNf}8u+<;zI+TZw7 zd6JT2{`(cUI~)1vAe3MQ?mqMEaGE(ics{psDKYp!{a0JAfvzBuzcgD@ zkCQD7?!~?K9=}sBIM`}F0<1wSNn$eTFzchJE^PT9ybJz84n<=WA8-W1vmQZuN^jEY zug9IBclp{Mn&~`uzDc?xbe$h19?DH3wYH0IS78ZZD~g6bXmhtKyIL(sZNjmREjJro zwL-ADdjcQJW3Z17lq2rrSDQUlwVF$suNa*txvuVf(zUt0CeM|PwBRN2$5u~o8Q%Dy zZV-WIU_boe)ZJd=QF$g6{Ck?_{`8*RDO8>rC6N>kO=Ia@P&dqbklhlY8&sSAo6nAwzk;3?mKsp1*P)oeaIi z!!mG9ow2a9r8W>TjT=d>!CHzTUQX12@`m}m5Cnwr&0eW?qimj3Ka8X_?F=#SH(2Iy z$n-a(^DEEFR0y6YBIniBTWF^cqZKiUGvl_D=;zqmrrx94o*up@K)|7*qH&e9)%e;y zN!Z|;_;71e9WZ{j%X}Z(N8a4TZA@MZO^sMY3Kp31gy9V1hYA z{s!8R6rcvKZ4))D1aPlG0jkkt=Wt#No_e zZmi}I{#-sb zM8!Xh!4%35xcpYf)cH#|clnvwP^}q0rGUIwFwqV*uCGq)+LYZi=ESTL?llE(+dPb9 zd;}+RPvEO0ua@YIeyRGfZq77D{*vHEbTOfsqhhMx|&fY5RYa3Ax-oR4(gbDFx zRwH7-NuiNqSV>}Fz_Z`&VsIBeoz5oEuTgy$<@szQo|qVl_4Mq1djE{Lmq1K7l#;9M z+1)vzvbhE}q&&fH4R315)ari)#Nk+G3Z0wbMU_~G#I`0s@C75o6wwtnewZ7EV?orI zYt3;WgCh9dB1U}4L+MS}gjpLf41o#;L>;Eb?tbBWjTt00Un zM!{3QQ>p)FBjXV7j-2(;P+uD3M=InNkab27b?>WCu3Mi##~8Ns+pB}Kmco^2IuW}v zkNMlgJn+4|34N??9hJ=$zcd*W4ld{sfGwPv*bW@!rk<4+=AGq|=X=h?=OSpdMM_W{oy^+b!Imv=RX|vF0NBTwlG{DD0q}yaMI@=#DBPy{6_qeZr}D#ZDD3&j$QyGk9H8D% z^sv@UIOFFVnD2Cy{Zd$u8#Z)4wa-45AnoU=JuMfW%((bNoD8(V;8s{G5U(f`C)u8I z`+c*Iezr%3HVLu->?llsPzPS

^|l&7Bl(1rlC_{V@g4 zHatxD)5FF{V?m$l&>pa;R;oQ*5ED-t`MNU+5gdh**T&~OI!ab~3QAUO^%m{Y*fR@d z@vAEEb3^wZ7R5)8Ud_h`qwc=|!NZ^Xe+ZsRw=dq}ipur5_E%0J*AS(JT1jXD;XQ+; zmw?k%F-FjV#7y;Fab#9i-l-wJb#-1U#b%ZzbKub3b=r_?j!xu5;is7m!I>hHo*NepA`6l1DGcgMR;>MZx{Py)T;-8Oy=h z!;AyAa{{HMY7J|C!IoYi9rSuLfsm@bkyXpg!8V&C)b0g3m|fVO{xUX8l`lF`E15g} zU+A|XC_IYjW>_IkFt_ujUm1|GuCPvPYKGzo_F(S5!&4vmIZc-K`II9Rx7?_^^Es3k zxq451-DmtCri5@9rnZ$EUo%}g$JQS!Yg&1n&4u(9E0eVHC{)`){AzT|i2};4fOE!= zt!@mOn2lYoA!=nlWp)EUSKHA^K+-6A{4z;D=iv!8`#5BykKLxo>49GhA>CGMcM6g# zCFZfTP|T8+^>ZGFL$Iude=QUILo^L)V{YCI4=yEr+{1c2=Y(n>RuWNmesZh3T!C5`DIMEZ6Ru#?;i1W@#y2wrl<$ z^6XyPAZ)Gh!n_ovbH$?w%A^;Y`M)-{7ozWU!N6V>)Ho>8DTaw zAMPWkVox4>R;ku1D=oTU*l%3zt|Ipno^|T^P0PQsjs6x=y;J8sZC&@Fzo`|g{rn$q z0yIItTz(JdGtX_W)Tk;xwZLGPMknJp!PB4CagDnTrjxJ;373U0?#@k<-vI`?bMe&| zjp`;M04jed_pVSB(=6&x z2=9pQwQNa$y38BlDCwSawfT_tHt8eLeh1b@3t;;@JjU}#^o!y=(K$xv%f<{{HTpKlo?YHFnP4 zJ3G7|&&Ttn%Y)zW53qS>`s_1+Wc_-C1YCYa5{l3XSYcILU@Q|hDwIjvaz=Xe$*dF! zEW(Cc0V-9waP_Y~?&}vm^K1T7z<s1^NZ7eKvB#hS0QR*FZ07sZ22X_%T4ISbjo* z-|2=Tu2;V@havZOWI7c70s{P`Xg7;^7)`7T3Kb@Q@n;1l2SBruSlO*FGg;3IqvHbq z+5z#gU0G&#za3{Q^H!jVx03CAy1VFtvmUl^5pa~x{-Sl)r*d1;Ay4QQEhuO_)J@8wZUnsKF@eqB`5i-&j^=qk(~R)@gaRkI(T&y_V|65i+i zwzTEG#0IyuJ_j0`byj;Z&oli*eGu0*s(U5BUhFE;YOFp@7JK}en)>DwG-g0R?{9uj zBm%5L35@#8Q6Oyg0aNQ?P%~YomuL^@4 zg+Js{w$Keu1&y)>>ep4H`lVa`P?W$=lf{za_Rtg4OjAoV(90Wcs4Kf!ur+NLr=#1Y z%J3o)jXccRb!S^CCONQzUQRtkr3|Po4$ux|9-U?&KL*4r;{@5QDw%3@Xw6?t}*kuid3)im90>V#7o7*yzq$EuM_-{$+TYlSbn~^fTiXG zGO^ER%0-^*-N6C3#-!jbT$on@(92D|VvnA2m+GJNlfJ+tmn~r4R5^4%gsNcUS!)c} z3*gg2#rI7V7C4vZ8hCdqMaMYpa^Es^4_JqpZZ3D==K$15I12zq02H6v5|TN}^WEVH z1hl|xSdV|eXQCdSM}UbnM4)e%cvJXeYIWLYdizN??PlnfT*P54OGy0bMSvAk-=W0i zZYP!3M{O_tqa9#DS7e>fuNN(&{}8t>e~mus;FCY?l_1_+)uS! zzWncJ>HWsZ7pn3bS1I@&x{I4@8A`$Ht5ZJjuGe12FT#GzLvRyDWjDF!<*s^~R$eo_ zn&5xmv%l)fUmk6J5Du`>Tqs0tVe&3<)*KGoT2^O{y4DO`iRU87VPM8^3NHN~pZfR9LDX%tZdDw@#UZX6v4 zO!$vZFVAm-TX{cIf_+ko0}v5h3VDo-q<-i76Qk)1M@0mNAVGhSAl&35$rR8vpVcR2 z7Q~I`T%vWf4ZMRMk+%9DTjZ4-+?ld&fEknbIwJZe&>|o|d|P<`y{TGi8tR0L6_Nko zl=Rau5CTc&x}3!gx&?hd3%=gFz8C6(J+KY2Bhqb@2Ub)WkkTlAjW6S;)BqAikPAZ? zPpW}Z1{0v6vy4{ccIu75bN|QvybpSP6GMQnC3B$sM%acc{w9MkzA`3&!}FRYE3ojH ze7gBls}D~~^|rRz<@Z80c@=t+lHP}!YC$S@KpLmORW5-=`nq-ymS?S8(ZiT(P#>X8 zsV$2|JAz*OW6%0V@N2k|$OS}{iJ~diK?qZ}g|cq(9jn8%DW0H&dhsC{?%I%y9)}0u zXGF%0u;uebs)+JWMxn!fV^9!ddoAB=`HQV?2RLzss(e87czyfj&IB(;%O3~}`mLop z6iH!{WDhbhg)u-MS5i3a3MsU*sNqa@c6fpEGph>^8%=SPmUH{kIq{9E32n2LJcCCnV>4D zM5rOTbIXU~azg^o{mnOr@w;J+IbH zV8oDyD-3=V%6`Go>u%AF|1E0cZd1Y)R~VqzI?Bd%!=9iNp2LfZM;xGssoqDbyS)|` z&dU*v^aL`JGm%g+I(u){^Zih9;pZviuoDk}8Lq3|o9WWy`Qca?N12n2(2(LK3GUU+ zaTwv|`~i;TE@j~{QF zXkLhul$r^s!DTg;bXj3qfhPxf46Xb#do4mNDpGi=r!ZEIxQiEO9WBzqkmNN8GnBHn zSG-w?$SW`_IZimEo8t_hQdQIhEy`C$y7y+P`8Jv+(ME9QIi_UqL}+%Xg4ieK+0@0_ zz;igPY}$rWZ~sx>?ptkZ7Nh|S69)z1iax&`l93t{_;^V<`K~i3A^pE#9tV0dta@QkyvgI5E4iepH)y_=5&D+EO|n}kwQZ0 zuxQz(mm}ID22&smFLnMP2O})BFp~fmuS}_tGx7dkFdC%4Rdx@)Gol7UeGwZPr@D=I^Co{^rcH_50WPZ5{fdF@( zdUFry2f&x;qg_&qMMr+mm0yXKJ9D%3)O{zz`-rNp48*-cSqU;z(3%;Ff-x#%^8r)z zorKyLR{}#0?eL(0=G=651Lmv8H!hI2$MXJ+%qMeQni)>Ll~}PEAhTuo9ud8~{(-?V zC4;twimCWSNbhFogn%r4LcZLa-aiBjgAS2V$Z0h|Ro3w2?IHLXJv^%37oPv>VI55y zQu14fEi^%rlKbkF{ETF51{-CjRDzc$kJv{UWX4@x*8MWQJ{yTa=L=)CIjMT%C;iFQ z68{u~m@yo7bdfWDO>rR$oa(TQ!}(wIVh{29_c`AnbV)B6d@*(U->FO-e55G0&I1qHHw>?=8}VpETwLuF-Ys%HtApzxdvd z1-B%rcDsL(IeKLg)Z=`$Ox($S7!oiY!EVdMyoW^02|1ZpcItaysDKoi58j;NCfXu+dY9_~SeNvmW$` z%2ms5Q&7(Jz70_Ps2d9&=)77Fh?J^fZC;aZHlJGm`zmw#@gafRQXZ6twN+(^u!%r9 zKL1(=^Gouj&f%OKJx6}`emHmhVT;})?!n~)Zu$G#FcqGDg2M2GSnzlc=9a@_l*lFg z63+q=ddtlP4Y*wHDE`ss$F;`SF%>`NGp?SW-c1@D_J>QGy&EaLBLXxS{W8dmm(t@9SI ziol5%7;oF7eD&%_)du{%=_FDKUTlgb`j|Mf;?UamU(7$D5 z_-2>d#UL;E8Qj$@E_uIgpV^QnGVLUFn~`%P5)W3412D{sPkIkX_6aF#+4u9iqLo2` zLou)2RgRLrqk7iSl*2l%*n+^7d*Lmr*D){@n{&!$p^bIb@bhaJQr&9PuenJ2%i>_q1n5b2TivA@(6!T zxsxA{&f{n>_V?7F za6nHf0yL))EKD-N8_fCLpZy5eqR5l!TBOGcNkWPLj@~+Zgs05nD(K5ckvHU>htQ?{ zSz5qB$?DpyNt+rKqGxf;Y@+Z&R^T3EfCu@C=;wVs+SKMM-4~|oj_F6C3K2`D)Eq-4 zFsic=5r4|dU4>HA>n%fWiGU*l)P^u$y%wALB#NDw{x;*5#PZPBVWAd>fDV`_>o)XBpBh=Dx~+nf`{f4)+<;6wPI z)_H+Cxm@y^xEknE!kG|c(fS-*iO{V)ZGMx!u3(=@4W=`S3lMwwGB{F(nm6-_qzHIg zOV8hY*CCvLIzy$^Z<>HzLEi*AdPGw-q9w&n2J5g2>OF^W_j) z(PX7G05Zw>5{0!6DL9jb5{@yZUg2mNO!_DWerBM7o_w#kq7t-ZbRAlcrpqy9hl~*s zX=S3AE5*>aays9x{_2(KT9MQR;qT`^7H!=2Z@LmkfByQFgF_Cll=Z;k1zO1M)`yNI zLn+?JBC+p=gW6Z_32U5w<%s_P@~oKDVD06Yg6XLSrQ%13N=YD6@tWQA_J>!^EZrMY*}oK8-+ z{E1vu(+{cy4e3{PR%!N&n?Yd#y6JYJ@{`>OQ+p#i-c(|GgnW$bWjo3?MFkatZd00JXhVXR#=t9&g+f;VI97?dsTjz#CHa?>s;ei6x|Ns=TA<5xPFp zvzW)#DVjiuBJqj!8!xY~XzvuA_-Ljk)@7L#mX zS`4w+`{NfRk)8-5|C;Kko=}gWd@e5K&le94&f|Tph1b**)hl|S&qhC|=vClz<^6<= zP(TXe3Z_hSEf{*aHQh4oD7_2anu>mB1c}+kj)46l=f!EvwJ)TLdo(tfyHomg;GDOn z$d6L}-L&u^@K37tdUt;Mr5|T-OIvt0d2z{;M@T@u?>fwrWu0!aa9N@8;J~0Njk*gH z;+d)XhgnbNR08e42<&}Xf5(mmx1zaBI`hgsP85lee>e`49;CJ(84W1v>bs0b)!;Dm zYs9ST3^TIhYIqMac>f9Pa=y}fBs?^JD}%9%mxLwHU{y~9bwWZQsXE+zR1cvfSbrlZ zX}hmF^#cU1`txxlVlD8uTI!>t_B%E_vlTDCXz>xUQ5V){CQ!cY-mqz(vGp-0tWM&> zgOn_Fn63b$YO&p0>^W3%f4r>H%H`6(pD{FzH4qD**X8OpRfsyLFP_lr6^S{>;Htki z=w;$0Xmg!Ds)u^da)96Q6H^hen|bXvwNudb=Ur<$;vE&r9|wZ@!EJ@X`t+%7mmU44 z{ST#Xy%)2F8u%vK^!7rmyA~e4tyeMPuQ>q;=cR#U-5IoI@>=VjQ>%dnXbBKXOB6k+ zqIPOxbqEmm@CbpUcxS0LjVXs}Ab}DAgCC9RDv(V1Lm;8eyV|o(q$4eZ)fISqo)*fG zw?zXdJVEddkxmL@ns#!i+zI z@5#@Pp+&~O6J@Jl_9bItSIJzZWmUWCkLw9NP&jijzg4JW;3#-XQnmVI4dE)s%dl1L z^G;45MCVSekQ#??Nx+zYgz&>SoDUy`fXP?y4j~&QcA%ko+H#QxBkDd}x*xA1UUFWJ z`?i33VgysAsARlFeGj5s64>949LV^Uuqarj&ca=Bxr`1zT=3}EXKofRyaq_aJLPq2 zPrEuKsi^lm%~nQxGbu!udY&;LD}+M9x75`e)T?jVBp%YE4*tPbja2E=cXt!h%?Vag;YN zMZ4mmAv@@}yveT0C5~qr#JVsp&TPY2eis^D4Li9Y?jt z4qOS}r^#-gLU`zOsmto!51u_mH`W0_Xg?3;5>TN`+A1_-TcxtLCD2#n<3K zdmdrrIQp4n_S~9oR#0hB{a|q&mXAA(g;m!n6T2ghb3SOd6kbB-)TxrHCzO=Le|WWS z%(mGGn!tI^Rn%rC2h(LuY^Sqd`QpCr+&y*mx;wYFcQGA#kJFA12y&NPSM9c;meEGkUqXebdWmO?( zR=K3+E?$@VpI`WbJ3ZDdLs^v%>kMmECFPVXxt4u~2Ai+{u`s+@^5j^|zC^7mZTkO* z9ilc^_|Lq&vyxakAtH_<6=XEn{=G}}+66A(r_KPy)I={at!%`q61l}%zz#ZU?MqXm zV(Qf8?&!DuPZ^PnnBJCBo0PGeT^cr2PPSwjZeQsA^?%~zNJ!NNU9xGU;Q!#WH;&>) zFW@Y`6)WZW-w&oZ0j48#l0|6s&VNYx|7#TKd6lF2l4X(9)+{I6|E~dDR zaMw)-RFgu%4Mj~G->jZ4wrub{PB;Gazn?))L#Dk=Z~x;M0d$^@AJux8xtDQ} zR2fkC=xP5`g@9~UtEXSTb$l<|`o&B%EOTM_H1HDn)Th_~JCq>{0$P)PI;na6^zq7p zUWJr}&*>J!_1hOdH|_9=GFOgngt*ruCntMJdzSFK>vcedka0(h^n#4X!wN~()f;&% z(b988@mj}mc^=RuGZ)bJ^fIFIy zBc}SK3OBum(j{>AVwa*&sHG^g&%^q8&q<>!Y@!oT17GP1jDn0fyfNEwD}3^hRv70P z`T(G9cgy5q$2L@N%}UpA+Qdrx1mkG6W_u$t?P?4GX=2c?I9Wy|R; zE}qd={@zLF(^E&s98yS=uVE0&vp3v#V(LsTVjG76df<$h*4w@*(F3Mr@l(LQbG|#p zAJ?K&B;Ut*4zn$F#7&y&wfsQlB$+%fd2<6`VWBJ&j({ckmF*y0ou^g5{nvVX<0D`k z%NDTtwvBA@O$(D-;heAy)s(%r2%+uq`n9?MAu!<1K?5 zcP$Av$4H#ui-B9*jC}3h)L`xHC1Q4oU-AAXd~dryu1FS;oMHh699Ico;o2<#R;1Fg zb9oufEk9qx+&G=3n>x*%C&~;JfLk(IT2_&UhE@)<?a28Yb;WGIE1k zUJY36DxZXBb0R$^vWom?iU*$LTO3wJ2(Llgu5p#YF)McLrWPJQVf9DpF-&THM-APN z9~KUZj)`Q_tpYMZ zdR72T6e-&0Gx7Nzu!mRYb&7+wCv}h<@8}t&^78&*n94qGS$_&hgy&-`0i#it#&E#X zoxDMBV_y&fzJUF_1)y!m`|orv)6{C=3J1l2js#_jX<;6 zJ=8vsiVz2Pp521>@aEWF?Esaiy)r6S&_TSeDRZ~hEkIlH$zD+&7eUP`GJvLMCsl6> z8zckIj<(9b52Zc%mrE7Me~^s)ga23L4Im|%rqP}`y9fhvGeM1srT^*{u+?up<}>U+ zEWxVqVj{|_YW{g&_KWS*@QVs)7L zP#pRn&-5Ga50!#6P)=DO9wO2~ZTAku+Ip*31E0cnzv2h&`9zM@@jYc zJrvlD(^Yt#^wgvTFp|s2o{R}E-3%kn+!&UTpAHeKo8}q}L6PdzC;or3cW|&d`+7Gu&BqMf0AQ9Cw2* z_R&wo!{htEPZlP}wS=+Il9>M%TvNm+$C(!6`;#vYq>9sm2AO*rQdSOlHCOMmhf@p1 z9D;J3eVod~#|&8dAABEBF%RJarSq{ZG%;%~wjE|hwD}gU z_l!qD{@iOlzA;VZGgsnUfDyf;n=Kv~j$mM2=~!pwyQn^GU%lfP?eVmz?dQn~Av-WH zc@nTbABB}4CFQo17cJ%k&eV{{3AjpRDf^_0&kzZcZCV57)aqW(tyM<7{adY$*MxA= z)q(d{`d;#0qZ2RMcHnO5LXPZL+UzpUf4hpIxRS)oF`n{My?UjN4Dv&ZXBh+n7bd+}y5Bd$c8U&$^U(?I+tc@k zXA`y$TUeH|Fa2+Lzb-O=9eUi@`>P)JwZCob5~3U4v>~c^x0!;#Gzds#8M_5;n`90(u+x zP{_k*5X!Mi9>^u|5g|+5q)}%Xl_6c}WY{6A+F(&vdCx)OrcLSNybd%Z zXVb}Dm2Lxpix+swC#n%(EQ($KHPG$MXUVSxI|L?(+1~$*XY?@6%4)m7B2tOHCv(Fz z9iLmc>K!}X)(wc}v2p_tfYDs^@eKpzl}Y>K1yPL#i1)5Gq{^H zV}Ww$6$p8-W$Y4BOmM7=upe+FPGJHcx$nhjX ztm`a<-x&3UG&&vM>TJyV(F=T!wwvUHjzuHjdh%s3@%7%dEY;%S&bCt^;TCoId%&C{ zkL}@JPcelJ$_V5Ot4<*Q`q0=Z{ey5G8Bkwp6fwCG34o|M0#E*gsedD=vV}11cL^pE z(_91}L|J8+G_EjMaCn_&fHOAAWC1AlCadT=@RZZdC=-JXbs;`wofDm1b1PV_Su)B@ zn>EoJ|Cc>~HWKW!)yeLe@+b$zb8gFaA4#c*s#>B6RGnfr{pWlJ3LqANIS=~gye@rm zx{^AiXu?&KK*`P8x*&g8J0DrjFki1jtT8MXa`>`3xd{xah@~N@ zNGP-)g=*VH|3|2`d=IvU)d-!k07b^6pmgicYiJ2>vMpHfxbMeI}i+aVfA zLcOEi%`^L~Urh{7s|WJl;vkDoPuv2z&pm1T3W#JF;jTgDe7PHR zS3D~isa`6FXK6)0;fgYcJn4F88oQs;XzcuS_cF!b9HiEzMP_u zB2|BY7JMplg13)&IyJIwdEB%jLMY#};%s@C<~=wiP@M58NL)XR#q-J8AQPVKq|VC_ z6@!5sT0J>Y9Vg19Byt}ncFnQi9lj}@%W)|B07xL|j`Cw@-a8}$0}y`yiIi9&ieC@c z6zrQ+gJT{%wl84A{#FyszDb~bVgdS89igJ{GxRSw^I23hs~(1pF+20j=IgDI&o*6)^(gPREyx`3SnrSm zdvQvO^o`4Z{J0A{>8-I=V)XKwnqDIGR|RUNUwURBeEju*09gTZZ-Wpa?AQD(kjNR2 zB+Y^Ca@uj5p+*TVLfW8WiDG(xOwBVf`ZoMywEhnrc0*HLzxl;G2f$@+=@R%O^5Kxe9_-hF;ZQjxmngz`K5}QI<6V?WN-y zAoiKQh!+eNx2H)1=cO#yEVJ5mr~~w+m{}AgDx0@`=PZ!w3Z)gp%D*hAmUg5e zox{LLtAWM$Kht0dx~e+=CjGHY|0aV~gnw#AUAuV7RM5zYIN*MNULkoX>p<8QkfTgE zyXz~G#b)Jq9_)%Y7Gf#ULmLPEi_9|BnE}hZ&pKw#ECdX`+f3wp^Y0~cZF{FgiSz>H z7~h{FAcgonqJ}nKR0aj^{uq2EazYg6=~OCyN9KfUp6K3ZJ}?Xw?pfDmS5?azzl}}I zms8JzB1&4pYKPk$w*t8Ms|h*vl&y83_Bk64i97Q*QD2D}elU`1wLR2rC@LP@*N30O zNZXepI;!5%f9lvA*<;RpP@@{|iT57I#~XBHbT`m|udT1U`Cql%hkC_XiD-~v&ac}x zluuN%I&;Z3HEx~rMa zl0h!`y-^ZKh z_YkA$lV79FC(4O(2RqxK3Wx25uURi7g$3Ybvy4uBA|)Aht4l30I_)u>Ok_Wj8BICh zvm96&a~dw4py)ctl;l7EtN_ad54lVyJeK`+;EUb>&&2wjggFht~!o76W5CW@= z;vDFCh6gCGH!8t|pn+AA3`S-H6BNgqbe5V#W^`?@evN}GOY;^`bK?@%Q+-F;D=kYz zIOJW?X%xV3u`i(~dh0&5aM%6367AR~(U;P=upP0s9lV%pi@lDxxl zA}=EMYAZiiqx1{O5D$3Ry&@{_?>ir3{|Z(ySXaO)y{--rw0)GM8xl0<9IyD@ZDUnX zMOmL?Mdd>;DCb51EEdQzEQ5)bAO@?*#y@70vc$gLB%a#j4|!}ymKb;Y0VD5luDDm= z5Tq2j6opEMd)`>VWB;lvak9(pDCULH=Rc>qNi%#`?oJ&%A?F;iO&lxc`}-~YTXsjXzF$BjKx zG4BiFHElI?l+h;=F}gLbyO2sn9ICHR4_8T*7UBd8L3+Y33|?PMx#b`aRO_CvaLA0U zs^?Ijh=+l{Of_TmZcJfW16bW zLvjw7-#h;Ipa&imLEf0#pdg_3;@-2W8VH#WE>4LjibW`a7%h(A`k;V*u()@U@B6c%+0j zB@?@v>a*(MXOHvaAw#L_7u*WScd8ks`a_#t@s+)?m+{N$O%Id?8={61Y%nYJNx~c> zGTqn*Gc>=!nlXJ9boQT zU6V90&%v(XxT`;w`i?on`&(CPe&_jg)0;=LSM^C#yut9*5p|Oh)!xmFs<#PkaLVy# zyXRGgyc-=ss|4Ceuz}+n-8bjE|AktKI%J1(A7;{Rf-R-4(mMXpvx^bRl=)8k3FK?Z zlkjg5XKpBWlLe_&iO-I_{+s3u_upFYja8sQF?(sJ)wQXLR9ot1U$ZzW5~PTS2iabC z5+!V-1gnaPp(J>z3`bmiZY*bUkECo}g^xbAlgiF%3sfi_8qDvL|I$RGb8BE-HZ zBV8b)yQQeAwIR(DrBFnZoF|`C7-MjIdW&3+AuQ3h?gdvKBiyG5kJyT+gUy6SYt=dG zV!T*SPa|0!IZR>VgE;IfUC1 zi~dDGpay+9bYe@gp}8?HCM1S#Sg6WKKu(h08Ts0udqss* z_6LD5EM6R2dv7OP1+8K46$xZyXh<5CUDgwat1ns>k*w!${4p2xf#hIqRFdd2^iRd3(BG)E~svB(bwap|TD z(HiOerDfx8nsE}j`}9gC_NeS>OVNh^6ibmCV%oy3P>sAz#@uz80-+=IF1RX6Pj@XA zgq)=7GI3_u8q=swbMld26zio|2`q{4FdYZ}m3w%(R3cZ$#D(p+a&oguBlBO&`4M=! z*#bbK2@G15d8>4qZFr)G*TOvCc1ckEGIgdu*{QK+a2*B+Zj0VN05 zKh6KDCw-xJrcWeduh#+xoqDE{&~wr=6^i)zv^++98&rs-o5oZuo{?`zsuoSABbyM- zuY!ZV4!h~TuY1OZwHb-=av?P!%9<%v2?@7;#e946-^RUu#1p!v2oRMgcOQQELFt_e+CcbUxinNx4ptgNRyQyA55MTBk&FW9eMI{jnUg<7`jJp z-%b5W>+qBgk-HRrlzS$FiPEEf#)Y`Xap#lz2N7!k}!il3@GWxiZu`C0nj>jgYY&I^-rsD26y^Xk7EONERI zfC|6(wvpX-?1^#)9X7;zBu;u4XniFbUoNa|~88Y+zDSYB4fR0jSO z9m2J^YC8mjA((z$PB}^9Bv#nKeER*bNICIMU0SyLC}=PH|zl0t@opQ{???fo)sl-FLoT8dA}@GN0@M7 znWyv~<0c~DIs1%q8uFjq0uHJ18&sezQInf>Z<>aL#i-Zbf`1zH=2KW*qU(((;o@am zq(MIG8`{ z`}0)xS4r7}g>PWBY8}<)?TKFuSr(SX-gKeq50m;G9rNC_y}8>L_i6N>LHKg`lQxe7 zFhU@M!v|l9j;W9DU|KwD?8Op~)zkMI*fg94e^~nGo?UyS#3te(@-P%?CZEU7#2J01&ObDljCYxGO=R+5*#d9Vsz3gT((22UBm}; z4qYsdoab+_L(?p_c5u;S&J_E!_o`;IvLvyc7icrR%|r8aH}8xEJU?%a%6*kr_nK;4 zyw3^QTJ%QkvFTp-ms5t%M>^kf+yc?XQN&ed-)ndKd=@3+c#k5_C$no$mpnc+Qw0BN zkSLSOYx;Bj$QQ;@(FbzF71ipfZ6c(s3nM;)J^UwT5-ZAvikdR;?^3`roBGH&EQgJ* z7bnylJvnxRRj4m^f%95XlW)<_T@~G+$J$alM@-^2d!B@U(6S3vusEx`@PBr?C)LxE^%$NJ zhq)ZYeDR&guXfC@gVumOR-j!)Qgo=rOhaKYz1Ttq5R`Firr zvkp~M{vY@#>axi#H;`En%f)ZD&VA@mAp(4kpzcD3VmWw>!1^9L<@D_;4(*CF3dv(C zU;3-({}alBTI+IG^v1W$-+itAKt`M2d~~e zUdeA+hGLWK{(m0nO#>|oKMCJ!!Qez;Rb|une?NLT0Fc@}hmXly|NF_`;TzTI+#|~{ zRgXLFXHQ>?by!bh11&GwMxZ z?qh(bw7nXKz4~TXzRTbF-MIT+AZIXgM^7%ubDHt@`SG*i=T<9<{R5-0o%moey@80U z0fUopK3htPZ9UWKGx04ibaS8J)PKLMH9pD*64os7eI3T2!!xgPb(ItGj*U0q>yrY^dJnB{W?COx9*s5cl>1wBQc5jVpP)?AT{i}ZC zBXuUD(d^#O3CG1l3>+7(90Bg+cs=05d#08`x`lT81(s-+h>sVkkz&TV8NFE5E2aPE z?+Fsq+dshES-s`Vr_~PNd1Fe@VsQJvt)qjt)x^T)k2egQ3*(@_1GfinIZbIc|98JI z+XxGyeec$Dq31g_>Ux3*}JJrq0wFeBAem1q2|K{|zUjE#!Xs&o}pY*h}@bh!0zjYf? zd`9Ih*gm`DKfImJl|YcoYf!TLp)wliSTqqVuTrWV5c7ie=DT&=3|tfRtlujAp4S|B zI63|TKvR=9i1%zW;VPZv>>F@^F^R%@<-+KVKUBefK~2Yd>AIKNKI5j)M+rfVxGcl3 zO@rH|_Qc%}n+~zrrlvdBb(&|jlrp^c0H$a@PSJ^eWq~E&^m-K|`HQhWJTb22PcO_> z?g}XQHTw#jHl&k!uf_i-or}o3bL21uG!5JF$o9s(x)KcM?+Otr9#R2 G-TwhX!}pf} literal 0 HcmV?d00001 diff --git a/alf/experience_replayers/replay_buffer.py b/alf/experience_replayers/replay_buffer.py index 6e95fe334..83eceb153 100644 --- a/alf/experience_replayers/replay_buffer.py +++ b/alf/experience_replayers/replay_buffer.py @@ -29,12 +29,20 @@ from .segment_tree import SumSegmentTree, MaxSegmentTree +# her (Tensor): of shape (batch_size, batch_length) indicating which transitions are relabeled +# with hindsight. +# future_distance (Tensor): of shape (batch_size, batch_length), is the distance from +# the transition's end state to the sampled future state in terms of number of +# environment steps. future_distance[:, 0] == future_distance[:, n], so only the first step +# is accurate. BatchInfo = namedtuple( "BatchInfo", [ "env_ids", "positions", "importance_weights", "replay_buffer", + "her", + "future_distance", "discounted_return", ], default_value=()) @@ -71,6 +79,7 @@ def __init__(self, gamma=.99, reward_clip=None, enable_checkpoint=False, + convert_only_minibatch_to_device=False, name="ReplayBuffer"): """ Args: @@ -151,6 +160,7 @@ def __init__(self, self._recent_data_steps = recent_data_steps self._recent_data_ratio = recent_data_ratio self._with_replacement = with_replacement + self._convert_only_minibatch_to_device = convert_only_minibatch_to_device if self._keep_episodic_info: # _indexed_pos records for each timestep of experience in the # buffer the raw position of the first step of the episode in @@ -426,7 +436,8 @@ def get_batch(self, batch_size, batch_length): # Taking first timestep's return, to lowerbound training value. disc_ret = self._episodic_discounted_return[(env_ids, idx[:, 0])] info = info._replace(discounted_return=disc_ret) - if alf.get_default_device() != self._device: + if (not self._convert_only_minibatch_to_device + and alf.get_default_device() != self._device): result, info = convert_device((result, info)) info = info._replace(replay_buffer=self) return result, info @@ -836,7 +847,10 @@ def get_field(self, field_name, env_ids, positions): lambda name: alf.nest.get_field(self._buffer, name), field_name) indices = (env_ids, self.circular(positions)) result = alf.nest.map_structure(lambda x: x[indices], field) - return convert_device(result) + if self._convert_only_minibatch_to_device: + return result + else: + return convert_device(result) @property def total_size(self): diff --git a/alf/utils/data_buffer_test.py b/alf/utils/data_buffer_test.py index 28acc99ba..50f06ee61 100644 --- a/alf/utils/data_buffer_test.py +++ b/alf/utils/data_buffer_test.py @@ -29,7 +29,7 @@ DataItem = alf.data_structures.namedtuple( "DataItem", [ "env_id", "x", "o", "reward", "step_type", "batch_info", - "replay_buffer", "rollout_info_field" + "replay_buffer", "rollout_info_field", "discount" ], default_value=()) @@ -40,12 +40,20 @@ def get_batch(env_ids, dim, t, x): batch_size = len(env_ids) x = torch.as_tensor(x, dtype=torch.float32, device="cpu") t = torch.as_tensor(t, dtype=torch.int32, device="cpu") - ox = (x * torch.arange( - batch_size, dtype=torch.float32, requires_grad=True, - device="cpu").unsqueeze(1) * torch.arange( - dim, dtype=torch.float32, requires_grad=True, - device="cpu").unsqueeze(0)) - a = x * torch.ones(batch_size, dtype=torch.float32, device="cpu") + # ox = (x * torch.arange( + # batch_size, dtype=torch.float32, requires_grad=True, + # device="cpu").unsqueeze(1) * torch.arange( + # dim, dtype=torch.float32, requires_grad=True, + # device="cpu").unsqueeze(0)) + if batch_size > 1 and x.ndim > 0 and batch_size == x.shape[0]: + a = x + else: + a = x * torch.ones(batch_size, dtype=torch.float32, device="cpu") + if batch_size > 1 and t.ndim > 0 and batch_size == t.shape[0]: + pass + else: + t = t * torch.ones(batch_size, dtype=torch.int32, device="cpu") + ox = a.unsqueeze(1).clone().requires_grad_(True) g = torch.zeros(batch_size, dtype=torch.float32, device="cpu") # reward function adapted from ReplayBuffer: default_reward_fn r = torch.where( @@ -60,6 +68,10 @@ def get_batch(env_ids, dim, t, x): "a": a, "g": g }), + discount=torch.tensor( + t != alf.data_structures.StepType.LAST, + dtype=torch.float32, + device="cpu"), reward=r) @@ -79,6 +91,7 @@ def __init__(self, *args): "a": alf.TensorSpec(shape=(), dtype=torch.float32), "g": alf.TensorSpec(shape=(), dtype=torch.float32) }), + discount=alf.TensorSpec(shape=(), dtype=torch.float32), reward=alf.TensorSpec(shape=(), dtype=torch.float32)) @parameterized.named_parameters([ diff --git a/alf/utils/math_ops.py b/alf/utils/math_ops.py index 79368c449..136f3e139 100644 --- a/alf/utils/math_ops.py +++ b/alf/utils/math_ops.py @@ -14,6 +14,7 @@ """Various math ops.""" import functools +import numpy as np import torch import torch.nn as nn @@ -457,3 +458,36 @@ def transform(self, x): def inverse_transform(self, y): return y.sign() * ((y / self._alpha).abs().exp() - 1) + + +@alf.configurable +def l2_dist_close_reward_fn(achieved_goal, goal, threshold=.05): + """Giving -1/0 reward based on how close the achieved state is to the goal state. + + Args: + achieved_goal (Tensor): achieved state, of shape ``[batch_size, batch_length, ...]`` + goal (Tensor): goal state, of shape ``[batch_size, batch_length, ...]`` + threshold (float): L2 distance threshold for the reward. + + Returns: + Tensor for -1/0 reward of shape ``[batch_size, batch_length]``. + """ + + if goal.dim() == 2: # when goals are 1-dimentional + assert achieved_goal.dim() == goal.dim() + achieved_goal = achieved_goal.unsqueeze(2) + goal = goal.unsqueeze(2) + return -(torch.norm(achieved_goal - goal, dim=2) >= threshold).to( + torch.float32) + + +@alf.configurable +def l2_dist_close_reward_fn_np(achieved_goal, goal, threshold=.05): + # Only used in non batched cases. + return l2_dist_close_np(achieved_goal, goal, threshold) + + +def l2_dist_close_np(achieved_goal, goal, threshold): + return np.where( + np.linalg.norm(achieved_goal - goal) < threshold, + np.zeros(1, dtype=np.float32), -np.ones(1, dtype=np.float32)) diff --git a/alf/utils/value_ops.py b/alf/utils/value_ops.py index 8c36deff4..2265381b5 100644 --- a/alf/utils/value_ops.py +++ b/alf/utils/value_ops.py @@ -118,6 +118,67 @@ def action_importance_ratio(action_distribution, return importance_ratio, importance_ratio_clipped +def generalized_advantage_estimation(rewards, + values, + step_types, + discounts, + td_lambda=1.0, + time_major=True): + """Computes generalized advantage estimation (GAE) for the first T-1 steps. + For theory, see + "High-Dimensional Continuous Control Using Generalized Advantage Estimation" + by John Schulman, Philipp Moritz et al. + See https://arxiv.org/abs/1506.02438 for full paper. + The difference between this function and the one tf_agents.utils.value_ops + is that the accumulated_td is reset to 0 for is_last steps in this function. + Define abbreviations: + - B: batch size representing number of trajectories + - T: number of steps per trajectory + Args: + rewards (Tensor): shape is [T, B] (or [T]) representing rewards. + values (Tensor): shape is [T,B] (or [T]) representing values. + step_types (Tensor): shape is [T,B] (or [T]) representing step types. + discounts (Tensor): shape is [T, B] (or [T]) representing discounts. + td_lambda (float): A scalar between [0, 1]. It's used for variance + reduction in temporal difference. + time_major (bool): Whether input tensors are time major. + False means input tensors have shape [B, T]. + Returns: + A tensor with shape [T-1, B] representing advantages. Shape is [B, T-1] + when time_major is false. + """ + + if not time_major: + discounts = discounts.transpose(0, 1) + rewards = rewards.transpose(0, 1) + values = values.transpose(0, 1) + step_types = step_types.transpose(0, 1) + + assert values.shape[0] >= 2, ("The sequence length needs to be " + "at least 2. Got {s}".format( + s=values.shape[0])) + + is_lasts = (step_types == StepType.LAST).to(dtype=torch.float32) + is_lasts = common.expand_dims_as(is_lasts, values) + discounts = common.expand_dims_as(discounts, values) + + weighted_discounts = discounts[1:] * td_lambda + + advs = torch.zeros_like(values) + delta = rewards[1:] + discounts[1:] * values[1:] - values[:-1] + + with torch.no_grad(): + for t in reversed(range(rewards.shape[0] - 1)): + advs[t] = (1 - is_lasts[t]) * \ + (delta[t] + weighted_discounts[t] * advs[t + 1]) + advs = advs[:-1] + + if not time_major: + advs = advs.transpose(0, 1) + + return advs.detach() + + def discounted_return(rewards, values, step_types, discounts, time_major=True): """Computes discounted return for the first T-1 steps. @@ -180,6 +241,69 @@ def discounted_return(rewards, values, step_types, discounts, time_major=True): return rets.detach() +def first_step_future_discounted_returns(rewards, + values, + step_types, + discounts, + time_major=True): + """Computes future 1 to n step discounted returns for the first step. + + Define abbreviations: + + - B: batch size representing number of trajectories + - T: number of steps per trajectory + + Args: + rewards (Tensor): shape is [T, B] (or [T]) representing rewards. + values (Tensor): shape is [T,B] (or [T]) representing values. + step_types (Tensor): shape is [T,B] (or [T]) representing step types. + discounts (Tensor): shape is [T, B] (or [T]) representing discounts. + time_major (bool): Whether input tensors are time major. + False means input tensors have shape [B, T]. + + Returns: + A tensor with shape [T-1, B] (or [T-1]) representing the discounted + returns. Shape is [B, T-1] when time_major is false. + """ + if not time_major: + discounts = discounts.transpose(0, 1) + rewards = rewards.transpose(0, 1) + values = values.transpose(0, 1) + step_types = step_types.transpose(0, 1) + + assert values.shape[0] >= 2, ("The sequence length needs to be " + "at least 2. Got {s}".format( + s=values.shape[0])) + + is_lasts = (step_types == StepType.LAST).to(dtype=torch.float32) + is_lasts = common.expand_dims_as(is_lasts, values) + discounts = common.expand_dims_as(discounts, values) + + accw = torch.ones_like(values) + accw[0] = (1 - is_lasts[0]) * discounts[1] + rets = torch.zeros_like(values) + rets[0] = rewards[1] * (1 - is_lasts[0]) + accw[0] * values[1] + # When ith is LAST, v[i+1] shouldn't be used in computing ret[i]. When disc[i] == 0, v[i] isn't used in computing ret[i-1]. + # when 2nd is LAST, ret[0] = r[1] + disc[1] * v[1], ret[1] = r[1] + disc[1] * (r[2] + disc[2] * v[2]), ret[2] = r[1] + disc[1] * (r[2] + disc[2] * v[2]) + # r[t] = (1 - is_last[t]) * reward[t + 1] + # acc_return_to[t] = acc_return_to[t - 1] + r[t] + # bootstrapped_return[t] = r[t] + (1 - is_last[t + 1]) * discounts[t + 1] * v[t + 1] + with torch.no_grad(): + for t in range(rewards.shape[0] - 2): + accw[t + 1] = accw[t] * (1 - is_lasts[t + 1]) * discounts[t + 2] + rets[t + 1] = ( + rets[t] + rewards[t + 2] * (1 - is_lasts[t + 1]) * accw[t] + + values[t + 2] * accw[t + 1] - + accw[t] * values[t + 1] * (1 - is_lasts[t + 1])) + + rets = rets[:-1] + + if not time_major: + rets = rets.transpose(0, 1) + + return rets.detach() + + def one_step_discounted_return(rewards, values, step_types, discounts): """Calculate the one step discounted return for the first T-1 steps. @@ -213,12 +337,15 @@ def one_step_discounted_return(rewards, values, step_types, discounts): return rets.detach() -def generalized_advantage_estimation(rewards, - values, - step_types, - discounts, - td_lambda=1.0, - time_major=True): +def generalized_advantage_estimation_retrace(rewards, + values, + step_types, + discounts, + target_value, + importance_ratio, + use_retrace=False, + td_lambda=1.0, + time_major=True): """Computes generalized advantage estimation (GAE) for the first T-1 steps. For theory, see @@ -254,6 +381,9 @@ def generalized_advantage_estimation(rewards, rewards = rewards.transpose(0, 1) values = values.transpose(0, 1) step_types = step_types.transpose(0, 1) + if use_retrace: + importance_ratio = importance_ratio.transpose(0, 1) + target_value = target_value.transpose(0, 1) assert values.shape[0] >= 2, ("The sequence length needs to be " "at least 2. Got {s}".format( @@ -263,16 +393,23 @@ def generalized_advantage_estimation(rewards, is_lasts = common.expand_dims_as(is_lasts, values) discounts = common.expand_dims_as(discounts, values) - weighted_discounts = discounts[1:] * td_lambda - advs = torch.zeros_like(values) - delta = rewards[1:] + discounts[1:] * values[1:] - values[:-1] - - with torch.no_grad(): - for t in reversed(range(rewards.shape[0] - 1)): - advs[t] = (1 - is_lasts[t]) * \ - (delta[t] + weighted_discounts[t] * advs[t + 1]) - advs = advs[:-1] + if use_retrace == False: + weighted_discounts = discounts[1:] * td_lambda + delta = rewards[1:] + discounts[1:] * values[1:] - values[:-1] + with torch.no_grad(): + for t in reversed(range(rewards.shape[0] - 1)): + advs[t] = (1 - is_lasts[t]) * \ + (delta[t] + weighted_discounts[t] * advs[t + 1]) + advs = advs[:-1] + else: + delta = (rewards[1:] + discounts[1:] * target_value[1:] - values[:-1]) + weighted_discounts = discounts[1:] * td_lambda * importance_ratio[:-1] + with torch.no_grad(): + for t in reversed(range(rewards.shape[0] - 1)): + advs[t] = (1 - is_lasts[t]) * \ + (delta[t] + weighted_discounts[t] * advs[t + 1]) + advs = advs[:-1] if not time_major: advs = advs.transpose(0, 1) diff --git a/alf/utils/value_ops_test.py b/alf/utils/value_ops_test.py index ebd526127..6477edbb2 100644 --- a/alf/utils/value_ops_test.py +++ b/alf/utils/value_ops_test.py @@ -23,23 +23,46 @@ class DiscountedReturnTest(unittest.TestCase): """Tests for alf.utils.value_ops.discounted_return """ - def _check(self, rewards, values, step_types, discounts, expected): - np.testing.assert_array_almost_equal( - value_ops.discounted_return( + def _check(self, + rewards, + values, + step_types, + discounts, + expected, + future=False): + if future: + res = value_ops.first_step_future_discounted_returns( rewards=rewards, values=values, step_types=step_types, discounts=discounts, - time_major=False), expected) + time_major=False) + else: + res = value_ops.discounted_return( + rewards=rewards, + values=values, + step_types=step_types, + discounts=discounts, + time_major=False) - np.testing.assert_array_almost_equal( - value_ops.discounted_return( + np.testing.assert_array_almost_equal(res, expected) + + if future: + res = value_ops.first_step_future_discounted_returns( rewards=torch.stack([rewards, 2 * rewards], dim=2), values=torch.stack([values, 2 * values], dim=2), step_types=step_types, discounts=discounts, - time_major=False), torch.stack([expected, 2 * expected], - dim=2)) + time_major=False) + else: + res = value_ops.discounted_return( + rewards=torch.stack([rewards, 2 * rewards], dim=2), + values=torch.stack([values, 2 * values], dim=2), + step_types=step_types, + discounts=discounts, + time_major=False) + np.testing.assert_array_almost_equal( + res, torch.stack([expected, 2 * expected], dim=2)) def test_discounted_return(self): values = torch.tensor([[1.] * 5], dtype=torch.float32) @@ -74,7 +97,7 @@ def test_discounted_return(self): discounts=discounts, expected=expected) - # tow episodes, and end normal (discount=0) + # two episodes, and end normal (discount=0) step_types = torch.tensor([[ StepType.MID, StepType.MID, StepType.LAST, StepType.MID, StepType.MID @@ -91,6 +114,100 @@ def test_discounted_return(self): discounts=discounts, expected=expected) + def test_first_step_future_discounted_returns(self): + values = torch.tensor([[1.] * 5], dtype=torch.float32) + step_types = torch.tensor([[StepType.MID] * 5], dtype=torch.int64) + rewards = torch.tensor([[2.] * 5], dtype=torch.float32) + discounts = torch.tensor([[0.9] * 5], dtype=torch.float32) + expected = torch.tensor([[ + 2 + 0.9, 2 + 0.9 * (2 + 0.9), 2 + 0.9 * (2 + 0.9 * (2 + 0.9)), + 2 + 0.9 * (2 + 0.9 * (2 + 0.9 * (2 + 0.9))) + ]], + dtype=torch.float32) + self._check( + rewards=rewards, + values=values, + step_types=step_types, + discounts=discounts, + expected=expected, + future=True) + + # two episodes, and exceed by time limit (discount=1) + step_types = torch.tensor([[ + StepType.MID, StepType.MID, StepType.LAST, StepType.MID, + StepType.MID + ]], + dtype=torch.int32) + expected = torch.tensor([[ + 2 + 0.9, 2 + 0.9 * (2 + 0.9), 2 + 0.9 * (2 + 0.9), + 2 + 0.9 * (2 + 0.9) + ]], + dtype=torch.float32) + self._check( + rewards=rewards, + values=values, + step_types=step_types, + discounts=discounts, + expected=expected, + future=True) + + # two episodes, and end normal (discount=0) + step_types = torch.tensor([[ + StepType.MID, StepType.MID, StepType.LAST, StepType.MID, + StepType.MID + ]], + dtype=torch.int32) + discounts = torch.tensor([[0.9, 0.9, 0.0, 0.9, 0.9]]) + expected = torch.tensor( + [[2 + 0.9, 2 + 0.9 * 2, 2 + 0.9 * 2, 2 + 0.9 * 2]], + dtype=torch.float32) + + self._check( + rewards=rewards, + values=values, + step_types=step_types, + discounts=discounts, + expected=expected, + future=True) + + # two episodes with discount 0 LAST. + values = torch.tensor([[1.] * 5], dtype=torch.float32) + step_types = torch.tensor([[ + StepType.MID, StepType.LAST, StepType.LAST, StepType.MID, + StepType.MID + ]], + dtype=torch.int32) + rewards = torch.tensor([[2.] * 5], dtype=torch.float32) + discounts = torch.tensor([[0.9, 0.0, 0.0, 0.9, 0.9]]) + expected = torch.tensor([[2, 2, 2, 2]], dtype=torch.float32) + + self._check( + rewards=rewards, + values=values, + step_types=step_types, + discounts=discounts, + expected=expected, + future=True) + + # two episodes with discount 0 LAST. + values = torch.tensor([[1.] * 5], dtype=torch.float32) + step_types = torch.tensor([[ + StepType.LAST, StepType.LAST, StepType.LAST, StepType.MID, + StepType.MID + ]], + dtype=torch.int32) + rewards = torch.tensor([[2.] * 5], dtype=torch.float32) + discounts = torch.tensor([[0.0, 0.0, 0.0, 0.9, 0.9]]) + expected = torch.tensor([[0, 0, 0, 0]], dtype=torch.float32) + + self._check( + rewards=rewards, + values=values, + step_types=step_types, + discounts=discounts, + expected=expected, + future=True) + class GeneralizedAdvantageTest(unittest.TestCase): """Tests for alf.utils.value_ops.generalized_advantage_estimation From 87b86c1a819136137ddd13bf945af8e2e9dad23b Mon Sep 17 00:00:00 2001 From: Le Horizon Date: Wed, 4 May 2022 13:38:48 -0700 Subject: [PATCH 2/7] fix --- alf/algorithms/data_transformer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alf/algorithms/data_transformer.py b/alf/algorithms/data_transformer.py index cf2082f06..eb156d8a2 100644 --- a/alf/algorithms/data_transformer.py +++ b/alf/algorithms/data_transformer.py @@ -984,9 +984,9 @@ def _add_noise(t): result, f, lambda t: convert_device(t)) info = convert_device(info) info = info._replace( - her=info.her.unsqueeze(1).expand(exp.reward.shape[:2]), + her=info.her.unsqueeze(1).expand(result.reward.shape[:2]), future_distance=info.future_distance.unsqueeze(1).expand( - exp.reward.shape[:2]), + result.reward.shape[:2]), replay_buffer=buffer) result = alf.data_structures.add_batch_info(result, info) return result From dc76b010706f32b2b4a3a9f2f4af5205e33c93eb Mon Sep 17 00:00:00 2001 From: Le Horizon Date: Mon, 9 May 2022 22:19:53 -0700 Subject: [PATCH 3/7] address comments --- alf/algorithms/data_transformer.py | 49 ++- alf/algorithms/ddpg_algorithm.py | 8 +- alf/algorithms/one_step_loss.py | 4 +- alf/algorithms/sac_algorithm.py | 6 + alf/algorithms/td_loss.py | 316 +++++++++++------- alf/algorithms/td_loss_test.py | 9 +- alf/bin/train_play_test.py | 3 +- alf/environments/suite_robotics.py | 29 +- alf/examples/her_fetchpush_conf.py | 7 +- alf/examples/her_target_navigation_states.gin | 1 + alf/experience_replayers/replay_buffer.py | 3 + 11 files changed, 268 insertions(+), 167 deletions(-) diff --git a/alf/algorithms/data_transformer.py b/alf/algorithms/data_transformer.py index eb156d8a2..c05548108 100644 --- a/alf/algorithms/data_transformer.py +++ b/alf/algorithms/data_transformer.py @@ -139,13 +139,18 @@ def __init__(self, data_transformer_ctors, observation_spec): @staticmethod def _validate_order(data_transformers): + # Hindsight should probably not be used together with FrameStacker, + # unless done really carefully. Hindsight after FrameStacker is + # simply wrong, because Hindsight would read ``achieved_goal`` field + # of a future step directly from the replay buffer without stacking. def _tier_of(data_transformer): if isinstance(data_transformer, UntransformedTimeStep): return 1 - if isinstance(data_transformer, - (HindsightExperienceTransformer, FrameStacker)): + if isinstance(data_transformer, HindsightExperienceTransformer): return 2 - return 3 + if isinstance(data_transformer, FrameStacker): + return 3 + return 4 prev_tier = 0 for i in range(len(data_transformers)): @@ -722,14 +727,31 @@ class HindsightExperienceTransformer(DataTransformer): of the current timestep. The exact field names can be provided via arguments to the class ``__init__``. + NOTE: When the experience reward is multi-dimensional, the 0th dimension is assumed + to be the goal reward dimension, and is relabed by the HindsightExperienceTransformer. + All other reward dimensions are untouched. + TODO: Change the reward field into a nested field to be able to support multi-dimensional + goal rewards. + + NOTE: The HindsightExperienceTransformer has to happen before any transformer which changes + reward or achieved_goal fields, e.g. observation normalizer, reward clipper, etc.. + See `documentation <../../docs/notes/knowledge_base.rst#datatransformers>`_ for details. + To use this class, add it to any existing data transformers, e.g. use this config if ``ObservationNormalizer`` is an existing data transformer: .. code-block:: python - ReplayBuffer.keep_episodic_info=True - HindsightExperienceTransformer.her_proportion=0.8 - TrainerConfig.data_transformer_ctor=[@HindsightExperienceTransformer, @ObservationNormalizer] + alf.config('ReplayBuffer', keep_episodic_info=True) + alf.config( + 'HindsightExperienceTransformer', + her_proportion=0.8 + ) + alf.config( + 'TrainerConfig', + data_transformer_ctor=[ + HindsightExperienceTransformer, ObservationNormalizer + ]) See unit test for more details on behavior. """ @@ -742,7 +764,8 @@ def __init__(self, sparse_reward=False, add_noise_to_goals=False, threshold=.05, - reward_fn=l2_dist_close_reward_fn): + reward_fn=l2_dist_close_reward_fn, + sparse_reward_transform=alf.utils.math_ops.identity): """ Args: her_proportion (float): proportion of hindsight relabeled experience. @@ -751,12 +774,16 @@ def __init__(self, desired_goal_field (str): path to the desired_goal field in the exp nest. sparse_reward (bool): Whether to transform reward from -1/0 to 0/1. + This also makes the task episodic by relabeling rewarding step as LAST + and setting discount to 0. add_noise_to_goals (bool): Whether to add noise around relabeled goal. threshold (float): noise added to relabeled goals. reward_fn (Callable): function to recompute reward based on achieve_goal and desired_goal. Default gives reward 0 when L2 distance less than 0.05 and -1 otherwise, same as is done in suite_robotics environments. + sparse_reward_transform (Callable): transforms reward from -1/0 to 0/1. + Only used when sparse_reward is True. """ super().__init__( transformed_observation_spec=transformed_observation_spec, @@ -876,8 +903,9 @@ def _add_noise(t): ".discount_mean_before_relabel", torch.mean(result.discount[:, 1:])) if self._sparse_reward: + # Assumes that original reward is -1/0 and 0 when goal is reached. reward_achieved = relabeled_rewards >= 0 - # Cut off episode for any goal reached. + # Cut off episode for any goal reached, making the task episodic. end = reward_achieved discount = torch.where(end, torch.tensor(0.), result.discount) step_type = torch.where(end, torch.tensor(StepType.LAST), @@ -904,7 +932,7 @@ def _add_noise(t): result = result._replace(discount=discount) result = result._replace(step_type=step_type) - relabeled_rewards = suite_socialbot.transform_reward_tensor( + relabeled_rewards = self._sparse_reward_transform( relabeled_rewards) non_her_or_fst = ~her_cond.unsqueeze(1) & (result.step_type != @@ -1022,7 +1050,4 @@ def create_data_transformer(data_transformer_ctor, observation_spec): if len(data_transformer_ctor) == 1: return data_transformer_ctor[0](observation_spec) - if HindsightExperienceTransformer in data_transformer_ctor: - assert HindsightExperienceTransformer == data_transformer_ctor[0], \ - "Hindsight relabeling should happen before all other transforms." return SequentialDataTransformer(data_transformer_ctor, observation_spec) diff --git a/alf/algorithms/ddpg_algorithm.py b/alf/algorithms/ddpg_algorithm.py index 25c3a73f2..9bd67dbb0 100644 --- a/alf/algorithms/ddpg_algorithm.py +++ b/alf/algorithms/ddpg_algorithm.py @@ -42,7 +42,7 @@ DdpgInfo = namedtuple( "DdpgInfo", [ "reward", "step_type", "discount", "action", "action_distribution", - "actor_loss", "critic", "discounted_return" + "actor_loss", "critic", "discounted_return", "future_distance", "her" ], default_value=()) DdpgLossInfo = namedtuple('DdpgLossInfo', ('actor', 'critic')) @@ -358,6 +358,12 @@ def calc_loss(self, info: DdpgInfo): actor_loss = info.actor_loss + # The current implementation is hacky: Instead of using OneStepTD + # and pulling additionally a few timesteps from the future to compute + # bootstrap values, we here piggyback on n-step TDLoss, but masking + # out losses from the 2nd to n-1-th steps. + # If this hacky use pattern is to be used frequently in the future, + # we should consider refactoring it. if self._critic_losses[0]._improve_w_nstep_bootstrap: # Ignore 2nd - nth step actor losses. actor_loss.loss[1:] = 0 diff --git a/alf/algorithms/one_step_loss.py b/alf/algorithms/one_step_loss.py index 34ee329ad..e687f9dc5 100644 --- a/alf/algorithms/one_step_loss.py +++ b/alf/algorithms/one_step_loss.py @@ -16,12 +16,12 @@ from typing import Union, List, Callable import alf -from alf.algorithms.td_loss import TDLoss, TDQRLoss +from alf.algorithms.td_loss import LowerBoundedTDLoss, TDQRLoss from alf.utils import losses @alf.configurable -class OneStepTDLoss(TDLoss): +class OneStepTDLoss(LowerBoundedTDLoss): def __init__(self, gamma: Union[float, List[float]] = 0.99, td_error_loss_fn: Callable = losses.element_wise_squared_loss, diff --git a/alf/algorithms/sac_algorithm.py b/alf/algorithms/sac_algorithm.py index f8a798704..b5c6f8e3b 100644 --- a/alf/algorithms/sac_algorithm.py +++ b/alf/algorithms/sac_algorithm.py @@ -847,6 +847,12 @@ def calc_loss(self, info: SacInfo): alpha_loss = info.alpha actor_loss = info.actor + # The current implementation is hacky: Instead of using OneStepTD + # and pulling additionally a few timesteps from the future to compute + # bootstrap values, we here piggyback on n-step TDLoss, but masking + # out losses from the 2nd to n-1-th steps. + # If this hacky use pattern is to be used frequently in the future, + # we should consider refactoring it. if self._critic_losses[0]._improve_w_nstep_bootstrap: # Ignore 2nd - n-th step losses in this mode. alpha_loss[1:] = 0 diff --git a/alf/algorithms/td_loss.py b/alf/algorithms/td_loss.py index 7e602c7dd..9eb5c51ba 100644 --- a/alf/algorithms/td_loss.py +++ b/alf/algorithms/td_loss.py @@ -35,15 +35,6 @@ def __init__(self, td_lambda: float = 0.95, normalize_target: bool = False, debug_summaries: bool = False, - lb_target_q: float = 0., - default_return: float = -1000., - improve_w_goal_return: bool = False, - improve_w_nstep_bootstrap: bool = False, - improve_w_nstep_only: bool = False, - lower_bound_constraint: float = 0., - lb_loss_scale: bool = False, - reward_multiplier: float = 1., - positive_reward: bool = True, use_retrace: bool = False, name: str = "TDLoss"): r""" @@ -99,27 +90,6 @@ def __init__(self, use_retrace: turn on retrace loss :math:`\mathcal{R} Q(x, a):=Q(x, a)+\mathbb{E}_{\mu}\left[\sum_{t \geq 0} \gamma^{t}\left(\prod_{s=1}^{t} c_{s}\right)\left(r_{t}+\gamma \mathbb{E}_{\pi} Q\left(x_{t+1}, \cdot\right)-Q\left(x_{t}, a_{t}\right)\right)\right]` copied from PR #695. - lb_target_q: between 0 and 1. When not zero, use this mixing rate for the - lower bounded value target. Only supports batch_length == 2, one step td. - default_return: Keep it the same as replay_buffer.default_return to plot to - tensorboard episodic_discounted_return only for the timesteps whose - episode already ended. - improve_w_goal_return: Use return calculated from the distance to hindsight - goals. Only supports batch_length == 2, one step td. - improve_w_nstep_bootstrap: Look ahead 2 to n steps, and take the largest - bootstrapped return to lower bound the value target of the 1st step. - improve_w_nstep_only: Only use the n-th step bootstrapped return as - value target lower bound. - lower_bound_constraint: Use n-step bootstrapped return as lower bound - constraints of the value. See reference: - He, F. S., Liu, Y., Schwing, A. G., and Peng, J. - Learning to play in a day: Faster deep reinforcement learning - by optimality tightening. In 5th International Conference - on Learning Representations, ICLR 2017, Toulon, France, - April 24-26, 2017. https://openreview.net/forum?id=rJ8Je4clg - lb_loss_scale: Parameter for lower_bound_constraint. - reward_multiplier: Weight on the hindsight goal return. - positive_reward: If True, assumes 0/1 goal reward, otherwise, -1/0. debug_summaries: True if debug summaries should be created. name: The name of this loss. """ @@ -130,15 +100,6 @@ def __init__(self, self._td_error_loss_fn = td_error_loss_fn self._clip = clip self._lambda = td_lambda - self._lb_target_q = lb_target_q - self._default_return = default_return - self._improve_w_goal_return = improve_w_goal_return - self._improve_w_nstep_bootstrap = improve_w_nstep_bootstrap - self._improve_w_nstep_only = improve_w_nstep_only - self._lower_bound_constraint = lower_bound_constraint - self._lb_loss_scale = lb_loss_scale - self._reward_multiplier = reward_multiplier - self._positive_reward = positive_reward self._use_retrace = use_retrace self._debug_summaries = debug_summaries self._normalize_target = normalize_target @@ -236,6 +197,195 @@ def compute_td_target(self, returns = advantages + value[:-1] returns = returns.detach() + return returns, value, None + + def forward(self, info: namedtuple, value: torch.Tensor, + target_value: torch.Tensor): + """Calculate the loss. + + The first dimension of all the tensors is time dimension and the second + dimesion is the batch dimension. + + Args: + info: experience collected from ``unroll()`` or + a replay buffer. All tensors are time-major. ``info`` should + contain the following fields: + - reward: + - step_type: + - discount: + value: the time-major tensor for the value at each time + step. The loss is between this and the calculated return. + target_value: the time-major tensor for the value at + each time step. This is used to calculate return. ``target_value`` + can be same as ``value``. + Returns: + LossInfo: with the ``extra`` field same as ``loss``. + """ + returns, value, constraint_loss = self.compute_td_target( + info, value, target_value) + value = value[:-1] + + if self._normalize_target: + if self._target_normalizer is None: + self._target_normalizer = AdaptiveNormalizer( + alf.TensorSpec(value.shape[2:]), + auto_update=False, + debug_summaries=self._debug_summaries, + name=self._name + ".target_normalizer") + + self._target_normalizer.update(returns) + returns = self._target_normalizer.normalize(returns) + value = self._target_normalizer.normalize(value) + + if self._debug_summaries and alf.summary.should_record_summaries(): + mask = info.step_type[:-1] != StepType.LAST + with alf.summary.scope(self._name): + + def _summarize(v, r, td, suffix): + alf.summary.scalar( + "explained_variance_of_return_by_value" + suffix, + tensor_utils.explained_variance(v, r, mask)) + safe_mean_hist_summary('values' + suffix, v, mask) + safe_mean_hist_summary('returns' + suffix, r, mask) + safe_mean_hist_summary("td_error" + suffix, td, mask) + + if value.ndim == 2: + _summarize(value, returns, returns - value, '') + else: + td = returns - value + for i in range(value.shape[2]): + suffix = '/' + str(i) + _summarize(value[..., i], returns[..., i], td[..., i], + suffix) + + loss = self._td_error_loss_fn(returns.detach(), value) + if self._clip > 0: + loss = torch.clamp(loss, min=-self._clip, max=self._clip) + + if loss.ndim == 3: + # Multidimensional reward. Average over the critic loss for all dimensions + loss = loss.mean(dim=2) + + if self._improve_w_nstep_bootstrap: + # Ignore 2nd to n-th step losses. + loss[1:] = 0 + if self._lower_bound_constraint > 0: + assert constraint_loss.shape == loss.shape[1:], \ + f"{constraint_loss.shape} != {loss.shape}[1:]" + c_loss = constraint_loss.clone().unsqueeze(0).repeat( + (loss.shape[0], 1)) + c_loss[1:] = 0 + if self._lb_loss_scale: + scale = ( + torch.sum(loss) / torch.sum(c_loss + loss)).detach() + else: + scale = 1 + loss = (c_loss + loss) * scale + + # The shape of the loss expected by Algorith.update_with_gradient is + # [T, B], so we need to augment it with additional zeros. + loss = tensor_utils.tensor_extend_zero(loss) + return LossInfo(loss=loss, extra=loss) + + +@alf.configurable +class LowerBoundedTDLoss(TDLoss): + """Temporal difference loss with value target lower bounding.""" + + def __init__(self, + gamma: Union[float, List[float]] = 0.99, + td_error_loss_fn: Callable = element_wise_squared_loss, + clip: float = 0., + td_lambda: float = 0.95, + normalize_target: bool = False, + use_retrace: bool = False, + lb_target_q: float = 0., + default_return: float = -1000., + improve_w_goal_return: bool = False, + improve_w_nstep_bootstrap: bool = False, + improve_w_nstep_only: bool = False, + lower_bound_constraint: float = 0., + lb_loss_scale: bool = False, + reward_multiplier: float = 1., + positive_reward: bool = True, + debug_summaries: bool = False, + name: str = "LbTDLoss"): + r""" + Args: + gamma .. use_retrace: pass through to TDLoss. + lb_target_q: between 0 and 1. When not zero, use this mixing rate for the + lower bounded value target. Only supports batch_length == 2, one step td. + default_return: Keep it the same as replay_buffer.default_return to plot to + tensorboard episodic_discounted_return only for the timesteps whose + episode already ended. + improve_w_goal_return: Use return calculated from the distance to hindsight + goals. Only supports batch_length == 2, one step td. + improve_w_nstep_bootstrap: Look ahead 2 to n steps, and take the largest + bootstrapped return to lower bound the value target of the 1st step. + improve_w_nstep_only: Only use the n-th step bootstrapped return as + value target lower bound. + lower_bound_constraint: Use n-step bootstrapped return as lower bound + constraints of the value. See reference: + He, F. S., Liu, Y., Schwing, A. G., and Peng, J. + Learning to play in a day: Faster deep reinforcement learning + by optimality tightening. In 5th International Conference + on Learning Representations, ICLR 2017, Toulon, France, + April 24-26, 2017. https://openreview.net/forum?id=rJ8Je4clg + lb_loss_scale: Parameter for lower_bound_constraint. + reward_multiplier: Weight on the hindsight goal return. + positive_reward: If True, assumes 0/1 goal reward, otherwise, -1/0. + debug_summaries: True if debug summaries should be created. + name: The name of this loss. + """ + super().__init__( + gamma=gamma, + td_error_loss_fn=td_error_loss_fn, + clip=clip, + td_lambda=td_lambda, + normalize_target=normalize_target, + use_retrace=use_retrace, + name=name, + debug_summaries=debug_summaries) + + self._lb_target_q = lb_target_q + self._default_return = default_return + self._improve_w_goal_return = improve_w_goal_return + self._improve_w_nstep_bootstrap = improve_w_nstep_bootstrap + self._improve_w_nstep_only = improve_w_nstep_only + self._lower_bound_constraint = lower_bound_constraint + self._lb_loss_scale = lb_loss_scale + self._reward_multiplier = reward_multiplier + self._positive_reward = positive_reward + + def compute_td_target(self, + info: namedtuple, + value: torch.Tensor, + target_value: torch.Tensor, + qr: bool = False): + """Calculate the td target. + + The first dimension of all the tensors is time dimension and the second + dimesion is the batch dimension. + + Args: + info (namedtuple): experience collected from ``unroll()`` or + a replay buffer. All tensors are time-major. ``info`` should + contain the following fields: + - reward: + - step_type: + - discount: + value (torch.Tensor): the time-major tensor for the value at + each time step. Some of its value can be overwritten and passed + back to the caller. + target_value (torch.Tensor): the time-major tensor for the value at + each time step. This is used to calculate return. ``target_value`` + can be same as ``value``, except for Retrace. + Returns: + td_target, updated value, optional constraint_loss + """ + returns, value, _ = super().compute_td_target(info, value, + target_value, qr) + constraint_loss = None if self._improve_w_nstep_bootstrap: assert self._lambda == 1.0, "td lambda does not work with this" @@ -347,94 +497,6 @@ def compute_td_target(self, return returns, value, constraint_loss - def forward(self, info: namedtuple, value: torch.Tensor, - target_value: torch.Tensor): - """Calculate the loss. - - The first dimension of all the tensors is time dimension and the second - dimesion is the batch dimension. - - Args: - info: experience collected from ``unroll()`` or - a replay buffer. All tensors are time-major. ``info`` should - contain the following fields: - - reward: - - step_type: - - discount: - value: the time-major tensor for the value at each time - step. The loss is between this and the calculated return. - target_value: the time-major tensor for the value at - each time step. This is used to calculate return. ``target_value`` - can be same as ``value``. - Returns: - LossInfo: with the ``extra`` field same as ``loss``. - """ - returns, value, constraint_loss = self.compute_td_target( - info, value, target_value) - value = value[:-1] - - if self._normalize_target: - if self._target_normalizer is None: - self._target_normalizer = AdaptiveNormalizer( - alf.TensorSpec(value.shape[2:]), - auto_update=False, - debug_summaries=self._debug_summaries, - name=self._name + ".target_normalizer") - - self._target_normalizer.update(returns) - returns = self._target_normalizer.normalize(returns) - value = self._target_normalizer.normalize(value) - - if self._debug_summaries and alf.summary.should_record_summaries(): - mask = info.step_type[:-1] != StepType.LAST - with alf.summary.scope(self._name): - - def _summarize(v, r, td, suffix): - alf.summary.scalar( - "explained_variance_of_return_by_value" + suffix, - tensor_utils.explained_variance(v, r, mask)) - safe_mean_hist_summary('values' + suffix, v, mask) - safe_mean_hist_summary('returns' + suffix, r, mask) - safe_mean_hist_summary("td_error" + suffix, td, mask) - - if value.ndim == 2: - _summarize(value, returns, returns - value, '') - else: - td = returns - value - for i in range(value.shape[2]): - suffix = '/' + str(i) - _summarize(value[..., i], returns[..., i], td[..., i], - suffix) - - loss = self._td_error_loss_fn(returns.detach(), value) - if self._clip > 0: - loss = torch.clamp(loss, min=-self._clip, max=self._clip) - - if loss.ndim == 3: - # Multidimensional reward. Average over the critic loss for all dimensions - loss = loss.mean(dim=2) - - if self._improve_w_nstep_bootstrap: - # Ignore 2nd to n-th step losses. - loss[1:] = 0 - if self._lower_bound_constraint > 0: - assert constraint_loss.shape == loss.shape[1:], \ - f"{constraint_loss.shape} != {loss.shape}[1:]" - c_loss = constraint_loss.clone().unsqueeze(0).repeat( - (loss.shape[0], 1)) - c_loss[1:] = 0 - if self._lb_loss_scale: - scale = ( - torch.sum(loss) / torch.sum(c_loss + loss)).detach() - else: - scale = 1 - loss = (c_loss + loss) * scale - - # The shape of the loss expected by Algorith.update_with_gradient is - # [T, B], so we need to augment it with additional zeros. - loss = tensor_utils.tensor_extend_zero(loss) - return LossInfo(loss=loss, extra=loss) - @alf.configurable class TDQRLoss(TDLoss): diff --git a/alf/algorithms/td_loss_test.py b/alf/algorithms/td_loss_test.py index ac7c9ef0a..2458fb89e 100644 --- a/alf/algorithms/td_loss_test.py +++ b/alf/algorithms/td_loss_test.py @@ -18,22 +18,23 @@ import torch import alf -from alf.algorithms.td_loss import TDLoss +from alf.algorithms.td_loss import LowerBoundedTDLoss from alf.data_structures import TimeStep, StepType, namedtuple DataItem = namedtuple( "DataItem", ["reward", "step_type", "discount"], default_value=()) -class TDLossTest(unittest.TestCase): - """Tests for alf.algorithms.td_loss.TDLoss +class LowerBoundedTDLossTest(unittest.TestCase): + """Tests for alf.algorithms.td_loss.LowerBoundedTDLoss """ def _check(self, res, expected): np.testing.assert_array_almost_equal(res, expected) def test_compute_td_target_nstep_bootstrap_lowerbound(self): - loss = TDLoss(gamma=1., improve_w_nstep_bootstrap=True, td_lambda=1) + loss = LowerBoundedTDLoss( + gamma=1., improve_w_nstep_bootstrap=True, td_lambda=1) # Tensors are transposed to be time_major [T, B, ...] step_types = torch.tensor([[StepType.MID] * 5], dtype=torch.int64).transpose(0, 1) diff --git a/alf/bin/train_play_test.py b/alf/bin/train_play_test.py index 400640d20..4826bef84 100644 --- a/alf/bin/train_play_test.py +++ b/alf/bin/train_play_test.py @@ -142,7 +142,8 @@ def _to_conf_params(parameters): 'TrainerConfig.mini_batch_length=2', 'TrainerConfig.mini_batch_size=4', # If replay_buffer_length is above 6 (3 iters * 2 unroll length), whole replay buffer training - # algorithms (e.g. mbrl_pendulum) will not get enough training data, and will not start training. + # algorithms (e.g. mbrl_pendulum) will not get enough training data, because the replay buffer + # gets cleared before the buffer is fully filled. Training never starts. # This fails during playing, when the config file isn't generated in root_dir. 'TrainerConfig.replay_buffer_length=4', 'FrameStacker.stack_size=1' diff --git a/alf/environments/suite_robotics.py b/alf/environments/suite_robotics.py index 780ddc864..14fe6971f 100644 --- a/alf/environments/suite_robotics.py +++ b/alf/environments/suite_robotics.py @@ -48,15 +48,16 @@ class SparseReward(gym.Wrapper): """Convert the original :math:`-1/0` rewards to :math:`0/1`. """ - def __init__(self, - env, - reward_cap=1., - positive_reward=True, - append_reward_dim=False): + def __init__(self, env, reward_weight=1., positive_reward=True): + """ + Args: + reward_weight (float): weight of output reward. + positive_reward (bool): if True, returns 0/1 reward, + otherwise, -1/0 reward. + """ gym.Wrapper.__init__(self, env) - self._reward_cap = reward_cap + self._reward_weight = reward_weight self._positive_reward = positive_reward - self._append_reward_dim = append_reward_dim def step(self, action): # openai Robotics env will always return ``done=False`` @@ -67,19 +68,9 @@ def step(self, action): return_reward = reward + 1 else: return_reward = reward - return_reward *= self._reward_cap - if self._append_reward_dim: - return_reward = np.array([return_reward, 0]) + return_reward *= self._reward_weight return ob, return_reward, done, info - @property - def reward_space(self): - if self._append_reward_dim: - return gym.spaces.Box( - low=-np.inf, high=np.inf, shape=(2, ), dtype=np.float32) - else: - return self.env.reward_space() - @alf.configurable class SuccessWrapper(gym.Wrapper): @@ -192,6 +183,8 @@ def load(environment_name, Args: environment_name: Name for the environment to load. env_id: A scalar ``Tensor`` of the environment ID of the time step. + concat_desired_goal (bool): Whether to concat robot's observation and the goal + location. discount: Discount to use for the environment. max_episode_steps: If None the ``max_episode_steps`` will be set to the default step limit defined in the environment's spec. No limit is applied if set diff --git a/alf/examples/her_fetchpush_conf.py b/alf/examples/her_fetchpush_conf.py index e4a15ade0..87a2f6d00 100644 --- a/alf/examples/her_fetchpush_conf.py +++ b/alf/examples/her_fetchpush_conf.py @@ -16,7 +16,7 @@ from alf.algorithms.data_transformer import HindsightExperienceTransformer, \ ObservationNormalizer from alf.algorithms.ddpg_algorithm import DdpgAlgorithm -from alf.environments import suite_robotics +from alf.environments import suite_robotics, suite_socialbot from alf.nest.utils import NestConcat from alf.examples import ddpg_fetchpush_conf @@ -29,7 +29,10 @@ action_preprocessing_combiner=NestConcat()) alf.config('ReplayBuffer', keep_episodic_info=True) -alf.config('HindsightExperienceTransformer', her_proportion=0.8) +alf.config( + 'HindsightExperienceTransformer', + her_proportion=0.8, + sparse_reward_transform=suite_socialbot.transform_reward_tensor) alf.config( 'TrainerConfig', data_transformer_ctor=[ diff --git a/alf/examples/her_target_navigation_states.gin b/alf/examples/her_target_navigation_states.gin index 469f19942..a70217e22 100644 --- a/alf/examples/her_target_navigation_states.gin +++ b/alf/examples/her_target_navigation_states.gin @@ -91,6 +91,7 @@ TrainerConfig.num_eval_episodes=50 ReplayBuffer.keep_episodic_info=True HindsightExperienceTransformer.her_proportion=0.8 HindsightExperienceTransformer.threshold=0.5 +HindsightExperienceTransformer.sparse_reward_transform=suite_socialbot.transform_reward_tensor TrainerConfig.data_transformer_ctor=@HindsightExperienceTransformer # Finer grain tensorboard summaries plus local action distribution diff --git a/alf/experience_replayers/replay_buffer.py b/alf/experience_replayers/replay_buffer.py index 83eceb153..af2da86f3 100644 --- a/alf/experience_replayers/replay_buffer.py +++ b/alf/experience_replayers/replay_buffer.py @@ -127,6 +127,9 @@ def __init__(self, Usually consistent with ``TDLoss.gamma``. reward_clip (tuple|None): None or (min, max) for reward clipping. enable_checkpoint (bool): whether checkpointing this replay buffer. + convert_only_minibatch_to_device (bool): when True, only convert a minibatch + of experience to GPU (if GPU is used), to save GPU memory. Fractional unroll + is also able to save GPU memory, similarly. name (string): name of the replay buffer object. """ super().__init__( From 639ab020d08a4c4802bd321b75ded3f55224bf3f Mon Sep 17 00:00:00 2001 From: Le Horizon Date: Tue, 10 May 2022 09:14:18 -0700 Subject: [PATCH 4/7] fix tests: ddpg or sac algorithm should not require loss to contain field "improve_w_nstep_bootstrap". --- alf/algorithms/ddpg_algorithm.py | 4 +- alf/algorithms/sac_algorithm.py | 8 ++- alf/algorithms/td_loss.py | 57 ++++++++++++++----- alf/examples/her_target_navigation_states.gin | 2 +- 4 files changed, 52 insertions(+), 19 deletions(-) diff --git a/alf/algorithms/ddpg_algorithm.py b/alf/algorithms/ddpg_algorithm.py index 9bd67dbb0..7587f5114 100644 --- a/alf/algorithms/ddpg_algorithm.py +++ b/alf/algorithms/ddpg_algorithm.py @@ -364,7 +364,9 @@ def calc_loss(self, info: DdpgInfo): # out losses from the 2nd to n-1-th steps. # If this hacky use pattern is to be used frequently in the future, # we should consider refactoring it. - if self._critic_losses[0]._improve_w_nstep_bootstrap: + if hasattr(self._critic_losses[0], + "_improve_w_nstep_bootstrap") and \ + self._critic_losses[0]._improve_w_nstep_bootstrap: # Ignore 2nd - nth step actor losses. actor_loss.loss[1:] = 0 actor_loss.extra[1:] = 0 diff --git a/alf/algorithms/sac_algorithm.py b/alf/algorithms/sac_algorithm.py index b5c6f8e3b..2666aaccb 100644 --- a/alf/algorithms/sac_algorithm.py +++ b/alf/algorithms/sac_algorithm.py @@ -853,7 +853,9 @@ def calc_loss(self, info: SacInfo): # out losses from the 2nd to n-1-th steps. # If this hacky use pattern is to be used frequently in the future, # we should consider refactoring it. - if self._critic_losses[0]._improve_w_nstep_bootstrap: + if hasattr(self._critic_losses[0], + "_improve_w_nstep_bootstrap") and \ + self._critic_losses[0]._improve_w_nstep_bootstrap: # Ignore 2nd - n-th step losses in this mode. alpha_loss[1:] = 0 if actor_loss.loss != (): @@ -912,7 +914,9 @@ def _calc_critic_loss(self, info: SacInfo): if self._use_entropy_reward: with torch.no_grad(): log_pi = info.log_pi - if self._critic_losses[0]._improve_w_nstep_bootstrap: + if hasattr(self._critic_losses[0], + "_improve_w_nstep_bootstrap") and \ + self._critic_losses[0]._improve_w_nstep_bootstrap: # Ignore 2nd - n-th step entropy in this mode. log_pi[1:] = 0 if self._entropy_normalizer is not None: diff --git a/alf/algorithms/td_loss.py b/alf/algorithms/td_loss.py index 9eb5c51ba..90b220475 100644 --- a/alf/algorithms/td_loss.py +++ b/alf/algorithms/td_loss.py @@ -266,21 +266,18 @@ def _summarize(v, r, td, suffix): # Multidimensional reward. Average over the critic loss for all dimensions loss = loss.mean(dim=2) - if self._improve_w_nstep_bootstrap: - # Ignore 2nd to n-th step losses. - loss[1:] = 0 - if self._lower_bound_constraint > 0: - assert constraint_loss.shape == loss.shape[1:], \ - f"{constraint_loss.shape} != {loss.shape}[1:]" - c_loss = constraint_loss.clone().unsqueeze(0).repeat( - (loss.shape[0], 1)) - c_loss[1:] = 0 - if self._lb_loss_scale: - scale = ( - torch.sum(loss) / torch.sum(c_loss + loss)).detach() - else: - scale = 1 - loss = (c_loss + loss) * scale + # For the subclass LowerBoundedTDLoss + if constraint_loss is not None: + assert constraint_loss.shape == loss.shape[1:], \ + f"{constraint_loss.shape} != {loss.shape}[1:]" + c_loss = constraint_loss.clone().unsqueeze(0).repeat( + (loss.shape[0], 1)) + c_loss[1:] = 0 + if self._lb_loss_scale: + scale = (torch.sum(loss) / torch.sum(c_loss + loss)).detach() + else: + scale = 1 + loss = (c_loss + loss) * scale # The shape of the loss expected by Algorith.update_with_gradient is # [T, B], so we need to augment it with additional zeros. @@ -497,6 +494,36 @@ def compute_td_target(self, return returns, value, constraint_loss + def forward(self, info: namedtuple, value: torch.Tensor, + target_value: torch.Tensor): + """Calculate the loss. + + The first dimension of all the tensors is time dimension and the second + dimesion is the batch dimension. + + Args: + info: experience collected from ``unroll()`` or + a replay buffer. All tensors are time-major. ``info`` should + contain the following fields: + - reward: + - step_type: + - discount: + value: the time-major tensor for the value at each time + step. The loss is between this and the calculated return. + target_value: the time-major tensor for the value at + each time step. This is used to calculate return. ``target_value`` + can be same as ``value``. + Returns: + LossInfo: with the ``extra`` field same as ``loss``. + """ + loss_info = super().forward(info, value, target_value) + loss = loss_info.loss + if self._improve_w_nstep_bootstrap: + # Ignore 2nd to n-th step losses. + loss[1:] = 0 + + return LossInfo(loss=loss, extra=loss) + @alf.configurable class TDQRLoss(TDLoss): diff --git a/alf/examples/her_target_navigation_states.gin b/alf/examples/her_target_navigation_states.gin index a70217e22..467a05a19 100644 --- a/alf/examples/her_target_navigation_states.gin +++ b/alf/examples/her_target_navigation_states.gin @@ -91,7 +91,7 @@ TrainerConfig.num_eval_episodes=50 ReplayBuffer.keep_episodic_info=True HindsightExperienceTransformer.her_proportion=0.8 HindsightExperienceTransformer.threshold=0.5 -HindsightExperienceTransformer.sparse_reward_transform=suite_socialbot.transform_reward_tensor +HindsightExperienceTransformer.sparse_reward_transform=@suite_socialbot.transform_reward_tensor TrainerConfig.data_transformer_ctor=@HindsightExperienceTransformer # Finer grain tensorboard summaries plus local action distribution From 6188278fbbe0e1ccfe6425f3bcac1376e8809de1 Mon Sep 17 00:00:00 2001 From: Le Horizon Date: Tue, 10 May 2022 11:08:28 -0700 Subject: [PATCH 5/7] add comment --- alf/algorithms/data_transformer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/alf/algorithms/data_transformer.py b/alf/algorithms/data_transformer.py index c05548108..c1cac1a09 100644 --- a/alf/algorithms/data_transformer.py +++ b/alf/algorithms/data_transformer.py @@ -195,6 +195,8 @@ def __init__(self, observation_spec (nested TensorSpec): describing the observation in timestep stack_size (int): stack so many frames stack_axis (int): the dimension to stack the observation. + convert_only_minibatch_to_device (bool): whether to convert only the + minibatch or the whole batch of data to the default device. fields (list[str]): fields to be stacked, A field str is a multi-level path denoted by "A.B.C". If None, then non-nested observation is stacked. """ From ed38b926bf9b4b47d623d036c7e1c38358a5713d Mon Sep 17 00:00:00 2001 From: Le Horizon Date: Fri, 13 May 2022 22:10:11 -0700 Subject: [PATCH 6/7] address comments, separate loss class, separate dqn_algorithm, confs and plots --- alf/algorithms/data_transformer.py | 237 ++++++++---------- alf/algorithms/ddpg_algorithm.py | 30 ++- alf/algorithms/dqn_algorithm.py | 128 ++++++++++ alf/algorithms/rl_algorithm.py | 6 +- alf/algorithms/sac_algorithm.py | 113 +++------ alf/algorithms/td_loss.py | 10 +- alf/bin/train_play_test.py | 6 +- alf/environments/gym_wrappers.py | 10 + alf/environments/suite_robotics.py | 21 +- alf/examples/dqn_breakout_conf-lbtq-Qbert.png | Bin 0 -> 71578 bytes alf/examples/dqn_breakout_conf.py | 36 +++ alf/examples/her_fetchpush_conf.py | 2 +- alf/examples/her_target_navigation_states.gin | 2 +- ...t.png => sac_breakout_conf-lbtq-Qbert.png} | Bin alf/examples/sac_breakout_conf.py | 6 +- alf/utils/value_ops.py | 159 ++++++------ 16 files changed, 437 insertions(+), 329 deletions(-) create mode 100644 alf/algorithms/dqn_algorithm.py create mode 100644 alf/examples/dqn_breakout_conf-lbtq-Qbert.png create mode 100644 alf/examples/dqn_breakout_conf.py rename alf/examples/{sacbreakout-lbtq-Qbert.png => sac_breakout_conf-lbtq-Qbert.png} (100%) diff --git a/alf/algorithms/data_transformer.py b/alf/algorithms/data_transformer.py index c1cac1a09..cc5bfbb1b 100644 --- a/alf/algorithms/data_transformer.py +++ b/alf/algorithms/data_transformer.py @@ -763,11 +763,10 @@ def __init__(self, her_proportion=0.8, achieved_goal_field="time_step.observation.achieved_goal", desired_goal_field="time_step.observation.desired_goal", - sparse_reward=False, - add_noise_to_goals=False, - threshold=.05, + relabel_with_episodic_rewards=False, + relabeled_goal_noise=0, reward_fn=l2_dist_close_reward_fn, - sparse_reward_transform=alf.utils.math_ops.identity): + episodic_reward_transform=alf.utils.math_ops.identity): """ Args: her_proportion (float): proportion of hindsight relabeled experience. @@ -775,16 +774,15 @@ def __init__(self, exp nest. desired_goal_field (str): path to the desired_goal field in the exp nest. - sparse_reward (bool): Whether to transform reward from -1/0 to 0/1. + relabel_with_episodic_rewards (bool): Whether to transform reward from -1/0 to 0/1. This also makes the task episodic by relabeling rewarding step as LAST and setting discount to 0. - add_noise_to_goals (bool): Whether to add noise around relabeled goal. - threshold (float): noise added to relabeled goals. + relabeled_goal_noise (float): if positive, the noise added to relabeled goals. reward_fn (Callable): function to recompute reward based on achieve_goal and desired_goal. Default gives reward 0 when L2 distance less than 0.05 and -1 otherwise, same as is done in suite_robotics environments. - sparse_reward_transform (Callable): transforms reward from -1/0 to 0/1. + episodic_reward_transform (Callable): transforms reward from -1/0 to 0/1. Only used when sparse_reward is True. """ super().__init__( @@ -793,14 +791,98 @@ def __init__(self, self._her_proportion = her_proportion self._achieved_goal_field = achieved_goal_field self._desired_goal_field = desired_goal_field - self._sparse_reward = sparse_reward - self._add_noise_to_goals = add_noise_to_goals - self._threshold = threshold + self._relabel_with_episodic_rewards = relabel_with_episodic_rewards + self._relabeled_goal_noise = relabeled_goal_noise self._reward_fn = reward_fn + self._episodic_reward_transform = episodic_reward_transform def transform_timestep(self, timestep: TimeStep, state): return timestep, state + def _verify_reward_function(self, her_cond, result, relabeled_rewards, + result_ag, result_desired_goal, env_ids, + start_pos, shape): + # Verify reward function is the same as used by the environment. + + # Handle multi dim reward, assumes 0th dim is goal reward. + goal_rewards = result.reward + if result.reward.ndim > 2: + goal_rewards = result.reward[:, :, 0] + + non_her_or_fst = ~her_cond.unsqueeze(1) & (result.step_type != + StepType.FIRST) + if not torch.allclose(relabeled_rewards[non_her_or_fst], + goal_rewards[non_her_or_fst]): + not_close = torch.abs(relabeled_rewards[non_her_or_fst] - + goal_rewards[non_her_or_fst]) > 0.01 + msg = ("hindsight_relabel:\nrelabeled_reward\n{}\n!=\n" + + "env_reward\n{}\nag:\n{}\ndg:\n{}\nenv_ids:\n{}\nstart_pos:" + + "\n{}").format( + relabeled_rewards[non_her_or_fst][not_close], + goal_rewards[non_her_or_fst][not_close], + result_ag[non_her_or_fst][not_close], + result_desired_goal[non_her_or_fst][not_close], + env_ids.unsqueeze(1).expand( + shape[:2])[non_her_or_fst][not_close], + start_pos.unsqueeze(1).expand( + shape[:2])[non_her_or_fst][not_close]) + logging.warning(msg) + # assert False, msg + # relabeled_rewards[non_her_or_fst] = goal_rewards[non_her_or_fst] + + def _add_noise(self, t): + # rejection sample from unit ball + if self._relabeled_goal_noise <= 0: + return t + bs, bl, dim = t.shape + assert dim < 20, "Cannot rejection sample from high dim ball yet." + n_samples, i = 0, 0 + while n_samples == 0: + _sample = torch.rand((bs * 2, dim)) + in_ball = torch.norm(_sample, dim=1) < 1. + if torch.any(in_ball): + sample = _sample[in_ball] + nsample = sample.shape[0] + if nsample < bs: + sample = sample.expand(bs // nsample + 1, nsample, + dim).reshape(-1, dim) + if sample.shape[0] > bs: + sample = sample[:bs, :] + break + assert i < 10, "shouldn't take 10 iterations" + i += 1 + return t + self._relabeled_goal_noise * sample.reshape(bs, 1, dim) + + def _episodic_relabel(self, result, relabeled_rewards, buffer): + # Assumes that original reward is -1/0 and 0 when goal is reached. + reward_achieved = relabeled_rewards >= 0 + # Cut off episode for any goal reached, making the task episodic. + end = reward_achieved + discount = torch.where(end, torch.tensor(0.), result.discount) + step_type = torch.where(end, torch.tensor(StepType.LAST), + result.step_type) + # Also relabel ``LAST``` steps to ``MID``` where aux goals were not + # achieved but env ended episode due to position goal achieved. + # -1/0 reward doesn't end episode on achieving position goal, and + # doesn't need to do this relabeling. + goal_reward = result.reward + if len(result.reward.shape) > 2: + goal_reward = result.reward[..., 0] + mid = (result.step_type == StepType.LAST) & ~reward_achieved & ( + goal_reward > 0) # assumes no multi dim goal reward. + discount = torch.where(mid, torch.tensor(1.), discount) + step_type = torch.where(mid, torch.tensor(StepType.MID), step_type) + + if alf.summary.should_record_summaries(): + alf.summary.scalar( + "replayer/" + buffer._name + ".discount_mean_after_relabel", + torch.mean(discount[:, 1:])) + + result = result._replace(discount=discount) + result = result._replace(step_type=step_type) + relabeled_rewards = self._episodic_reward_transform(relabeled_rewards) + return result, relabeled_rewards + def transform_experience(self, experience: Experience): """Hindsight relabel experience Note: The environments where the samples are from are ordered in the @@ -855,34 +937,11 @@ def transform_experience(self, experience: Experience): "replayer/" + buffer._name + ".mean_steps_to_episode_end", torch.mean(dist.type(torch.float32))) - def _add_noise(t): - if not self._add_noise_to_goals: - return t - bs, bl, dim = t.shape - # rejection sample from unit ball - assert dim < 20, "Cannot rejection sample from high dim ball yet." - n_samples, i = 0, 0 - while n_samples == 0: - _sample = torch.rand((bs * 2, dim)) - in_ball = torch.norm(_sample, dim=1) < 1. - if torch.any(in_ball): - sample = _sample[in_ball] - nsample = sample.shape[0] - if nsample < bs: - sample = sample.expand(bs // nsample + 1, nsample, - dim).reshape(-1, dim) - if sample.shape[0] > bs: - sample = sample[:bs, :] - break - assert i < 10, "shouldn't take 10 iterations" - i += 1 - return t + self._threshold * sample.reshape(bs, 1, dim) - # get random future state future_dist = (torch.rand(*dist.shape) * (dist + 1)).to( torch.int64) future_idx = last_step_pos + future_dist - future_ag = _add_noise( + future_ag = self._add_noise( buffer.get_field(self._achieved_goal_field, last_env_ids, future_idx).unsqueeze(1)) @@ -897,69 +956,26 @@ def _add_noise(t): # recompute rewards result_ag = alf.nest.get_field(result, self._achieved_goal_field) - relabeled_rewards = self._reward_fn( - result_ag, relabeled_goal, threshold=self._threshold) + relabeled_rewards = self._reward_fn(result_ag, relabeled_goal) if alf.summary.should_record_summaries(): alf.summary.scalar( "replayer/" + buffer._name + ".discount_mean_before_relabel", torch.mean(result.discount[:, 1:])) - if self._sparse_reward: - # Assumes that original reward is -1/0 and 0 when goal is reached. - reward_achieved = relabeled_rewards >= 0 - # Cut off episode for any goal reached, making the task episodic. - end = reward_achieved - discount = torch.where(end, torch.tensor(0.), result.discount) - step_type = torch.where(end, torch.tensor(StepType.LAST), - result.step_type) - # Also relabel ``LAST``` steps to ``MID``` where aux goals were not - # achieved but env ended episode due to position goal achieved. - # -1/0 reward doesn't end episode on achieving position goal, and - # doesn't need to do this relabeling. - goal_reward = result.reward - if len(result.reward.shape) > 2: - goal_reward = result.reward[..., 0] - mid = ( - result.step_type == StepType.LAST) & ~reward_achieved & ( - goal_reward > 0) # assumes no multi dim goal reward. - discount = torch.where(mid, torch.tensor(1.), discount) - step_type = torch.where(mid, torch.tensor(StepType.MID), - step_type) - - if alf.summary.should_record_summaries(): - alf.summary.scalar( - "replayer/" + buffer._name + - ".discount_mean_after_relabel", - torch.mean(discount[:, 1:])) - - result = result._replace(discount=discount) - result = result._replace(step_type=step_type) - relabeled_rewards = self._sparse_reward_transform( - relabeled_rewards) - - non_her_or_fst = ~her_cond.unsqueeze(1) & (result.step_type != - StepType.FIRST) - # assert reward function is the same as used by the environment. - if not torch.allclose(relabeled_rewards[non_her_or_fst], - result.reward[non_her_or_fst]): - not_close = torch.abs(relabeled_rewards[non_her_or_fst] - - result.reward[non_her_or_fst]) > 0.01 - msg = ( - "hindsight_relabel:\nrelabeled_reward\n{}\n!=\n" + - "env_reward\n{}\nag:\n{}\ndg:\n{}\nenv_ids:\n{}\nstart_pos:" - + "\n{}").format( - relabeled_rewards[non_her_or_fst][not_close], - result.reward[non_her_or_fst][not_close], - result_ag[non_her_or_fst][not_close], - result_desired_goal[non_her_or_fst][not_close], - env_ids.unsqueeze(1).expand( - shape[:2])[non_her_or_fst][not_close], - start_pos.unsqueeze(1).expand( - shape[:2])[non_her_or_fst][not_close]) - logging.warning(msg) - # assert False, msg - relabeled_rewards[non_her_or_fst] = result.reward[ - non_her_or_fst] + + self._verify_reward_function(her_cond, result, relabeled_rewards, + result_ag, result_desired_goal, + env_ids, start_pos, shape) + + if self._relabel_with_episodic_rewards: + result, relabeled_rewards = self._episodic_relabel( + result, relabeled_rewards, buffer) + + # Multi dimensional env reward. Assumes 0th dim is goal reward. + final_relabeled_rewards = relabeled_rewards + if result.reward.ndim > 2: + final_relabeled_rewards = result.reward.clone() + final_relabeled_rewards[:, :, 0] = relabeled_rewards if alf.summary.should_record_summaries(): alf.summary.scalar( @@ -972,41 +988,10 @@ def _add_noise(t): alf.summary.scalar("replayer/" + buffer._name + ".future_distance", torch.mean(future_dist.float())) - goal_rewards = result.reward - if result.reward.ndim > 2: - goal_rewards = result.reward[:, :, 0] - - # assert reward function is the same as used by the environment. - if not torch.allclose(relabeled_rewards[non_her_or_fst], - goal_rewards[non_her_or_fst]): - not_close = torch.abs(relabeled_rewards[non_her_or_fst] - - goal_rewards[non_her_or_fst]) > 0.01 - msg = ("hindsight_relabel:\nrelabeled_reward\n{}\n!=\n" + - "env_reward\n{}\nag:\n{}\ndg:\n{}\nenv_ids:\n{}\nstart_pos:" - + "\n{}").format( - relabeled_rewards[non_her_or_fst][not_close], - goal_rewards[non_her_or_fst][not_close], - result_ag[non_her_or_fst][not_close], - result_desired_goal[non_her_or_fst][not_close], - env_ids.unsqueeze(1).expand( - shape[:2])[non_her_or_fst][not_close], - start_pos.unsqueeze(1).expand( - shape[:2])[non_her_or_fst][not_close]) - logging.warning(msg) - # assert False, msg - # relabeled_rewards[non_her_or_fst] = goal_rewards[non_her_or_fst] - - final_relabeled_rewards = relabeled_rewards - if result.reward.ndim > 2: - # multi dimensional env reward, first dim is goal related reward. - final_relabeled_rewards = result.reward.clone() - final_relabeled_rewards[:, :, 0] = relabeled_rewards - result = result.update_time_step_field('reward', - final_relabeled_rewards) - result = alf.nest.transform_nest( result, self._desired_goal_field, lambda _: relabeled_goal) - + result = result.update_time_step_field('reward', + final_relabeled_rewards) info = info._replace(her=her_cond, future_distance=future_dist) if alf.get_default_device() != buffer.device: for f in accessed_fields: diff --git a/alf/algorithms/ddpg_algorithm.py b/alf/algorithms/ddpg_algorithm.py index 7587f5114..3191cc2da 100644 --- a/alf/algorithms/ddpg_algorithm.py +++ b/alf/algorithms/ddpg_algorithm.py @@ -40,9 +40,20 @@ DdpgActorState = namedtuple("DdpgActorState", ['actor', 'critics']) DdpgState = namedtuple("DdpgState", ['actor', 'critics']) DdpgInfo = namedtuple( - "DdpgInfo", [ - "reward", "step_type", "discount", "action", "action_distribution", - "actor_loss", "critic", "discounted_return", "future_distance", "her" + "DdpgInfo", + [ + "reward", + "step_type", + "discount", + "action", + "action_distribution", + "actor_loss", + "critic", + # Optional fields for value target lower bounding or Hindsight relabeling. + # TODO: Extract these into a HerAlgorithm wrapper for easier adoption of HER. + "discounted_return", + "future_distance", + "her" ], default_value=()) DdpgLossInfo = namedtuple('DdpgLossInfo', ('actor', 'critic')) @@ -358,19 +369,6 @@ def calc_loss(self, info: DdpgInfo): actor_loss = info.actor_loss - # The current implementation is hacky: Instead of using OneStepTD - # and pulling additionally a few timesteps from the future to compute - # bootstrap values, we here piggyback on n-step TDLoss, but masking - # out losses from the 2nd to n-1-th steps. - # If this hacky use pattern is to be used frequently in the future, - # we should consider refactoring it. - if hasattr(self._critic_losses[0], - "_improve_w_nstep_bootstrap") and \ - self._critic_losses[0]._improve_w_nstep_bootstrap: - # Ignore 2nd - nth step actor losses. - actor_loss.loss[1:] = 0 - actor_loss.extra[1:] = 0 - return LossInfo( loss=critic_loss + actor_loss.loss, priority=priority, diff --git a/alf/algorithms/dqn_algorithm.py b/alf/algorithms/dqn_algorithm.py new file mode 100644 index 000000000..8b0a3c1bc --- /dev/null +++ b/alf/algorithms/dqn_algorithm.py @@ -0,0 +1,128 @@ +# Copyright (c) 2020 Horizon Robotics and ALF Contributors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""DQN Algorithm.""" + +import torch + +import alf +from alf.algorithms.config import TrainerConfig +from alf.algorithms.sac_algorithm import SacAlgorithm, ActionType, \ + SacState as DqnState, SacCriticState as DqnCriticState, \ + SacInfo as DqnInfo +from alf.data_structures import TimeStep +from alf.networks import QNetwork +from alf.optimizers import AdamTF +from alf.tensor_specs import TensorSpec, BoundedTensorSpec +from alf.utils.schedulers import as_scheduler + + +@alf.configurable +class DqnAlgorithm(SacAlgorithm): + r"""DQN/DDQN algorithm: + + :: + + Mnih et al "Playing Atari with Deep Reinforcement Learning", arXiv:1312.5602 + Hasselt et al "Deep Reinforcement Learning with Double Q-learning", arXiv:1509.06461 + + The difference with DDQN is that a minimum is taken from the two critics, + similar to TD3, instead of using one critic as the target of the other. + + The implementation is based on the SAC algorithm. + """ + + def __init__(self, + observation_spec, + action_spec: BoundedTensorSpec, + reward_spec=TensorSpec(()), + q_network_cls=QNetwork, + q_optimizer=None, + rollout_epsilon_greedy=0.1, + num_critic_replicas=2, + env=None, + config: TrainerConfig = None, + critic_loss_ctor=None, + debug_summaries=False, + name="DqnAlgorithm"): + """ + Args: + observation_spec (nested TensorSpec): representing the observations. + action_spec (nested BoundedTensorSpec): representing the actions; can + be a mixture of discrete and continuous actions. The number of + continuous actions can be arbitrary while only one discrete + action is allowed currently. If it's a mixture, then it must be + a tuple/list ``(discrete_action_spec, continuous_action_spec)``. + reward_spec (TensorSpec): a rank-1 or rank-0 tensor spec representing + the reward(s). + q_network (Callable): is used to construct QNetwork for estimating ``Q(s,a)`` + given that the action is discrete. Its output spec must be consistent with + the discrete action in ``action_spec``. + q_optimizer (torch.optim.optimizer): A custom optimizer for the q network. + Uses the enclosing algorithm's optimizer if None. + rollout_epsilon_greedy (float|Scheduler): epsilon greedy policy for rollout. + Together with the following two parameters, the SAC algorithm + can be converted to a DQN or DDQN algorithm when e.g. + ``rollout_epsilon_greedy=0.3``, ``max_target_action=True``, and + ``use_entropy_reward=False``. + num_critic_replicas (int): number of critics to be used. Default is 2. + env (Environment): The environment to interact with. ``env`` is a + batched environment, which means that it runs multiple simulations + simultateously. ``env` only needs to be provided to the root + algorithm. + config (TrainerConfig): config for training. It only needs to be + provided to the algorithm which performs ``train_iter()`` by + itself. + critic_loss_ctor (None|OneStepTDLoss|MultiStepLoss): a critic loss + constructor. If ``None``, a default ``OneStepTDLoss`` will be used. + debug_summaries (bool): True if debug summaries should be created. + name (str): The name of this algorithm. + """ + self._rollout_epsilon_greedy = as_scheduler(rollout_epsilon_greedy) + # Disable alpha learning: + alpha_optimizer = AdamTF(lr=0) + + super().__init__( + observation_spec=observation_spec, + action_spec=action_spec, + reward_spec=reward_spec, + actor_network_cls=None, + critic_network_cls=None, + q_network_cls=q_network_cls, + # Do not use entropy reward: + use_entropy_reward=False, + num_critic_replicas=num_critic_replicas, + env=env, + config=config, + critic_loss_ctor=critic_loss_ctor, + # Allow custom optimizer for q_network: + critic_optimizer=q_optimizer, + alpha_optimizer=alpha_optimizer, + debug_summaries=debug_summaries, + name=name) + assert self._act_type == ActionType.Discrete + + def rollout_step(self, inputs: TimeStep, state: DqnState): + return super().rollout_step( + inputs, state, eps=self._rollout_epsilon_greedy()) + + def _critic_train_step(self, inputs: TimeStep, state: DqnCriticState, + rollout_info: DqnInfo, action, action_distribution): + return super()._critic_train_step( + inputs, + state, + rollout_info, + action, + action_distribution, + # Pick the greedy target action: + target_action_picker=lambda t: torch.max(t, dim=1)[0]) diff --git a/alf/algorithms/rl_algorithm.py b/alf/algorithms/rl_algorithm.py index c0d8e2a41..760dfdda4 100644 --- a/alf/algorithms/rl_algorithm.py +++ b/alf/algorithms/rl_algorithm.py @@ -223,12 +223,13 @@ def __init__(self, replay_buffer_length = adjust_replay_buffer_length( config, self._num_earliest_frames_ignored) + total_replay_size = replay_buffer_length * self._env.batch_size if config.whole_replay_buffer_training and config.clear_replay_buffer: # For whole replay buffer training, we would like to be sure # that the replay buffer have enough samples in it to perform # the training, which will most likely happen in the 2nd # iteration. The minimum_initial_collect_steps guarantees that. - minimum_initial_collect_steps = replay_buffer_length * self._env.batch_size + minimum_initial_collect_steps = total_replay_size if config.initial_collect_steps < minimum_initial_collect_steps: common.info( 'Set the initial_collect_steps to minimum required ' @@ -236,6 +237,9 @@ def __init__(self, 'whole_replay_buffer_training is on.') config.initial_collect_steps = minimum_initial_collect_steps + assert config.initial_collect_steps <= total_replay_size, \ + "Training will not happen - insufficient replay buffer size." + self.set_replay_buffer(self._env.batch_size, replay_buffer_length, config.priority_replay) diff --git a/alf/algorithms/sac_algorithm.py b/alf/algorithms/sac_algorithm.py index 2666aaccb..845bec9c6 100644 --- a/alf/algorithms/sac_algorithm.py +++ b/alf/algorithms/sac_algorithm.py @@ -34,10 +34,10 @@ import alf.nest.utils as nest_utils from alf.networks import ActorDistributionNetwork, CriticNetwork from alf.networks import QNetwork, QRNNNetwork -from alf.summary import render from alf.tensor_specs import TensorSpec, BoundedTensorSpec from alf.utils import losses, common, dist_utils, math_ops from alf.utils.normalizers import ScalarAdaptiveNormalizer +from alf.utils.schedulers import as_scheduler ActionType = Enum('ActionType', ('Discrete', 'Continuous', 'Mixed')) @@ -55,10 +55,22 @@ "SacActorInfo", ["actor_loss", "neg_entropy"], default_value=()) SacInfo = namedtuple( - "SacInfo", [ - "reward", "step_type", "discount", "action", "action_distribution", - "actor", "critic", "alpha", "log_pi", "discounted_return", - "future_distance", "her" + "SacInfo", + [ + "reward", + "step_type", + "discount", + "action", + "action_distribution", + "actor", + "critic", + "alpha", + "log_pi", + # Optional fields for value target lower bounding or Hindsight relabeling. + # TODO: Extract these into a HerAlgorithm wrapper for easier adoption of HER. + "discounted_return", + "future_distance", + "her" ], default_value=()) @@ -154,9 +166,6 @@ def __init__(self, q_network_cls=QNetwork, reward_weights=None, epsilon_greedy=None, - rollout_epsilon_greedy=1.0, - use_epsilon_schedule=0, - max_target_action=False, use_entropy_reward=True, normalize_entropy_reward=False, calculate_priority=False, @@ -208,14 +217,6 @@ def __init__(self, Breakout. Only used for evaluation. If None, its value is taken from ``config.epsilon_greedy`` and then ``alf.get_config_value(TrainerConfig.epsilon_greedy)``. - rollout_epsilon_greedy (float): epsilon greedy policy for rollout. - Together with the following three parameters, the Sac algorithm - can be converted to a DQN algorithm. - use_epsilon_schedule (float): training schedule for - rollout_epsilon_greedy. - max_target_action (bool): whether to use the action with the highest - target value as the target action for computing bootstrapped value. - To mimic the DQN algorithm, set this to True. use_entropy_reward (bool): whether to include entropy as reward normalize_entropy_reward (bool): if True, normalize entropy reward to reduce bias in episodic cases. Only used if @@ -274,9 +275,6 @@ def __init__(self, if epsilon_greedy is None: epsilon_greedy = alf.utils.common.get_epsilon_greedy(config) self._epsilon_greedy = epsilon_greedy - self._rollout_epsilon_greedy = rollout_epsilon_greedy - self._use_epsilon_schedule = use_epsilon_schedule - self._max_target_action = max_target_action critic_networks, actor_network, self._act_type = self._make_networks( observation_spec, action_spec, reward_spec, actor_network_cls, @@ -290,10 +288,7 @@ def __init__(self, ) def _init_log_alpha(): - alpha = torch.tensor(float(initial_log_alpha)) - if alpha_optimizer is None: - return alpha - return nn.Parameter(alpha) + return nn.Parameter(torch.tensor(float(initial_log_alpha))) if self._act_type == ActionType.Mixed: # separate alphas for discrete and continuous actions @@ -333,7 +328,7 @@ def _init_log_alpha(): self.add_optimizer(alpha_optimizer, nest.flatten(log_alpha)) self._log_alpha = log_alpha - if self._act_type == ActionType.Mixed and alpha_optimizer is not None: + if self._act_type == ActionType.Mixed: self._log_alpha_paralist = nn.ParameterList( nest.flatten(log_alpha)) @@ -395,8 +390,6 @@ def _init_log_alpha(): target_models=[self._target_critic_networks], tau=target_update_tau, period=target_update_period) - # initial q value range for rendering; adjusted as playing progresses - self._q_range = (0, 3) def _make_networks(self, observation_spec, action_spec, reward_spec, continuous_actor_network_cls, critic_network_cls, @@ -552,48 +545,22 @@ def _predict_action(self, return action_dist, action, q_values, new_state def predict_step(self, inputs: TimeStep, state: SacState): - action_dist, action, q_values, action_state = self._predict_action( + action_dist, action, _, action_state = self._predict_action( inputs.observation, state=state.action, epsilon_greedy=self._epsilon_greedy, eps_greedy_sampling=True) - info = SacInfo(action_distribution=action_dist) - if (alf.summary.render.is_rendering_enabled() - and self._act_type == ActionType.Discrete): - num_acts = q_values.shape[-1] - self._q_range = (min(self._q_range[0], int(q_values.min())), - max(self._q_range[1], - int(q_values.max()) + 1)) - info = dict( - sac=info, - action_img=render.render_action("action", action, - self._action_spec), - action_dist_img=render.render_bar( - "action_dist", - action_dist.probs, - y_range=(0, 1), - annotate_format="%.2f", - x_ticks=range(num_acts)), - q_img=render.render_bar( - "q_values", - q_values, - y_range=self._q_range, - annotate_format="%.2f", - x_ticks=range(num_acts))) return AlgStep( - output=action, state=SacState(action=action_state), info=info) + output=action, + state=SacState(action=action_state), + info=SacInfo(action_distribution=action_dist)) - def rollout_step(self, inputs: TimeStep, state: SacState): + def rollout_step(self, inputs: TimeStep, state: SacState, eps: float = 1.): """``rollout_step()`` basically predicts actions like what is done by ``predict_step()``. Additionally, if states are to be stored a in replay buffer, then this function also call ``_critic_networks`` and ``_target_critic_networks`` to maintain their states. """ - eps = self._rollout_epsilon_greedy - if self._use_epsilon_schedule > 0: - progress = alf.trainers.policy_trainer.Trainer.progress() - if progress < self._use_epsilon_schedule: - eps = 1.0 - (1.0 - eps) * progress / self._use_epsilon_schedule action_dist, action, _, action_state = self._predict_action( inputs.observation, state=state.action, @@ -741,8 +708,13 @@ def _select_q_value(self, action, q_values): *self._reward_spec.shape).long() return q_values.gather(2, action).squeeze(2) - def _critic_train_step(self, inputs: TimeStep, state: SacCriticState, - rollout_info: SacInfo, action, action_distribution): + def _critic_train_step(self, + inputs: TimeStep, + state: SacCriticState, + rollout_info: SacInfo, + action, + action_distribution, + target_action_picker: Callable = None): critics, critics_state = self._compute_critics( self._critic_networks, inputs.observation, @@ -764,8 +736,8 @@ def _critic_train_step(self, inputs: TimeStep, state: SacCriticState, probs = common.expand_dims_as(action_distribution.probs, target_critics) # [B, reward_dim] - if self._max_target_action: - target_critics = torch.max(target_critics, dim=1)[0] + if target_action_picker is not None: + target_critics = target_action_picker(target_critics) else: target_critics = torch.sum(probs * target_critics, dim=1) elif self._act_type == ActionType.Mixed: @@ -847,20 +819,6 @@ def calc_loss(self, info: SacInfo): alpha_loss = info.alpha actor_loss = info.actor - # The current implementation is hacky: Instead of using OneStepTD - # and pulling additionally a few timesteps from the future to compute - # bootstrap values, we here piggyback on n-step TDLoss, but masking - # out losses from the 2nd to n-1-th steps. - # If this hacky use pattern is to be used frequently in the future, - # we should consider refactoring it. - if hasattr(self._critic_losses[0], - "_improve_w_nstep_bootstrap") and \ - self._critic_losses[0]._improve_w_nstep_bootstrap: - # Ignore 2nd - n-th step losses in this mode. - alpha_loss[1:] = 0 - if actor_loss.loss != (): - actor_loss.loss[1:] = 0 - if self._debug_summaries and alf.summary.should_record_summaries(): with alf.summary.scope(self._name): if self._act_type == ActionType.Mixed: @@ -914,11 +872,6 @@ def _calc_critic_loss(self, info: SacInfo): if self._use_entropy_reward: with torch.no_grad(): log_pi = info.log_pi - if hasattr(self._critic_losses[0], - "_improve_w_nstep_bootstrap") and \ - self._critic_losses[0]._improve_w_nstep_bootstrap: - # Ignore 2nd - n-th step entropy in this mode. - log_pi[1:] = 0 if self._entropy_normalizer is not None: log_pi = self._entropy_normalizer.normalize(log_pi) entropy_reward = nest.map_structure( diff --git a/alf/algorithms/td_loss.py b/alf/algorithms/td_loss.py index 90b220475..8c6c32c57 100644 --- a/alf/algorithms/td_loss.py +++ b/alf/algorithms/td_loss.py @@ -88,8 +88,14 @@ def __init__(self, Note that the effect of this is to change the loss. The critic value itself is not normalized. use_retrace: turn on retrace loss - :math:`\mathcal{R} Q(x, a):=Q(x, a)+\mathbb{E}_{\mu}\left[\sum_{t \geq 0} \gamma^{t}\left(\prod_{s=1}^{t} c_{s}\right)\left(r_{t}+\gamma \mathbb{E}_{\pi} Q\left(x_{t+1}, \cdot\right)-Q\left(x_{t}, a_{t}\right)\right)\right]` - copied from PR #695. + + .. math:: + + \mathcal{R} Q(x, a) := Q(x, a) + \mathbb{E}_{\mu}\left[ + \sum_{t \geq 0} \gamma^{t}\left(\prod_{s=1}^{t} c_{s}\right) + \left(r_{t} + \gamma \mathbb{E}_{\pi} Q\left(x_{t+1}, \cdot\right)-Q\left(x_{t}, a_{t}\right) + \right)\right] + debug_summaries: True if debug summaries should be created. name: The name of this loss. """ diff --git a/alf/bin/train_play_test.py b/alf/bin/train_play_test.py index 4826bef84..9264be0ed 100644 --- a/alf/bin/train_play_test.py +++ b/alf/bin/train_play_test.py @@ -141,11 +141,7 @@ def _to_conf_params(parameters): 'TrainerConfig.num_updates_per_train_iter=1', 'TrainerConfig.mini_batch_length=2', 'TrainerConfig.mini_batch_size=4', - # If replay_buffer_length is above 6 (3 iters * 2 unroll length), whole replay buffer training - # algorithms (e.g. mbrl_pendulum) will not get enough training data, because the replay buffer - # gets cleared before the buffer is fully filled. Training never starts. - # This fails during playing, when the config file isn't generated in root_dir. - 'TrainerConfig.replay_buffer_length=4', + 'TrainerConfig.replay_buffer_length=64', 'FrameStacker.stack_size=1' ] OFF_POLICY_TRAIN_PARAMS = _to_conf_params(OFF_POLICY_TRAIN_CONF) diff --git a/alf/environments/gym_wrappers.py b/alf/environments/gym_wrappers.py index 6e0c634e3..e77c7bc8e 100644 --- a/alf/environments/gym_wrappers.py +++ b/alf/environments/gym_wrappers.py @@ -714,3 +714,13 @@ def __init__(self, env): def step(self, action): ob, reward, done, info = self.env.step(action) return ob, reward, False, info + + +@alf.configurable +class RemoveInfoWrapper(gym.Wrapper): + """Remove all the info from environment return. + """ + + def step(self, action): + obs, reward, done, info = self.env.step(action) + return obs, reward, done, {} diff --git a/alf/environments/suite_robotics.py b/alf/environments/suite_robotics.py index 14fe6971f..cdac169f1 100644 --- a/alf/environments/suite_robotics.py +++ b/alf/environments/suite_robotics.py @@ -34,6 +34,7 @@ import alf from alf.environments import suite_gym, alf_wrappers, process_environment +from alf.environments.gym_wrappers import RemoveInfoWrapper from alf.environments.utils import UnwrappedEnvChecker _unwrapped_env_checker_ = UnwrappedEnvChecker() @@ -48,12 +49,14 @@ class SparseReward(gym.Wrapper): """Convert the original :math:`-1/0` rewards to :math:`0/1`. """ - def __init__(self, env, reward_weight=1., positive_reward=True): + def __init__(self, + env, + reward_weight: float = 1., + positive_reward: bool = True): """ Args: - reward_weight (float): weight of output reward. - positive_reward (bool): if True, returns 0/1 reward, - otherwise, -1/0 reward. + reward_weight: weight of output reward. + positive_reward: if True, returns 0/1 reward, otherwise, -1/0 reward. """ gym.Wrapper.__init__(self, env) self._reward_weight = reward_weight @@ -154,16 +157,6 @@ def observation(self, observation): return np.clip(observation, self.min_v, self.max_v) -@alf.configurable -class RemoveInfoWrapper(gym.Wrapper): - """Remove all the info from environment return. - """ - - def step(self, action): - obs, reward, done, info = self.env.step(action) - return obs, reward, done, {} - - @alf.configurable def load(environment_name, env_id=None, diff --git a/alf/examples/dqn_breakout_conf-lbtq-Qbert.png b/alf/examples/dqn_breakout_conf-lbtq-Qbert.png new file mode 100644 index 0000000000000000000000000000000000000000..89e9c8bc2d4f75522786cbe193c6f7c17b0aa251 GIT binary patch literal 71578 zcmd43byytDw>63ecM=G00fM`0a3_kpJHg%E-7UdFu;2j(O@h1Y;4Xs?FvxB4p7XuG zbH023zs$qaJ>6ZstGaeo@3q&eiBwmW!$K!Phl7K|QjnK^2M31$frEoLLVX4t=^U3f z1-{5zNlB?INJ&wtJ2{wJ*_y$@F(#TA8saOkF!mW485;HtGrvZ6a(@>Q5%bQ_zo&Dc zbC7Dl@Q-0mhOX`^G43kjhip&k|S@auQK_f=`hXDw9L#^5TAnEnpysuvLkPp?6Ws_Jln{%25(xRH$TxA0N3v zPL;{rd-)1Iu2Ws{B}7nM@_D%T*-(nc4NaI*8`oMBJk8{ zJjsDZ-pq)>zga^v1`wV-dwB6I%x(t*KN^1x+wiXEU5>delF_*UC&v#k?D4UV1Q8Kt z`Yf!2;NkJHZTIo9BRX(<#mG<~0R>K*(Bz|_+aVe7XpPM@70gvs;Fy4YR5%29d^kj4 z4<7iyg5Bbg8J-2Td1E6=fk42Rk++QwL)+HV-?;ry_779zwvbotcXfm4}_J zy|a*qD9zt1gn<30!|XIve=l*d5v9>oQKyn}a5AIfW8+}spbnv;$-FMV&z~@^>kk&V+U6kQ5u>jL;vURpL&{kSp93s-ub_p1vHTT=?pt38wdOU zTQL_a^Z$=xPiOuq_P1UCm=k%rnUK1bhncOGw3Qw3RDr6Aar1JE{5A7`o%y$-|CIdb zZ0028Ue*_bAYvV(235|Hn1N(3NZzQsCeu;1r}KKX||&=Az{g{<-X> zgbq7l&@?$=e5ESr(IJ$UmLMFrls=!$zzvS-`G$vuE@?u;s85`Nn;LQV_5-GWFi97p zx&)pyQnMr*d^+il!Ae%;Ja|5HJ~MBAIdd5*wz#PA(e<+0>%0H`{R*o;ngC zyo|)Zn+b}*S}Oyo$^x1KR7QlTfPXidNGYI$;QxKp6cs@c%w6=QKI=cPeJYvc?)&V2 z-kUHGQ&OVeJB%mg)BArd{&US))#!iN`fE9X8kNy=G$Cxln(2R3gb}cK`SNeYsgO|* zms>URe6QvSQ&O5Deud-gh$hrQC)LFJ!zuk3O?u;=R~>1@QHbvI+f4begHD2MSK3AQ zJw?9&tt*o>yuZEVvRkMRLEqq4GiY#F5zDUTxD5zlDvBhKrD8i1vz}sXKCB(YX9Edv z-JPX(q%_(sE^J*)s-=4$H_N?X(d!GIw+5&q+ zQiWXir9`9EocJ3&UAj8@Gu;Hp3;NelJB7LgV zenr5b8pG|%G5+}7Z7BVJt5&8@NCW>%HYeknu!W!6s*JjMhNrHF^R`A`P6&1_X_ zb^Gmi4;#|n=`u*Q|GkY?qiv=yHM#Q;Czs3a*iVWfo2q761A)(m6$*rb%5MU2&KUB2 zCa%z`z?NAPH&VV6MG9OtGb}f-l}Gyn-;G%Qfs?ggW?KWP$1gUInpV;b?)Npio0&d- zpM%}>ia!`lPCnL%T+Nw$xftim^V)yCH(9DyKk20lc!DAa!u0o#$?hw@8YKMox3*Y;IYz-ey`BQL!N)91)m`J?y9@6osUBhQ2efELr4Vn8(mv)oAt-U%6Y z!Y%d-!3HZTSR{ovLsZF+aFB?*?}qtT`X28_A2nn7&H4Lw^8MLx#I6g-7qkUJjZ2-LMM*qYJTBc-7>$ZiXO`3mdo>4M}u|CeL<%;5tFpJ=+Mz%9$qA=(1l7*2P5jZ4m~1$>m<;;SCVZj9_{&xw!!=Au%^^_ zng7>?Zg@QEi`xrj;$>H8g^f(XH{1fXH#J&K_Il-*q1m13-LGaKtRj_*?)T*AAH+eb z{=L}!H2NsU5uTAqF-NAI%3-x-G3In9+u05acIWRfh~JRSHQMQP@^}^a!mi*=Hi$WL z9Xc()Ajq1SO=hZ!8%B6_ihO`COvTAam)eaA`+yR|*LzIbO-c+*2Av?(qxPDQI58BW zLMbzrw}`TZCeUFeT0@GZF~JFv*py|4;o@|C5)P8HfE$k=mtS@1J}nEZe3gj~@HDD< z$ssLbD8_im_c(+q-RWaG=gUoFSiPo7@|BXqpd9|ZwkiCl#Q*~B-$AI4G}PcU{|P#O04p>z&?w$y3m*A!naFJO) z(GJhE36!6H0qsMPHBwFYsuk@FAn_ZB|0Aiu&1SM5-WOh1J`%{7*gf8T62Wn$H}n`@ z>Q*W`lgC=kIp`s4-ue#I66%ocIxs7VX*954?=+)VObgsEz8coo&=1NF~KpQ_}DKh^!eVkLJZoC!Ct=|Q^Cpk zTRJ?lnF)w&5{Te$i3{a_%%Ewc+8dG69ffS9c$ zI8V^oe79m8Jh!G9AymR6OKYOgdM+D`#bBD}0#29t1~>Z5YDgnnkJvaYhV&>yC2&u% zetuLJanUQRk~Z>uq*E01_yC*1z7Hd{Pmtd4!N^#n`|F>AsD{vy!LY~z)gO2kKQsR% zs=Bky^KQ;qHdiV&THe#3KdUy}Q8oA@Lc<+JEEL+^ZKFdev}S@xBe8P){8mYxC(q)9 zFqm2J)-`PRZM^_D=^4lGxM(3Uda4(58%g4z(fH||uRqpk_eX>w^#jD)1aH|&!vD5D z8qVC(|89eFt6^53fE(X3H7U`Lp~lalx%dxQF$M4*0lvIYSHADV!sL8eCi+fuiY>^k zgBk|D(=#a9%JqwuBY?Nt#ebRZQ6(hxIHVnKc|3CPuUlN>4^(-zKqBNzuQ;41sx$VL z)^cK535oO!>U20P*t^L|uO;ZDCT4*%>d63-KW;f-vmp(`vXXTD1&($>wHha>8THNh z@>-*!>NyYyN#T9dZLZDD3B~$J(8rbK!rvQ;R@xu&-fhgn&>dVT7mom=fL!i2jd@7r z)$q!FBgKXz7Cd3DiZ5E(k_r%;VFrba_Z6NPUL>Up_);oj-n#n?a63erfQX2nb8K_E ziQ&;ED!UZ3%N33?BI1{i%4q$%KN*+S%Se|d|L_!vu_GuehR2WlcZE}(R{R2HNxRu9 zJ?as{ywTQt#_GwH8ltwZi*btJ@fK3jUY|USjIli-E1pjtsH1D#BLLU-asQRm*=NafVr)2F4Jfg?apH zkZ;62k55Oi%3OM@zC5;h&O6vfN~-GZpqx>h_dV1-VqY7t&lYL7I>To##4j{a_g&=_HnXYXMMTA96(QyBq$ z57&6Ax5!cbq;1E+NKF`Vfo1biB(BuSy(pqM=l#86kX`n?2axt2mb6jm&KQjpY)>IS0KcBsceWs^2L2mgwX z^Ecx93m;!0oyoCC0lu%(v55$>h)ccC#PRj5pFCN;7O!CRi}1xl1+2~FmWL2KZpmvf zat7)#>CyChP>phjH5liGuF;$Bjm+p@9HIlsj*?2MzQlPrL??xZ@;maRN9}vPU7`~r zUr%gy+iBFZ@C;4S#bj3AB}m`46?gM+ylbA4@$W@$khdNsV)9fg`Thv?f3#@Wx>h%T zL%84=XT<;g>p#E>R~#V@RQKuaKL8Bqo3E;xtjCMpP~ipoLv~K1E-*f8d(=PpOTrk{ z5I=&v^}oMv(dFsv6)Nh%d}JHce_{0N5)r0*A^6SNdGl!*AV>6v+*XdC1KH1HG^DN z7KokN*21IB9w{HM!`q0Cds#<xPV=Q^#t4HJRji^w#;;5w$z&x+l z!p}hoa|ZR6Z<*K=KWKrJbzvJiWV#uc-5Kd!bCjJ21+GRrXmj|_om2q;wA(5H=(79F%IuY-z|o2 zxou^|Q5*WOqQv;N3a2=CtA!P?jtFuXg54cMok!EbO0ova4q zEN~qIqpfqbD;Tk7Fq?JsHw`K=QHk3k?u~l>4%qi1xaRk7HdNwa-wHb9ybbN~pOS%@ z+>C&wNd8X>+a_Db{P=s`J-F26Xgks5rT0JKNkqNsMW9R0Z#v=5skCO}N}Lmk41C5Z zdZ&rnrB6HJ3|yCrJQ)-!HEX)9N87q&$G#C5;TRlVhcb8(8+B27?NT2YJrnQ59L!Yx z<&{f4KIy2VYfrgHDXUakBbv7)Q4^)9Uq?LFyvY1sq}B5 zQ-O@N+R&L>$jeZ8p}V5lgRrkzY>773tspx&xS=uiQdxEiXI>>-3%H$ z7*|VOd+p2ftjL==4FkCJ&+W}KqM+>492`=k!{br2Q^sDS>x!0A4?mI({EBX&4$|m?KQoQhZdI8&h`wKy$#Z&+_`}Kble7`nTDN+Il&Kg!#*vk z_j={eAxzxmUzse#cd4}bV6GFs9g633oKRpuzRubnoI^T5&A_HNq=|c`e3BS&6FBFy z-AlwFDT?4@l>MbH0FA~lo|J}=gJ|GA9|^ORAwuvoEOcT7o6$_muumq|@+$J@;A#W$ z@-|*c83g)TgU-8*$}4=V>Ph7SmDQFKlNzO!c3(TssX-*BnyiDD+~!D(>9^U2D)9Bc zErl_)u!)^gOYQA@F*lGgsT*2*2!DeaE;%SM1MLITQlxSatnX&M$h_s!(>I&mel!-SkDavm>A#gSujjXM(t1NK}H!#3wbFJjaS zhF^*H#xmpuP$$B3=Q!}T`Fts?;e1Po80LtgyyE~_#@*W-8au;iS>oCmo_^IzS?ZAYFgC+~ZSSjIFdosQ5sI=Q*`huD zcl)6$1OkH2!I^lQKx%E~Jl;H+^Ha-mLoxe9?m_I)E&!W&dyCx7f3@oxMe0Bs$`$7S z0l=Maqe6w~DWeYENZ55p@?R1MHlc7=sIGjJDe&j-8e!YC%$?_*5%3Fc9Q>5Mffjk5 zJ&lO8yis9x&SIn+_E8NrLZBqu;d?KrLr8wo2=1X)c;y|1zB04*r0sd z8FBTTZSL|c^lsGuVKT3~F}u&m@(Pb9Q5u1#gOWOjKkKdS_wldqaAs5)!>fI(3vOgF zmX<6zzu~;aUiKw+#xAGsyf@`JqNDD@l5EnBygA=lj4-zx880HiX9N;~jU-MG-d|0_ zI4YQE^%yrft5sMk93+N3yHnAOjS$dz+;kO(BHnQ?1uG)n(AcvLHUi!8srNYhw=+5p z^Xo6s_GT)xry~k(+(~q4xV{d{CI|K)FG}z*mekeJ!kNl?V~m6-mttF<;8m%3#eET! zJ;!^{#ihx;(9LF-I{%6*wR>J1?S>Vkg3Fm6kua<8UFnJi%vGp@qxY>;+CZkLu8>*8 zW;Jjdh5C%$$}XC`3L`(GnlPOHto{Zyx(b)_m`nOzGab0k%EZbDoZULHPBq1 zUrCRM%l4+T>X^f9!L(BZU#d;()1j1Cr&GCHdrjlAzl3AX=t@)ie3>C|VCX66Q3|4-F099jRY_4e{0Ty+g9Ho@XQq^f)K z<+nLv(C*hcugL&pdgfKEm!-blp%;dvZ1vRaBqk_U!wN%|vAIixd2D$$CrJS(G~)}= z8@hvNunag(tEfh!2qI_lf%QMK77nsH75!?U(N9GdxghY)fS&lL%saNBzN9~)JBw>_ z6bmSzm-Z##)%*&N{BqMlrm`E_23F|zd(DRzvwnx;J_UrI#JAZ(f-t#NhNSURE$>^K z$iNmC;vl8PtiozYHpkqk=4 z1$TgKt{FQ)Jou1^nI>89{J9|R6JX{a=rvAqZ%@mW&0#l`e%hY0csiGWxz)_b;4g_G(3ovXZ~IOjJ36dt6UsfkU#+GZ@z^(Gxp;fqCKr%aL!5)=UVOSc3nOm6eEs#Fp#eVYVZAzBGn3ZfP!QCQ>S zNx1l&bI+&wBzpQtW3FH{1(vl91XfOSd`rvfpX=x_lW-rS?oKgtl$!dRO&R8a2)H2n z5bU8jk;Qm;p$tug%q?yEE@}F-u{A?J{Na?|EtyFPZF-)!Zq6no(t63Mqz<6*B*THP zw4k-)r2UI}V@z-&A55xDG9vvjrRGAqbViC8{nXi3@{sG7D=48vOO#K9*lp zoZgV0enuWI3GQu6BPx_1O&dIzv5cV+J;2;@Qd`1U@P$_NzNhaX$fJId_qP6ei8Q8BS8YQ4n z>p;9uGF98&^aIaF07$5$F|uzQE@`^FAOb7wIE) zJxik4AMGY$LGW~f{uXm{_27!6$K-xTIbcdWGSAU^VP5-vnymG{TU(d_)19)cMU@Uz zm`V=Do{CJz%J6{{MUtVcB-xKX6H%zo#@D<#Yq;!Ip-*FZVwi(XHuH0sAP z-yYB&+Rm)Ie$?dgxYqpY?dxy((6sQEMnwqWoBtIeS7LVUU!8ZIjz z=}KR(+vPgtS78{#BV&rhRY@7`Llo-%;q;lCA%PyDUZy%pmxrgn{an|36PeNzhXjY>Ev<3r zLN=e>0OFrKQSVI{q$3?*J+}oZAS3O*Fw>rxmrlpcJBmOicYQdGf!wukWaKJ0zk8-i zkGYr3IpzaEn5oU59+L?;N<=U%RA|+C)NLv?d7hh>eX7iws?g$IY4*Uz)%OMx=n66? z%QYt9Z6B6fTsN9cfAf@os!V#psx9B_v_+H0zFW{}f&1k{gGH;`5qc14Dwn;&?+{>b zlEbh_Z@b^;!>DYdP8XS#Y#3s zQ3_4sMfoVT!*M9no?($}3R8^2zagY;u$jHmll>F*y!c(QV*9!CeVaE_5nzP@@{s9} zo(14_(@h;$gHUiNQb>5M~j-#GjniKJ-s`4e2 zlUJDS>F=iVs`OmRr6piBeQH7J@i(xGINd6WkN=CJCk%We4bbS+LFF?4dY%NDv>}YX z?1#5E8i=#4SXn}1*qrN2YJJ?l9;kkYI^-#{SxHo+dqp9bx1h|uUEo{&To8J;UY$Kt zFhy061${R;c-;i{vgvl`a{!BEih4Ji znwcr&i}@A-%o1AqmzUDOFna%=!$^V!3GAV7`Vq7BO0~3iAop}90>^Rbb8kOD_qG0n zgu=x}V0>sc`t^(yHlD7)JM_0rh=BnRDwX0&(Szaq*RJAT%iV3&3Z(NZ-%_Wp z-D*B8;l&L2{m?g=_q3**|E0EdP)%U{Vu3p|KnG2arT@2d4uiJ$&y#EtaX-+#uiM^) zj2HBn+^Z*@Ma#H}+>?Q04nUHJb-jFD-Hi?_Yydfe!Tt$~AptlLJlviY&@b}j2-Q`h zRFppc>U_slua0nzQ@P{I=!kgE5Rw z`)aTv7^6r2{S%86gvv7aS@T5}^5+8r>$G(Ls?Q6H=x4)W!CaFpg=*ia)5&(!3bPCw z6n&iZfv~fnMq>3og4UxyrK@3iy%c-%^UXb2uJ-2r+i)(kfskX`mt?wnNv{z_bQkHh zRfP<-c3nHA!;r0(Wj2!zifyym)-O-qmRP&K)+zoDS$QGjOqodo=s;4c^jZ~xc`r1P za!8K?NRcUZ3%eh);5yyr3p#J3ZgXJ}vDMS6EUlnW`cUNxxh8c5qbx4!H9HkQr6l*i zAr=bj8*RqCl_*s{$&0qgP86q`=OXsK&_StE^V<*VPoVtrP=%d&zIomJez*{L#mK~U z{)l(O*c8>JA*UVA8Ad5~-%xzluIz!m{CC(EMBts^yGt!lA2W3&GVq^$Y<05>RF`@~ zeaNcQh$)HGZ;?_3%+EAdnubXJEX#~zuehMfZLELx(84Y zON9MG4rpRXLYW2DSw}OYU!8Y_W)j{cb3PWv0E&gZkZBK6)kz@hFTsY855ever2kQ% zL0`4Rz|uEd8BBrJGE+&lJQUR9J5<}pH=1uR&LWPiO#cesBTB{fas)BRM_;V=lOP;x|2wnx!3VhbD9LsORkWjw7ZQ4dR>mh zb984<;@8&WUqhjeON$vt)L5i^^(A7TEiUMkLz{^yh5NG}1NjBMZ7)v>w3Gqn#_=3~ z`>YQz(T~4PWN*|l_DiPT&*=Gij_WQ5*0WmPH+oHy3DV$Zf<}Q)Ez=cd|4v_o$O!uD zRSQ=U5e9DaWllp|(^R~ANRG~` zAW`JcFE|NP zo>UcRH=JRQzUyg&f^7yRl8@cNq0AjT)v2JJda++z`cR!YXU^7zd5>4O2?C_$v84P9 zD7@o$MJsg{@%NJIL>;w$_g=zXfz)i`-M*{FShL1{$nivX-!gaCbz^D+%Gcsnni#$<(WZg%P?_gbD2B)EL8xf)=u)`a~5A{m40N^yXU zcUtj<%7BuA#D~h04#b988`l#D8h?ns`A5)ffKb@9a&!61R~s3s&q&tokKaG=ML(Zz z?8Pcb|LM&gO5hG@J~^^93n`tv-+hv(b%mi3v#2ipM(sEr&siq2U%X38L?>m|6dBqHq%7S4Yr!C{ocI1#Ot@WuccF)nNs|wI`!r8st<`tWy z0f>~346A`*1d-jEfHJ7a&dFAyebHOhqg@Z#mq(?F02 zzmt}*jQe;H{5AyyQG6MZg9#D-re`cbs$CWRT-a(*Ph&PpqD-dKWU+mx zLNW+){6=8fy__plNZKTP)>0pG22e${H{@Pj#ryS8( z$>g>e(6Y!)W#?j$R9wG|A2Q_}GNZcl8IvQBbqB5Ap?fb^d5#j*AIajrbUo8~Hm4r9 zayONDm$JeChH2Zs>>Hx%DeN>3pXMc5bK`jyUX|^-E@*v!6)MJ6Bg))%j#G9GS+$%V z|CGDA8Y*Sxoa*&%SmeFmXZerb^aPH~V7^(GS2*ymAg`)bF5?IT5%2<1Ho7_WyS7Z2 zOrb$I@82qGr^0{vz0qygw44@?Dxt~gx;I;$|Fe@WnL#a0GWgkiVvff>ZDF$o;o)9s zek0P0b#Xvm^isglkb;HEoJDocyec&Nd@ZiGHSCUdDIDk|CD9~K*L8|Mh(&$)t%+MY zCU#1o0wL}91U~UMu;Q=Sx{K^%`+9~F2i@&o!xC3o1ta{3#j_8~TQDBFtNvS3)N3ugE)899{{5Id<`nM(!_ zx7z0ehKKoMh?am_QAxhxx@VM9Hn~XOlE+4@(-3FxW;p@h>12rt26^uldH3)V+RzW6 zLwtT=@oRpSl(bCD6!Cr1Fb%kpCf?CFs4u_;=ooi663p5vCj_rAT+v&x%z z3Qi}MAH1N&xPcmtD55xP>Eupv`B#?O9&+G7D`=2s?OK_*(lCrcbBZH4Fu^_JqQ%rz zSTw|x?wlA>6RRdpvvth)lAb4{mILoO>(6B7s-N8+qgrOerWpnMm$f04VtM9N*pb(A zVfBLR&-O~m#4!JA01%#D0C}b~a7E!<sYv@dC}Nzdd0)p(fK z8xJ3~VjOgGV`9@|deN`$Ct$ z6afaY&A3PtJnHYC#xrgh@J=jWx#DniX`-WE9o?;@qb`7cMNOeW%F`pi2nuQReK*R| zTHq>&0ih}KT02qI9Nu>e6LsB<4h7TQH0FB7KtqIOU(VR}c-N*lFsxHHr$`kyRijGF zbt)~5H?D~3>q*GUDUF&6Abj^6J$W{@&x~+N1Bdr`7+y#}HZh*CO}^ef#-g7@p|5{1 zov)ajEvGnKWkGCr8}4*6D3f%5!|=sapS5*Xq{nJ=nR8%{a|bIM(#z@ImnMc-^iV8S zq`SrRrPf92Z-gC!dy?cf-N=BX*FsNkNgqu>A4zbScq8oh8@WZd&OA2v$5AQf_9I82 z_)?(;jTv5F{9V5?2*_2mR)!}*r4wfBqa>bD!Qu6O67}KQe=pk_r~4Knk=II>l;8Axq{>IQ|cb z6?_=l*Cy4l*bflLci()L|B)TDA%TAD*)47au@K;M3N8!ZP|%E`ngJqA-;b2*_gyHI zsM|4r1WzfFdrcX@!N{l5Q?@;`g$0>}ffurHTw;fXOP2Y#UTMiTsmXEEahv5LB-f5d zu*O+~#yxvc2L6!m+`02joA14=%x5PLNmzx0JTa{@en&Zy;jgULeyP1QNXGJ9v5#-{ zUtGB{xLN!4=-k@!=?R7vNFXtR?!VTPcvqrslH0-TdZeG7RRR8h59Alplfq^eSwv9< zu!K!PlNCp4DXk{_jWLJS%gsS9NEGkWQ(0AqQL<kp19i> zW#;oNx`t@nAF!e>gnN?Fcjw|`F+WimPv$-Hr+5Q=$@V;u11TyML{Mf2s8MI0?OjYd z-%Ga&aS5(IMW$T}5z+!4Cx)*L!2YR3?D{OspDGfVt3eZbZ-Dnp=O7^VBDn z%Yt#?nZu(b>%N~Kc2;blU_pkAcUwIw(oSMXWb*p6@v6U$v3f$h`y2y@AiT$~3KcHn z803VDdkLiBy!%d2%XL)f6soIlq@!H$KE0;JMWsvPO_`C|ydcd?2?8LhEp0YE_joj% zRzFC*6RuTO|hYYjT4y*YB(~ z*xXAPo);-Sy;laI!!|uJa-HG?s%JADxBJYVfZn>|^Z`&zGbHxGfSG}Tf7vCOcLabD zB|w5i*y;37O-KFSW+EVk-YQ!IMByO;OfK4r;B$#W;a8~e78HnY<;EI;^&`~dojS3% z&i;6uN-~95?f@@!By|>^iD)^P`)tKK?`R;m1ugj#ZDjfqr8@zIbr9bvuaIl%3->z( zz77!u>hJE2#uw1P-fVfs!#O?t7FmJX)MHp*8I4{fvKJKy;YQc*n* zEWX<=MZn!&Xs}s)SR9STolkj{DU^`ksi~#KCTn5L8+iDpJ-IBHcBnrkunGOg1*uU3 z<6_-}@MA*MW1e-O*uwW@GLYl96yAe{R~4cJ-bp+SG*s_wH`@UC-(71tPE(E*KSf-c z>z4KEI7b-Jb(?mFprwHbn7r?vaBkXnwwTOgZ6|GxB@ zaFG1$q)C*7f+XC;@pGPoa}?xZJ%N$m@01Neri)f98jXi+&!5HPo+^7>rhP2iczyMU zlIP25UVp2~Jot=M&$;e{)x5Tw|5c6c$`9473xW6h(AGBtDa?)bj1Hkno%%rX=xhmD z+oMlpb1`0E*0w)o4YhPdNZmo+8NlUIZ_NqXhEDFvuRw6Em=0EnQ1oBahp*gWqs#y`1wx?WbVZiJ28M;R_txp7?)lmIzA208w;k7& zhYn7lyX#H|(^&=xDXW);w-4m&O|2`~I?*kCphU7GbtXKhE2m|ODZ9F2Al<+r@`ITa zA;F)zs2js?bH$~H`VES;3-yUoa&b{V=opWosfK(iRR9Friwo#&T=q&bdS^|#@mBU} zwaOdRB3@7T2Hkd_W{#=ZN*#W0OMpq4Xm+-^xmt8May-ltbgod@><+~+1M)I_SYo2W zuKRR8aE5G%Yw?%eD-Aa3KLi3(=uDUgaG7iEbjGS1nE4YgfkbD;{dWVM_f?_3w3|fE z)BsFz#KH82J-672JSu;J)tKTp{TsG2dac*5!-Wi;57I%UwW4wkl?*t1FwRjsmc`!D z<`yyoHl3O3QP=*A`vK>q`+>T5Uw-vFodm(x+B|r_v9ZtQ*ND^Qnajb*bFk+*PJniv zlA9fC`(dOO^{FE%zDOa21q*wjP41-cQqNZ!66FAaIXceknSBI}vvUn07 z?qQ4J2$iN!3n#3GZ2HcDWh@xme7KLmN{pw433_g=afE~c2W0+tr=AmkTBX$Ciau}j zC?F#TPgA$JdQI9w7$ zgndTtf1dB?d883Xdw8087w4c>gl}n{-+2wLBkWD(P02m=KNlH1yVWdXnbmSeb=y^z z&sjf%+f)P9dF6><1Sy|w0-)#Uo2xN7623ofg`@%1FTT7lvDuo)0!g`0cy8i(0wDZN z2%0f=nLQZP%o_@?$zR5V-^ah?7|h$eN~S$~c`!RDL!?Lex{TsI60jO7y-6c&)DJZ) zlLPH}*C+CuJICj(jRfz9)3Otye1*H)XGMmZ-CXe01^w$@Cp&R=NOs%p97+C7SVT#D zf9kyH#!~d@H?L7%5vSpbdR?HV=J%0F6B^LU#{BS!I|3+Ro$@F^@*Fxa8GDT@FMJk< zhH2m|Lgv2gn$8QA@IKy*6F&Z7nn#`OcC?_yjszrKQlC~K(SgIG5G`M7;mZ31>1(N? z6^beUCa}sxJUi}l10uA6+npfOT(n~E7k-w} zsJGgqtiv~kJE(MGOBTYtsXv3?oOPolw}4bAUQnEz_)vBrD5_wW&Wppkm-)Z1=_xffK2AgB{*>3{1kj6IBjIYX18H8YN zXE1Y(GB?%foHd~7-V1^I?VWLY81!t}YW&v$%4dr`&C72w%yl2yDPjCt26fRxLXg?B zxuxo@7*$E`+V;mx!)1}Lb_RSE`O760mU%`IMI<7WE_=H)l?%bQ?5|M24KAj1ehW>Z zLfZEckQx|;t*nxlHHy+|zi?mnxX0j6RWQY-sqsokW!6Cd?EW>g2EV#nxD*{3Roh=g z^#+fBHugq4W9=4iKN!k3WN|;x6H99MV7l?qF!0^s<}y3#CI-WCe@e0E^S}=@aN};$ z0%^|{@!YG9&Jx7ZKjc$C4(%I{CT>T0p_oFvDYfiw)Vck=(*&=ehb?Et%@pq{ha)|Q z%yRl;X3c2#PCDXxmNSPOF;8cvr^P*FA7L^Qdf;0b)$>#VBqwE4>uge5MeBJb2X!^B zmt%dn8cTN2gIRWI5JPrz1fX?Nw8W;NBe4@cZqp)847qdP9izrU_o4C-$9L33u!@tt zIya5`v7uc4_V|_+v0FvZ@BBUMnF&)$44@=Q`6eZs3`Mmo3MULSmV;w$;Wqi48c1m1 zYbz1Qeh`N=%k~X_hdj~W`JV0|JeRO;WYHmwkMER@z_x4zr_~H2Ff9gm-er|`bnv3_ z-d11P+=~b2*+!QixiPjR4u2-KgF3;Zg}0b{ekE04DVriV*nFDc zShe*76E^6z<;4ERd=d7sm~Eig!%Z8Gd)bG^Qf6+w+28URe0JL3yezvu-v)BWN^et* z`0y=)s9dsyM6>AHbS8IQ* zqqkqFUwLG;^nL@2Z8_AU7!Xl@vF38e4H_9=!cc}9pzlLsA6v|Rp9 z)iYhgdT(kr_}&iO4pB{*>CI@Sa#d@Bf&FnW+8=Wa?N(~sM@b$&?YeMuTY(wB9~Y%p zw3^sg7A;dJwzz(70fnBlpxsA;@uacrw19*`vXiHhVE^@&Tx)v8ppLvd7n?)>D87yU zM;4#)sr8mw2OV`jN?ds^DX1m2PLnnhhdWzDIp zA-hqeRki4%pB^qrz{KU(3#4SLW1A6AE#97n1FKa1e_FB0P~jjXMzr$7hg>@_q$QVr zBIiI7liwDMPg`Wh^^k9)J?{9{DAm$o17g*Cp3`?bq>*KdPQ?82ByN~>df)SR-ACsg2^2O=e%3h{3 z$&TkhpVf;;GmOGnbHwKDtXKNy5)N>^@`s~Vd|{}8PVC*s){ZnY`mV6j2V2{@U{q!~TsnWNm0ii#RO@Ze^q4FPYmz2e1$*6ck*!=z^v{lM7mxap9un1xPw>>!n?mDj z^r^A_f#juNPh=Nn!GtP_-mu6gHN4-Pts|JpVjUU_w3*i z*Nyp7siolcSMv>&IlI{5Gb4V>nG+$Zgg+$3hOF_2H6&`hgd~yhU)rnwumplVLM))> zagFzGCi}DWO*qV1in|{P11Ew5tieiZItRHg5CFBj`{R*SZN=itG)!;yI`1x#)&o?x z>*fMk-@NfE6o$fD%GV38Z-kV+^crU)MYs))X4?Ap8=uzVp&{r`09kpOKwb8)r~$w| zF9p?+)+OMw-AqTjsa{%SD}3+kd@jjv>Bg`_Swvnb^i~zhf|=F&^L*s)4|#7Mc|X<= zVZELEUNFhotQ67;Ikf7d_pyvr6R_;0VC;!PnQkN!TJnOI>5K;M-e$Sm_>`KNtU#H) z+-k8lLyC6p17#O^bw#_+>Bgm?M0$@QVlw2s9%_G;zu~|fd$9??JgS~XCNDFMUu zny+3o*wRY;#ZB3`9}5>7pcvqTByvs^7UROp!*24^hNYIOc{hP!(soPm@^p#!4AzXJ zwf^AjOg4cfS?!cWeIa|r{xFsFxl!=gbO(dr(EQI}m23E$1E$=TdAkS@zZzp8z?nZK zG?Y<2BE7GggUzP<`fInQqNB_BytDuw77l(Iv?GE-(aysD-@gSAVvU^+ENO?$9OH?R zlj_T1VT@D=1`zy??b+W~`;KS&hCzAgjaF(UG6D z5_8D3#1=Cy4R>wVUC6SkEi`*}SysCcOa{9H7`w1g| zQ56rH0Xz5&sDejt_{F&TUa+tDNHFM#{_`dbj+fSEV(* zS}$krNct55F`wfQD{Sp!^w?+6R!IN%T`mWy<+)-YLd1#s_rle>{R>DN6&4R7pD~wx z5hsvj{A;!BFj(zGXOuAa3JkXJDC?y!9B|@^b`8yKaf`ZqA!0sC@RQaR`~L`zzuB;r z*9}s4t>+^|2A}^NY_7?l5*QwI52V~vDfwMq zz_m3uMk5$ZXE%%R0V)vI3SAAcWiVaWg|WrQF`}62SRcqvcQp$6PJX zw{d*JV?P!cGCGfOP5sGJ!_Y@@8Qp9=knmLxshP zcq#0yKPY%5#pZRuBB}s(dQtobwOj1_L{}fV{}Kv)E-M)560k$)*YcvuGzoIPI-E~P zN}>Vq^y&wTJXdcfZs0mH*~qn!`-o`&XFvQ^L+YxWVdV6i1ZNMh<;wHDrsMOv0iYEe zyH2}t^1H`ulp5q;P$E=U>NZv|={5|vUHhFz8fsDfUuDPFOSG4ONsD3`>=FBVd@^4y zU#H%B5=cE4SzQECVd1{WwX1|Xy>-%5V*ivgndpV>2@Jl^MF@$r<+?)vf3j8H2u%Qd zX-tfMma8GJYi7=ndz>o&6+B?=%s>!l!L5RiL zy}-bGE01*e*^%O)i-E_x08cCmHcZxHKu;S4NU_66o9Uvj$-^GUOWJJ@SGuJg=^Pd@ zTuItZjyYwzjidO_Lb=RyKSRwBOYJs0mY=-EA2j2hM!Q9PK85e~<+_cp0}}307r29N zu_(G@+33FCjRLG$;_~)dzLMXM*&A%co&fY*Qa|MP0G49?a`c9NV|OAKzpuvZLAMk@ z0HzGmEf}ziTOMtS?S!_F0OCEMQ8~S(d1kPmuIKR2YB?=$;reLD zwCf75chk(=vZiFNp1M1eLY_&^@S*TWA?k-H1Rc6Ce#D!=``>OQ$~j_Ws>B9i3Q;5z z&w7lpnT!FT$Jh$}Pmj?>jKKH5z1ZtSpJ8mp(EepRUn4n~!paIrht8omI{cl>O-?4d zjX~@?AI!VM!>0O3J$_{$<_JjW%hRfGcCWC$Rr_&2Af*+2LZkYMN}W0!tF6XC5g&iE zFtReyj2WzpqhS=Jo(F_K&)cvU1Fn$}Q6BV&|%0r6w&PbAA= z%1?F(qNqv0-#cvSlLY!e8*q;jY5&IA9VFKDyztN4?oh`|8I)I=DNTb$D%NkE`ZDCT zYkwNHEzNe-+v#n2;xvO!7V#?{E8Z(jI61Vm_GpKyHCQb17!0EkopvnF94~{#xdMXv z_M)16ET71AsD6J{uBJsn_h>NwbF(9z@AFLG<1W=IU5mHr70KiQBipyrU43 z=~(%RSJo{b^<5YOEt{n*=wg>LMb9~0;Bs2^3=k?w7=tUz(cT~%iSzkfJ6g?EWcm`l z{T~l9M4PQS{+Yj=1BsM_@Nfk<28W>*(WQk|?TS93is)o;*_hJ}*ZKkh!hWEyBz+u} zYBTXMa^QwZ{(JC$a)43~1Shdb}i2t zoyns#(Zz$60+jL?=)j4$sf1`$J@{PCXm0J7R>p+aE5?PPDHRLvaiwYddsN8DSgaf) zh`of(w177Nw~31KZybAMB05XcUppPuNC3XlA1CwTx0C-$tB`JLh?+X(nGbY)K$is; z<2(-){(Bi8onlcV$>iS#=-M~;Txr^Ohsa)s$560O;2|E#+mDN=DmEEW?ci;nIqJ73 zUVd5>XEU^2s4H&VZE93CX6b#PwR!C8d-WNRfGvt{A{Z;GR^1O~6`-CQZ{(nEKO~mM zGH5Zx|0<}_{9%wnwP!Dp+xDrEc1GTt+||K83H%P=8|R?_zQMb!fwV65S8l=Sh1cD6 zQ?0d_lu(%ET~cM#xh{X*bRZ#xH9@=rWNl|n@Y7ujdsuIF^B><% zrMH@pbbnx8`k7b6=B+{2KVw){`)D?xi_ zH@Bay-}Y-qh2xh5HUT1k9q^E12`mY0X3O?UG0ZPq$o=eQnm)RJgPF~7>(=KR*6W)K z?^+-Andr*Q`}I0BpbZC=>C_KaP(S3p#lUUZVN*%h>H|*6yLlyoxeD$PGbrAQ*dM>5 zddh-P^cZRi1gjqT=g+sOcukuIvL1wGMiN2z^YxK;ViTXg0&k?B{v19WQaE+_{8}&v zgHZZ$Sbb_lr24)y4?;(h>PMa=(!GHIjwP5AD_FFv?Ct)Q<`1h2tE5FX9oBW&YM4{@_Ok7 z^S?%2ZHoh*3Jq)_tD`V_#hGD&Lo5AJ6X^^6^Rx_%g&NjAw6NMy`7k!ZPXEISRZ}WewY$*8$RxSKe zm`Qy(tPJo4RT!hI2D_(p0`8eW6Cgn0p0Ud%i9?v@Zb^rXeqLzSBOWMltT=w8bBZcYx=- zJUR2Pfj_OiIcnMTPYIK|v|BmqPbQ?+$CPFcDKer@XD#}2n=#9%-`LH@Jyc@=W6JOh z_y@IkH7Ks=;&uWe&u3y{K_VJQ5K#D&F@0N;!l32}{Q2Owkb+Kja@oM>mrKJ6H1Zr{v$}po`9l zc_cd1n*IB5QG)pW*OQSk_9;k66h%ymY#@^gWEY`()En%u#ufHo{-*8c_Dw=?5kDB{ z`^M>OSRBA-^Z5{d`tuqMP961)Ry9R((|!_oZjxp z>m)K5zW~B2NGQzy&(n6Xc8rv1e#gM7zFu2xF!Z-q-(0=n2#8_(IL`%iY``}5rv(5~ zLaFTfUb@Vp+2PtRI=U=-R^(c}dx6!DdRC#3k9sYNP)md=FnMS>t{dauF6d|d+w1>~ zL5MFfod`@8vz8$aIKfu4GN>Z6aHjilrgQ29H$605uM9$pCY;CheRMO}-1TR)K&-#;fiN7}I$NOtk#U%q#;u6m|W1Ksxj~#8U5Ri_oZ)KtI4GC2O z-!b;S=B;V7gX(EFx{79{i%iFF*b`gL#jpF-Wn8n)pB(1a?9|4ZJ!LJcfE8X@InFB= zUk==lmgs@D8_5vhk*NZ8 zSk=@}C8%L{E_F;LZmcEA;|I^vix}!hCneO1lWqf$+h}m2Tcx?LL$0~(h%st=!*Su; z7eCBvJ{u=|?jTER-;hqNiUBeUy^6$eRg`o8)W2iDFx$n(70;wI`m~jsm->KXpq=2) zR)V19tT8R5Ul1TO>pxKF{S6MQc7y6FDB&-#MO!UG&U zN{@z8*hXJQ60j!YF>3S!^<6|3ZxxcSP=hiCmV|_kN`Jk@iwZ3R9nC=8pkU)b56#9Y z)EBgcAi5RtO(*5aY7=S0`XEpLla5!Q(`V!iW|&h|A|+9^VQhAj0s5F|3zuCt7v36W+vshd))?ap+u z$aG7VPi5;!@L1Z4CKra@n$cxR!k`fGw#NU!^{#=t?r>bQe^tipZu`?RDU&8?A>VP} zsm$B`-kce3{oYhp`=3|~=Eg$_>lUpOG8zZ9!1T-*ms^=0FVq zpv%PpDZq5Dtv3Lw^j!Zp^%N`T`~co*X||OAJNFcHDAuYLDt5j?_`fsHRUFcqQ|A>h znLSJN$I4o?8fPn#)MSc|Ar$Z1V*h!^c-jvy`?zLmq@^Vlm}AQQV2X3u~akNsC|yP2z_ z&3_o~P&_YhntEYLOMx{Y&^rU(2w?7=`kTUCg!1CE+Pi>t34}9Vpd`<0Hz(AVi^3Vp zowkbaIbX+VH7Rimhq2aLj5TZmAsB-z&TXzuFQ9b<*@YzR>0)onY*j@3UH~!vBq5xX z-(73n06?IXO0IcoYisS+f#^-x3VYb*0OWQ=37KLTNbNM_aXc>U6nvB1Eq}kx(ioi*Vb>zlgGAC}wFHbV zL(L5ra!_0=rAi|CrbVg!^*M7v(goc2?~|7z$(@e&=qk7ANl`zqznx2SRFS+nUWqJ2 zS9>xEqrqrAA;-2~*81f?F5xiu#;*2C7v&~lKq*t-uU$N^xGd1{{qBdZ#q|NaJG(@^ zl`_4o0w-4dw`Vo0>B2t5)2%5s9tp69-vR)w!U|FQraifU>w}98zx(n*>1#m`63UO6 zTxrZ^AfsB=OL)|r2t<0i5&(8BRE{5a6M5upJV!hq=vd%JyJ+OiO1Foh8~^s&({}CqDo*>z~^Y7Bs=B;^vlQfOZ2; zHQmqP0)y9>)qy_P9W4?!OoP4Qk7ZvR@T;1sa1CJdwS5bfxAk0GKmPHE zSynq;P%piLBN(4BPQQ4AOe`ZT_MlI+Y1mNN%Cw-qj310L$#~A~Bml2A5Cj#>s3o9) zjOJLN0;l$>uWaof5s2Ko)+Ih`a;zIjY@;KN=SmF!EZNh zIe`St^Qh#%aWB-0Qy~h^3ha(06I=kvG*02Yh~-K9+cP)NFk2`GaV;;6_3oK01HGjB z>Z;MWM2NaLB7m7REQWw=!LY7AoWV1ej{MDUwor+j-&nXd?&H!zMfPWnlL!LErRM$E zNTCGrUO|TTa=m2Lrc2qoNooM;r~b5&VR- z8O$|FbfU<(?iS&Gva7ZVX0k`iMp?gq=1zTjk0p!4%@3jZm-|YGIVQ1owugbji42)a zNxi)ilxYK^meA%b!?TJbBHR3vD_6Ue{`7%R8p+OZkZg0Mia$BoI!lTktMFyBhUwN| zR0wiSmL1e?Maf%!c1xnno#$O-8JM$WR1Zl7yYgEgS`?Q_F9s36dzo(`K<+QI+b=Z@ zBh&P7VaDiPE)_ZD0xBU(5lp~-54W=~H6h+K zRsw_MZJR$T@2k(BQXTGn-G#^t*uyed&o(T`=Y6HBf41s=yf7@FSnc<^vojphyj+y{ zRkz@MSturBOXdk5*W7#OI+j8YFTPX*d>0G}3i-vDUco7(&f}kC6;G~k1no+omUr{} zryR?f!gl_$1C{S0z{lG3+oh?m#{_@&&6lM>Ow6V22@a{jFI_f@0#(|ymjV-6j^yJC z_GI*z6(shwWo^c>j9#qB!Yv7|kxFT-Dw}`Oa<@uJ41T^Oi5>wRa9^cTUmY58H2AG& zA36OGPsyQ<%jp*UtVZseVATZN<=RDJv@LT?N(f~5krKi+KLJM;kT7HP&A;`Y$lUa5 zcHt3z>9-(|Jk9CnE$Jma9-dd@N<`DIVEUS)2C;WTU|?KR znTNm{a|O1xj+j2F`Jp5JiX^mG)fx>m_8*hx%D#?(SER;$Bx4aae{D%@bS=-mMdFz>j=onqQP{bgYoSL( zro^hi`{vAsPTEFJ$(kTfq_@cIx4#N`h9C^t;UWM8;d!8ZH8RlogxZGf#~8AJY8Q6$sjtuBq@9`ri;M?u33jP@Q}_~>jWcvQj&kBB zk&9uv)4OWFZB=mo=ec+`IH7SwW3#KE(jc!*ze4$0>^rOgW-`5;(Vo{(2q)W^S5ybB zzhS}sfOpC;D*WUF;ABxz#9V{cv*C)1K#WGd=|QYCml{SO-`u5$;6d;Wigq77Q6-aW z^a#TCKn5m};P2l-zv*uavk}kMDpm>L3gx9CWV>IY16e+Ymn=9g7@C@))9eP1ztb^A zp~M>4Vh)-^_WyFBL1Aq<B@aBcsG z4Wzut_Ll~c0ws(YYKPxve%9m~C170Eu;VEXtNTwAO-{or1N`OpfUs+2CFm z+T9;`0Qjcxp`1S)$dm=hdrZD~VX^2}J5l4oC2>Dx7ixNL^Az_QkFh`Na#2kEkgF(L zph-QddAMkPd5Sm>KzoOb{n|h=K#K zWTE;s=RI_$GYLxPYW#ZOtew2$=NYk@62U5Ts*1Z#&;GNcmNz#Oxr*|)M|9cBc`hKk zK>z+UodAfwNQ+Cw5&#zDdb7ftg*F=Am>)fli$?#lMI+o1W6Cx(sHz*!|BGtTbi?_f8Sp2;|qi=&hk}UTN z7yok4w2MhRZsrO-?EKU^wB$8|d{cK&EyI6QM)$8OA>ZmsiQ(FJ~KqwK(o{nCz5K<+fYj2Q6R%y1MNr6m8Savfy zNv8iiKS4620(*K^RSUbu2+5z3ai@Ca-u7rQe_hBZRT%nE6)y+x)n}16%g+5s-&dL$*0A+RFlEj&8Wtc%4h^l+g$*fmdhOctOKCJ|n==E-MpE1aZu<|!a1U7tSw zdC>aAS~cz8?iVQ8eMVPCJbdzlG46gLc^9e6IsPj_iiLQuhrjb^jfpbkk=0oQ1oo;~zh9 z+zX*meQ#oGZQ4%%WD*iZ%0JtJ2EIE6n=Li{rsWZ+JSJd96u9;b@4!556K*H2c>(r6 zRjm=~lf>bo66t7j3CtrUaW70;e{WCuv^UWNCdb+VMj=-bgmq|v0?B_()Pa~naX=fY zdPUpqeyaME9}}%Wu>YCO#>0zf{(zKUX;EG&c{=NqhsA9L$?B-gV zt~g$dJW|FNvoR@4OB{s!9rRQKB*(}lU3qioX@gBR-C#w~Sxmu;IgMwybQ!k;@44?f zSKV!IvPWva(@U@h=-M@2jQ7yPjwbxn+4A9A#;(ToUAbl@MV~>=(4N#=M1fzUdNf4J zt?ovXrm6pELcedY4|vCXB5+Jpa2k;OY(@v=JINJYBB)C62u-M%LJk1@VwDP@>qR%y zez|1;C~Pqv0AHExO^O*`o}>;0g@LpZOx;P13!`Lo+#Q~hhut_U0f+rVEP@ZR=()WB zRL5zQvkVS)xkWP`7pgSVTOqnB)8N;J9wT-nU26BuB@Gp>82j%G5AjdiD66nI93eYD z<_mg-=Ec6}olJ1kbozCVNyzp$yhlW+)1zd>! z#fJYrj}rG{K!Is-pN(Dpo1O|8O2ktf5X4-0G7oLFNWNU$mosyDMBCs+AxmeFwK)@V z*_P)8q6%m8&En5-aNB_IQDcB&Kxl68Kd1Ct*NH3Ola~3?zZzd+F=*B54AlKh!DF|u z$$gZL>Ml`xZet=NAsWPf|F1D2h0oN&;x2|kh6s49I&vm}aa5BSEIKu5c<|Qaw@8+V z&}acvw;KOFL>79vtK@j?oktyw1_AYM`D3<=C8i*4%x|3-2W_+%Y$`kHx_P5%`er-m zv5UT+?$j0e)EUykir#`aVbu$N?6%lzLYrz)3G$ph64r6OhU<)!rV?~<9|$y%E_a9q~c)&N*w}vKJB?qfU*$Ot-0tQ z*q%;_USVT*(LW9Aw5_KZspa5E17~9-ukxTp*`WrCwesoOn)eI%yPO9{F{O5O2MT#h znEab%@jEDnI@>+!`Kmm{E_Q6?y3(bx6mi*c(q9&v&3*Q$wwQOdTHQVty%%IvNw4|^ z6giB;L>6uwjn@UKBQE69CS#K>uFi1a`dwskUCey9k7-iWhxHY0^9*Uk@aAEU`C{u( zwWN(ZsV^eEYDTFM>|YRmO1h(muRg5z>c!v||M2;#WbKzGU^xGgK6?1t3;fG8`i&;v zj8C0X+>k$D$e%w-d2I$ppRGcBx0!V`Fkmr6 zEg7-%cvX6Mz{^)U38QkAK=?t780pBBld_J-Q1|&HD|@X|0g#|=n?caG1a)S9^`M|w zDZouS0%NqDpxPZbzSYZ)r8vAVRV(haUrON5NM{|x-!{fbu+{zByY$Uo*O=^tu|Vd| zr9Ii&jz45rlhAx0{D^MTZZIErk&J7Q3}IDBx}>#%OpS7jqWyVA-p?%9guIsXCU?!z zy}66s%w$}_w2rf4_{S*NHGWo6Zk@9*ezKcSanH;|ezgYqa42FfL>MWYQ*3ES)JfaeW zr@lVczRuxdjxz5Xb@Bfjg#Z9nr7G-Zd4I^|4z64&@Ty{FvXD*d3*~f$@iA$Oza+l( zDOjdkocWk~6OtMt}|YLv%y7c?QS@LpHxSWcp@Vn+sJqf1TxvJ~Eq`xxSe zgR7cUMIa8^qrnTHk^thp%}fy{L)~+o$G>;dvlL2)PI>Xii4>sAkUc(LBu{01kpS$mhb5t+A?Jt> zA_Y2n;^1Rs{S3XJUx_(z5!G38cf=&c0a)$h024sAOr@<1Hf@Kn~T^ceh7pf0#Pitcj zed**{yiN`-KG#ZoE8@uq4xM|^BEgM@KQ(|rFXO<_EW|NajsI6&u5rpNY5r`n#g`2i zmZ$_d%;(h-EmY$rgMQj>ze{NPEVAP`nmJ?OYlGr=EhtmYd*s`5{qamxBhmAU zgnj`e9d)P!W3<=cJl$sf#}jC@syR-i`2#qr&DZiS;M0>fUF^tWg3XR!_@^ZL*K033 zyZknp}haP+eK)p zzxSaGn>x1&AM!!kr(=H8Gltb!up{YsNCLH>wo6T44M0a+B=={jCNZb#7p-3BwgOqe z{u&@?7Yjg%P)*)&b3Ujg6vSC<&IOYYiq7*LqdCUKaKcPDFH2Y70zXol9=6ZT%dIm@ z*^f=M(&?ZH?+I>yw!A-8ae4X)PinpEsQIwjp%)S?@`a$)GqkL-VI+OP(ZTOdG69 z)!pnWm(GK=!xy}@UD8lre-|s~4BqJI?ayqy|0)&jqgmyzIRXyOyLfQfX7_#K=zt1b z9fh4Un0Q<~Z^|)!hnU)Q>g*;aQUM@;5rz)B5=Byg7iOWbtJfS}hE^4%yW(kh9Y468 zuk>h12GYy&KDYR?9BbeKuF16!p0{r`C%xY$iA0}JfBDyP3DRZM0`F=X8u z@s8we0a~-fcUnnS!z?Hq#+hL^KN`^XU&eH^oiaY3k+nUJ7PcBu<^H1LZ_n8|e{Ha+ z%jiC07`ut5mi|LSf8yB!e7G+fU$+jtqZN^hkx#|i#Hg@b_#QTKFXR+1#|eT*u|CtI z{AZ&aJb-w7cUY}}FmSZkCK@ByKP{T|Mh@9-L^*s~d48W@AmHw^TFg4sWw^a|67~*i zlvY-V^H}djgfW}6naL~F-Ypw`^nw!@={LETA2Z7PE@QMDi=I#T>COe0*^uD{=tLuA zQVPcdl^kinO*$Q*IS0!nq$!tt65FI>^H&HJaZth3wY=&;usG4PbURs}AlE&j;;BEn z;1Os2aSk6u;#Uq)z8>+q;XS{5%Cy+#&4bLOAEcUL3e4)#es0y#6XhZN9CTDgyb1Nk zJR4=T)5glecz;IebIA0CcrpR5l2k}83{Ur{{ut|1#Z-9adahx$WBa4|;c9Vli?wmd zInwKVglA;Irjj)E4drRJ|6}tFupAv_juW*Yif+6LP@)z)IYSF<+Df5)bB+BJaO0TL zzxuCMWoBn56W`fN-jq0uCv>`0lQl8+y`ng|+~XGEKHGZ0ruZOtr`0O;sbLsr-kaR_ zlgj;P7)UK*_F|nt*Pt!M=598?g$-8eZ3CQc4Tpm1MlgW^TWMKMgg)cgB>ZX%foPCI}5){bAri&fGR@Qgyivn)c$ zYdG7+pv%acNR6w0FnllfkVZ%g1hL(ESeu4;&mIsIK3az8pM|77og;GC zt(Elp2u)$#O4uG;mL48RV;ops8Rj=fYbHnLlRDxE4?zhzfpHnQd5a05<^uBrkn+pb zTqDz36xupeIlK%xOj23rEh^-t zd(~u#h!=oEh43jf3J%O|IA8w6)W{11dlEv-G{MI$r3y5g?LZm9jLI+$ucl|ptI0*M zhXdUG&?WhZ@}-b(kG%Gy@;OL%tVZ};?!048J%c;?_CU@>{$Q`5haGyA&mqiZ{%^Y% zgFN$6GF^;`fV#ao`AO0&LKFkW8X(@%JY6(?sXb36(#M^zbfUwFQ*M|>zxZb==y{n6 z#VU@+*yN*X%A@(VmkEc4`>z25y!Bv3rRU`I>)5|U7Qc+&3g0_&IIfDE18bSmfU3h~ z3_wVc;sjRym7(%1SW=r7?%oA4u5=5w|U(YI_63jNDwm{}jXn7s5P>sLteI6cNWE z`rESVTl;|oqE73(E`se^LPHX2a-W1qrO14|Hc!!9`x%C1(1K4+fDfBsiUv>`W|XcW z^KvT43QsX-h(kfhTGcBAa`+P^MdjMqC2`Kse|bF1B_8g4C*(|jf)?FXTL|4hxbQfTM5U~9<*KC>MpRouI`P137mTskUx1ynhy*U z+mZAt;!cgk&D41zC_x9u;J7weSw52#tYJ;r6Nxo3piO3{$++ZJf*OITw+uGAUcJoV zboWr9Gy$&~r^0x~y}N#{knCEC?Uas#3@eyCr?9?Lu|iD36)_`Tv=(-eC6Z2~wL#J& z4QrF;H}mJWnoF3db8v`tu7JWg_^p)`p>e^Rm;PEusyFZk6!N=Uf!aR4AN-p?JH=LD; z<=;QecW(ka_!=|PWULvF?)nps6W)_fK1$CCZG&0OQ};9tjdL|74H9t1W!?qo1d0TX z(VA}|XHGi)YQ(7gAvpj-CNNN&?DJL?-6&{ad+=sqW6W;OO?w(GEg0C|C{Lk0+Q5bv zom{2?UjyCZKxTrB4Orq=)}U5ExFWqB{7F-bwkIn7Zk`M7RpF8*I!uSrb|LcFX>xq^ zJiH)hZi3tI3_qjJ;mPuwP&*781hqZUDL8nP&e$&1gdh>n2@H^b?p6rAJ|#zvtPAvr z!n#?N@!l*Y@-s;kF8?~05K?y|qmpI3halG}3l_JwV;K9Q`$(Wsu;ueKr$k2t<7X$b zYFhl7cK${1>zP3g4IO__G$};P=q<%@zq$v+dh{~v4pZ8|RLF9n$B&AAJP`V9Isc$W z=x7B>|3-lA0h#luo4q6soF=_d)f~LRk3#HuGBUJj#%b?hI{p0QvK$``ttiQ@y8f~T zeNCg9LT$+XYpde*sk$T+ljbKgucBz2lJ5JfZ>@Nd7U+2A4duy z6sh4NvCb*fT7zJjmrjK=&}*7Ett&J@Js@j%H}hB#o9xR=$|H<#c8p^m%PzD7u#JkN zYE7&H#96f+fqO5C+atEj^%#7a zl&SY7Pn7BSM$3js=4-OmD+Jyl+$v{Od=F-&#M?kZ-qo2tN>rnFyE+{QOUt(OULF%4 zH(XEww~lCL^v+x1uHx#w*jO!*9Y0C`U1PQvILM1X`1EMfL|1`Fchqo9Z{AGqeCdsD zD&rl@HpJE|^|J84-XkBRD*F~g?e$FLHqMt$B|l-*1b8$6_M7KB-n8Mi68;CVGJp}$ z4Kn~GGo_p>P2I`w2Qn+)AKRKK)}a_cgtk!Zy#;$PyilN7cZ0 zX85Gx_oMj?*9gjDGN|zE_4L!ye@RhrVqicj;52 zeV}&S*F=dS&$DHONcnRb8Tnl?Qy{PJIo@#59pq9c>weq!z;l{kDT9PWfepk;Y__kE zZSL)Q-6h2T5SudmPuP&X)b;bM?GsdC8-HpgO81nFqXhoJgsi0_^`*3y#}y`3C6XUK znYYG!qyh!@%4Mg}W>+Z}7n@u`+xO+dx68J|8&2o3`ivws02?DrK=f(RKNy84mQ&h_Nwm0Vv!e!0NQU9VLlblK*%Y}5aLJ1#bWWL;JWj9X?-fy`JR$uO{CY+2pQ&QxN9^{+#~XL32t5UD{1Zq zM(q!`8EcG&RzU>yUI8mll9eBfM}F~6-Hz=@+HsF;E)&tfWaBDD8%rNQ&(~#bF?*eE z{z*Pa4IOJ6!ALvLX+BO;9oCP|;v#uaS!tavY6OnHbviDjCgws$=`iQ-ynA!z=I5m= z#oqI7H*V2HO%PB znG$oSn@TZfSuG0~`VTnA8g9Qr*=ZkOslW8oH%ehU1zJ6KN{NQld`bInDFysUThx%( zf?{z>=UTT^XnHy3ScSSFOCA*jtk?4=Gm$8747hRq#52`X4Z|PJjw|lbwQG^=^Wtz>xE(+(L z?h5S=TU~D*pY)Gz(z^uNIX$?hx@%+FERWgfo~pBm_*=3qwBZepm&sii3;SjpZi(im zD-FqI;yS@iF`y`l5YH9kbH5Z{GsP93ffy6+&64)74+|N_?Y`Cvkb^gE$*-o`V{aq1O%?0fnl*fR{VXRvr}7%3v7D&(nH8062AQ^w~Hy(o81 zyuqaHNb5V6eYi?1E#rJhZa&KNT$_RW5!u7gpQN)Fnc4+(*M(eGAO<+?jiH3BTz5Tf z(v*2woJ`oqx&slqS?Y)b2526W|o3}PN$-ipNt=heXW+0Z`UNv*c4nDYU5xW{&!EDh^Of%{{t-POj*LLFuXc{?8C#i3! zGags@xXokWC~SKtOh5XI=G6m{rc8oMY*-x(eNKbn;qAxU!49n+=-!@zAI`#bVQKo- z)N=Am&a2P_8>wS?2+;o9bvtw)E({ClNqMDsJ)7w*tc`XEtFFT7emfl&+)MveQ9T5xrQVhRCIo9Nps!LA zsr_jfQ(=l#gA*$4R@zguBA7UGi7Uk8%g@ab_;fq!)83Rbe=`Rk2KzVg9QoU%Q+6-~ z8kgL_xwzTIWBAE+y&6R7zEx<4-k%O7w4t}Yoz)#Xg!EgNmy*f4GIwtHUCd7Myf^L@ z+>4*?Gt^J0$nXAVkFup*)CUPqBEBivH+t&Hly;}{SzW56^2HTE=U&TxOtYAWC!0p; z+~*U`8A;TCKCujk;|$&Qn~`LE4FvJ7sb}io96{Y=J_b#t*2~|+s5Rr=d72C#t*!AZ zS0S7NRo2a6rzU&dDW?Rktmlpc;Tz^^{yRCs_cXDOI(}^AsP6AWe(|kEM7FnbfuP^S znumo@n)(9TM~5wMBOjKG0m8K8{%(p0OEI?GgWY1xtNRznsmJ+IwmJu)r~OvuaYIGM zUJ1oz`w#4sa=WLtjkAKDoW69?+{W@%d8^6)V3d#KNc%`$*}%e&=9}#d=KZ_F+E{HT z$yntSk&t`N8BWhB-Q%;~uB-DXt!Z*U-x9A2duj)=|Jwx+y~lmUMLf*qFm_c|T#ygR zG73@8FKKI>&R_Eg;hKB}t824@*v~|ksAHnvV@|B^Xtzl-qv^!xWv#|THei{SEFilE zni(IH$BTG%n+YcEtFWhkV$pHsTF@wiYE>HZLZ+$V1!e9v?6qG6kR_n0uJ*I^ym1oSxO8)nzKSmEJPn@BUF^ZJ z5wUH3M%qDsZVr!2l1AIjHVL@aN{H3_(|3n|Y`!V4P|Cmxcb&wr^x<@_p^$j|SMysM zgER`C^8)G{tqdO%Nj0X9Ipxyp1xmlfoJBb(e^Fi7FKnQSGe}g&q5|eI$TL8y{s>H? zJOl=w#)4U_1^^W^)wND(BBzmNtz;T8YSg;p zm2aovk-olda>n`x>Pi|W%SG8YDwB~!X?7JBPt>k5#b(u79U1oO>{V&* zEb`dkE$DeQrVC$roapA{TVtQHxy(sEtjVxtd73rpCl6`{*yUu`kk8?*BXdO8TZt(5 zwI(dAQTf>YET+uaoT$d5bWuQ^ycW~caaDvcvy#b;M9v_Gg0G6O8%cnU67;!1(#9%< z{t-yN>mEo0YwIHn&P*UQ6imF;4jatc?W1S0QxGNInB7Cs_Qm|!{gJ`d{L;Y4tg>lr zHR|JH@xd#Zrk>aQ{In1iBx@eTKRX{%w4jvdaWGDmN&~?IW=+C&a!9llyMKlLI*-F9 zi#7=}+M0X@3J!vm{Tjq%f@##b`g_zjC%P!HJW4yl)j0IN?|Bx>IFX45i{}>{rkT- z!YA9IEGc{zJxdFOD!zQYGJn?=S7NK6J?{eTG$a;GP%4yu*Bc-jvxD!7tD;Y^om4 zx-f+a{ivFr=j(?7h-sg>d8J`N+D^KLZw#ygdN5z<z?}Sqp{I*;{K3 zQoiBB5bgLgMlfXDmE20LQuWSCL>BA%p`^MXym5%5v)2jBqs=k@|5y`*h+LO%ziD|h zT>6#M!F-1OCi7&?0aU9Ys_p(vj>B~~L1}?EAHR)0p0pr$7kh~mv$loC zHqcuAU_;}hN1#sdO)afe{=N+dZDmTJ5U{vms?xB%tiveb!qQ74N4@_{{zrQY^Ls=V zr|da1DF*O_ARPP|@M|vIvmg!NA=Y#&T0Y^lOvvILSN|gfDTpkh*oA zh1JP@IDyN;lZLIo>AHLo?uBvw30+sHR?(S~sMI+q{X%P_V z4hdh^>8))KZlG(5 zb27!*Nj3wOhY@9vPx()~XPn=EUbhDHsFiPwRUIiE|2>+kvIR^lkaNb;b#6NeADgeL zpBYbFnCxxYiyp=DYD9bikZ#N%CP%D4GK`O&W`sjNib1`TTTB>&xF{k zD13Ty9InMYC`PJ3iH-i(OEL;|`rO^^hs*1XG^~3845HWlIE+WB&@h3~x6-lnxR;c2 zl41J+*r{L8K*MaivZ9h46&8Y$jrQObx7WN_XFZreYa3i+cE2IZbU%K=gcienocPhN z{eGaJ3_zY`V+ofGkE)FR(wjqQK}}Pmd_63-J6qw-w$o4w>#S?~?CFOs!!Tc3fJ;8nucY=zspVlT-~7TO@`~l3 zD-+h=jX^Vyl3TEShB^f5%B)5M>NHWz*{FNqp#1JGnRzx+W^rm;wtH|uO2zY383CFU z?>8lFdWt3s!wl_wwBh@YK5;_zmz3}GJxQUb8AHPn?I9U~J|Gc+S{4fe3QC})qbryi zl@BvO_$O_o2&7q1Ui*KuhG6nu*f+F+kIW1{c75=%qxzx23X;=tdm0`4>=Us@ zGWO{&VC7%7{M-br)(hBbUClpL8TNaM+VaVi zN7s!pV_n&}J@#Z)rB9F$9RmxWL6buInIk0-1P^qs1p4U&_C1 zRt?a+-;HlD`(c3_%r#tow?QotE~!QzHeneWw3&<(!)l(XA8+fUpBg|rT+b$Vha)R} z0VU-7K&1oF>rbH$OO&2c)k)2lwG-z48m-Aoz>o&(tx&#zvqLz z%)@a+IiEzm#y$bYES{#e2~DS8BfvfQ?Urty_6Of;nNG8gz4HX1L)CktbTS@8>dWzd zhFl_?YKR$Yw@YTX)^Ae!ixU{jd?gV+V?UPj#Al7BFI_{jLA**6@QNv}GnRz#l>@l_;9)S z*?Sz1zCeen%ZoO_M7?ObgQsY(B>y%J|6bTZ=$Ar(Ev+Vy_{{M3pv;Zf=%NB#} zEX)6V9K8PP>oB4mCZqrONd-s1&*{riYV$VqKOdI?=2E7i!Ir|E8*yS|6mBED+W>7J zx3@J3t>A+!gAAta(8uaLHaRZgK6|Cu3nVD*Pr0Fqxus1EJDZi(k=>za2NunTEHNX!E0n&SXC?Q@EJdcGrui zw9$>{e2?Rsh8?s}Igrv;UWJp`e$??0ppwPivxwWu}9xp|cB z)c(#>DxVxyw>ATJc23$wuqXK;seD+t=kGr)4CMgtT0p)HebMVml_~`pj zZZjV+JvsuE<5n&1dr5?*P=5{uUnY$_`@Db~+xj84s(BL^Mh)R*?_xuc6ET`;lV=+z zeyZg5vvF=gQR?Bg-sfV*z@ep&a{zQaU4GYF>ES;M^j;wc)%faEPp1@9xgD6XdPMHm zX3bb8GpW`2+W%VZsvLMbv#LVi=r9A!*rGLqUGQhND@@)Osw0Gf> zQIvQ&K_k|g@H$L{&sk(MLAGv?eJza-3z`Z9gO2s}AgtZ0dStOc+|=(x{!fh5V355* zUUTMYXb}7sEKaZ*w`kw8a0*6Zx^=>#WT@wgT9p<$t+ZR;UCe#HtRM#=S|j2sw&l5e z2o%lA5(^--drLcB@Qt+Y7ghz=2}{77dok^Te++Zft1?NHPSaKnywZ<<-?aW#Ki=bT zo>%8Sr-EuYxR{RXRTa2mnTGBU2&jShEBis<4$Xa2mkEb}oMDx2GTV;=>3{p9Qjh(0 z>;V)1?SQNBt*PZw7mygZ2o-kiwuPg7+FhS zHwGb5o>Ac(hOAa(zvnG+T3>9#W^`2KelNO>tIpDt9I3)QeqOylEBOe;*P~WR%>&{| z(m~Q-y+l!VnI)ixaN^hh{juYIVZ!$9YnSqYNjh9Tr?-jt`*NT_wV$XQ(2c)7fa>~l zC`$BmRP6HxvZmdORxT6J3zkm)J-oiS)&?}cN^v8f*nKxlqk(7bt&3`;^Z`p{6eP!~ z@(!*urW-$x6In-%5_Nm=Q(~Cq#gdo`V`l{zG$Iua6Ddr5?qRJac<^9Y$8hjLB^CJ; z?&RfUNfc$LpyT4asmlF)W`^t2O$t}NfMFcEcVd-f8k+=1 z>Kdb|>AGhW&BYry%!5VJ-Ws9~@GFr%QX_>ns3WFUEfZjbW=&hR1$OZbfWPO&TOS5q z)|XC_Ymb8!&N@}>tNOHrFHY;CEaM5wr@sqv61YC*XG;3FVEF)dI?r5VyrQq@_;!pZ z*ra?62kpi+a$%(m;I{u+-=f*9g5K`7R)8YlYV9`)>(-)!NL%mM)ip4SQj>Ni)5p1H z?oTU8%F_lGQJF@*UcoodFbOJh}y(y;Uaao09DPJ|AwNZ_g*1 zWfpNHugr~5?yf^efbrxCmrpnPN9{Azri<`P5$Lgz{X1KUPrn!J=9;K1o1x9SybGM& zwK?ZdO@C0K9L^eopjds0*&4p+;x-*D$ct;#7oq8lZ%>}c5yNpb%^ADJn%GIYtjqn0 zB*wdrG7poyJD_f%soXfst?NrYpf_LNSJT#hG)tzhgF|Byc)BzVQW72op$@L zZwB=<1-#7c^`_7c1zz*G50TFfa%ipz_V+sDfJFm%oWK3#|CZFSPhP{r4IZ<0IJHE0 zO8X1nGQ15aZ@R~@Q~mO9VUa9Gt@X`MO_Gr?&K_?4&-yR+$?UVtz?ZVnBkIPdYFWJ0 z2EUPp5Z*&AzP4)z6bcqXKkWH?F89JFq8X`h`B!`*+GH1r0Z0>YcLB`gvYoPY)SoaC zPK5nVaq42E{kAe@X5Vh~%K%b0@y^G8syPh*D)`O2AQpJSxHtQ?=-kq3?VtxBnXxRb zLWJ3mkn+8Vw4JTaT}T*J-uk}Xd`D&UOsR4p?RlZiTj^%H z0$+}n7i@a*T|EXS9(pQ^Ydg1 z?t8A@yIbTc7Ab-6L914(?}gVseb{*9T&JE~!>$1|4Sy?0aTG~-Y!sH$@PFsZPq zi5-t-zr<{rI3hnuvVkb3h0{Q1O}bn;XYCXu8s$mWo(4P6FUaKkbj+3t|8IF8uzwbib0JtTYYO?*V6O8lEqG zVN9-I?3-QoK+(Cs*3pUOq?cpnz|>hE|fY*5$EN)4sRG z$h_LP>Epzo^Oy9|?f^Yof)ps%D=jO0t3{c774cYXzyi|a&RzI0pq+flfG8OCV_)9< zH1LGKpQeZUqaRWFchnsto`QM(jsy_~ zlSc{av>;;AZW)7Ph_kK)QRvgXeqr)p-+PGb#p?~o0i0v~Tg`)o9ej(qbf zd9Kik9+)-WNh?0RuQA!LBf;U zWe(hjH914%U@2^wNM?zGH9Qy#zI~A-bx3}B)5MdUvFn3Xu8j51%V7CyZtBEXl*wT2 zm%*c|@Ja55Y76I8Qm|#u1&+nr=#SyBw3a?aun{M`T7-o#BWRBb{BQ?L%UvZjl9K%{!uD$! zUIGNF;D{zcMPti150Q1oQh9klWCuGaStB#jqs}nZ?>le&19) zduWEsQFNGOoRP=h-Kj{Eo;P$MuKZRno)Mz5#0D0OF{10O=<9KTl(UoiG#YI%~`Z-$)7$NLOigN7NG$<1aAJ1K3_dMiqFed82j|T=!*E z=m)@N`w=Ccx;wT0{KPMISR+d(buF?C@&re~Sne5G(cx{HN-hgB&oJlboIT*G!W%rx zGmIF7B{bi8`_n0q|KfnTyaooNgHRaOh0~n!_$qTr5O(ck>buA22KtmNU_@lq z{pGN3`0!51Vd2HmUesR*#RDq{gpXu_CT1LF-~>xy-3Iw2JisQEg%uNohf+Ilqi*l9g*qFVUVJn24f5n))Au+=QX zEwtk22GU0_=S5Gz+q3w`M0z;tNBD$<0d)6yYX*ep){QO!Z%W7En0u*RMN`M-Euyph zhud=dvjVg#8FS$E1d@t?pVdo8;4oSbd>OOBd$vPg{`vV?Sdq%-McGlf56BBDeLn@W z=2eHm-)Erj9H(#{q_r?*+YXRV@=`DY6{BEG4Xb@3F#G)RXVP%cUjnQEa2(QcMRs!B zFu^cHo=?9q=(x({qlvb=F}MZTt7~MwnqLh_-CNL%jAjdS8|FapzJMagdaXOm8nk0$ z`-gyn^8aBb4y2-xF+1IW*N zSiAVs3DkO@Zb509I`YLV@NVYz!0dq!NM)6qtszs5h(SQ2y>R;aa>;Ad#B0e4)W^ah zV@>CiDj4Bf<9QMexyznE92_!0z@!kdiF+btClN;y8G6O~TpKt9N85>tRJ;N&ItpU0 z3ae|RB|to?^~rkQHt1$Qr*4C4ktf^XQ)Z@^*N3iG;|)iZRUck~y4dQh69Fe^G^a)sXv&bJ+4H2&#jam!|6T!7xqOfNS+wHt=|!aRR2xyzW`5 zPl|`bF-e~{T>{afXUT1AbbSoOuTZtaI8dt&#NpnN2mP5m*3C$ft?MxW9>Z{2d~29m0FLa$E+8_ zkvLE2a%Jap-__)Q{o7tl@C1*x7nqsX(_NL7ku@C3P3hhonW{7)PH5lo`;urft->y z^51M^xuw86{!%ILlMpQe*<$n9on}G*I^M3*j}Q>kyf~_H8)10Ey%IT*%{duzlyWi? zX;})b_^xCOjeB0g4Gi*lcK8ttkWT?|^nvI5##uhF=kqMCkq~hxhkbvIOFb-N!=pY9 zK@ZKOrc)d<978C)x&L!K3YxfAoNJ-fr8b7kuCV4Gtl%+oLMBX1F6GTA=DeCsN>?nO zok(ZuC%|bJrgwC}5{Fh~rbQge5}9;S^P|Zu13PkCYT4!D)75Y-ogI#R(cOX&jx;cO z@(F?Yh98lRYwH}Gs^8tJ(*W?&XyGrurShcV)yxprKrlz5*^OErZ*k9;T+fAfWp(LM z>7s)Lm>66kfXs|V_dHfyqm5ZUdslF!`3CMBwH))Q)JFLnm#NKtxTBaoDj z0Yi3#M4P=RbI5nzD+IeiQ6k4kv3fjAR1*uPU}E3QyRtlBr1#z5m}qBVE?u5ClnFi; z;kqAw<(&?*SSx~Aw9W%6!M)I|(M?|~D@SHGO;AA57c-9;Jt5P5e!C@;E}8K|1_W`$ zu?RDFz2?tWlSbdty!fOeLI@+%iH+?ri5{2vYKg0snoCIIMe(2)%SzXXe0#^^&rUC> z@RG$@;iu}mq9)wxODOCA?%Cam8VZOUueNkeQ^3RBCBHHeK8Ip0->{&lW?Fk280eg( z=Fdc(eQkI|XAuBqR6N670W9%bUg==s5+yOXfqr+z*q6wTVrsEhB`@-^e=LFQrg!?g;yHt(ak$2nK)ungt>+PAgPGD0)*xV zy%j`izg@Ts6TKT>7g=oNnK(d^ew#Mm+M1bhG2-Q5bNpm-7(=qD*64~qI z3#;Lx+WS@=H-2?XX7|+w0J=-PchQseyI$S&i+)JMe4Dwp9+PZo*4=%^oHc7KjY>qo@lNU8-A0yZsxO-rMd*L;OVU2lFKN^OT}|OuA!|h6GS52-o{X-g8&n z*C>!$^ta-d;tzeqcZp55w!Y6`?49T=NP=4xJ8rYYs=ON4d!Si{4xMPUZiP)ZA?qWb zlL*%=^qjrKN!2E1^Zg#9i&=I{0%;EBgM6FV`xV94F4x2TPBWveVu(zs`WK6|DY6cT zO#uZb1)lkms6K<~s_4yb@lU)1{O(pU<UgdX}`Tq-18z;-g*j@D&l;~UcTWh z*HBx#P%DAv_EySLo6KcvG#nobv^Y!#`&G1i?=}4OuPV;&6Yh(2xL!9H)?_cPH4;4X zNgv%Hw7V(tpNE%gWh1XE5xAXB8M)d;gMZB+pL205^O55+YWWfkF9QDcHDt%4#0=RqXJhQ{2=tZHbLx$!auf zb49a$jIVyF2rA;IAlr(3wvS(|=S|EOciZ{b=#VB0^Nswj@I%cx=&9!3iaz({DZ8G; zyLaADz7X29k8~&MPjhY*pIxJ(t=!sV3P_n2D7@@w(6}NGH^9V%(1n|!wkk1Bhxsn+ z2Hi0{N<&S-o2sg2@0~Oz58czc-n5jw?qZWg*?vZOil%Y*PsxamIAPT0p8rw$=Eb`h zDUNn}4GNW(IWCzD zY_Uz=guCm_3Mjc*|59VDyL%^t>s1%`TpL78q)I2hN1e7nM29Q}amO^BaXc07XFLjh zTxydXDhst`z;i^{LB;78Q{3D{}Z@&x^SSP%E1qHSAU+hSl zQ}NC5`WfuJ3d({MygE@G?R{VScgOt)wgeBtiD6=%91nN_?vzF7aIGA z5W}a@oEALp;F|h=JMD>7UvOP#t(B}VWEdNeZH@z(bD@MpB7O7;Jy*ji$x zzYYGWc091yGEpL?Et)h>mO|Apgw=ogLXKLU?)!d z#i85pFG;7?ceq0+dz)Ij(b?zU=Y2mFjjhq45VLLmVO5UCJcwgaPe1#4W%h_s=J3rY zP(zY?KKTPozxgyV$%RI$KP|lSByJxW71sdnZq}lJP$k3>L$34p`oP!AeUxV6`F-Rg znDN>M*(D?Vde^UUs@P%9BTWhp)kzwM(|3Y%##1A?Vsl3+85+>iQ08XKlF3#A?jk% zbAN=PuYjFwACJ8y!Gg$Iiv#MN0=4#-O#3X<6023EP|C>z3+GexWneL--vq)T)xmm? z#6p(Y!$e+ZtXl4^!~2X2nh)4cQ)RJxGgaxC{5I+yZAP0qKT4mc?kjQ+A&|b4(0Z}l{*iK zVT<7Cf_8lWNYfxG#Z*Y}%etH+5WM{90foGY!B!gm>9XO5WYv$z>c|b_+u7=3Ti(Dj z+Xj{;qGecHt|tnM>CM@#Z2tZtS2wP5CL7+D9nbbheX1+;XBA~83n4O zEUmW!(*WHc7Y99E;>oHPj30|F0N%sl)MAK4#fS8d@qh7Df2eMpefH@D;HqU$3@th_9R;NiI6xQ68dBA!yQ2NzfU+?7ql>HPd30gW#N~{Y9pcWI7;(Awb zNV=((E9znU#wQ1FR$c7p3o75;LU9?^xH;egk`;KnK^>lpq)hMO2Leun>O!zZ2R~_v zW_R5EblL`BK|LahcE}IVjUSf*ePibDZ4KIzkQ&=`%b@o%<~z8Ck|H+FXqfX z`<~{wjn!`M0@v-YO*=2ZVQ6qTxw~BXsE|-kL3Bm;0XqV!JqB3Np{H>7mkV~>A+_Uz z3!3@w=mEdkcxRAS9!k903}#V{{7B|oL?hah+9~cJQoq5-(Tf}2KmjeQPc!;Pq?nQ< zJTNMcJR-ky{lV<|P@7tGBPs@^EGZZxqX zJDOh&8t0L_tV28BZSn=d6s}!{K~E<_E>2vEpf@N8r_v%O26Vg;pCU`_nZ#9C#%S5jP1l)14@hqBM7QkQ%1Q5$L_~d z2tsLbQ7{jB#4dn3M)e6!dhh}hp5#So7QV#Qjf2(b%Vm=d_TMLAVIb=xRHS>4QF1LN zUXlfOp`HzMr;qU^5TS;X6Q#-0ChLl(przKBbp&`aL}`00wMNmbK)=heCaRNt*rZm) zRalEz?WK&R*y{7(%2~YH{befvE<1kKn<@iwe9iYa2PWl0cbiK~{TU5~G{##Bw+Yxo z@a&Ss`?Rv*Btu5dg^8eRz1RP4Ex&G@;8l|i^3Aa{sQdwSvkm(FM3vVJl=$34$or43 z&QFIZgIQ+DT_TWz&o#Ns6>{H*v|Q7y7Tmhz*JwTKmHmFz_ntigvzAAP`YX;n?r)HF zvvHZ_L8(9XJgD$ib1-07Q=euztix1`Y13#3f5e!b>msYJC?}!DH-nwxz~?JnMxeklE>*bcG1LHT?pv^rPT1pQopg?b!0J8K@ID5)tzO=+-?;m>DZ;ux56SoY$2i zWbhcjq~{y&uG$UnzR=Bp+Q)2l?;e=m^ZxkZp4;tpuE*@xeP(am>7zZJ!OyrV?z}fs zGll1Cuo7EA>M6snQBzmo{TcP^=yqR_*R}nj^$4HYaG-(@G>|{5zhMUO0f+diVK;Z07! z&0boHcBlKTyF0l>1bg260QU&{dzt}vae(t_ZB9}W;3MF2j8C%?AkUf0 zFpmpv$=<(;W^SO(4qJ z%+x@$6HWP|8iXHN198YUKn$ZBZo^a0V7i12%TMbdOrUZ$(*}Pd7jcH2W7Snx31Mpb<>M7a;(}#42qUm2-{=}?|uvkMJeKfHL;Rz0foSZYJS>?H@|G+bJ=_7P# zO~&>FDwvAdxSpD#4@fgeXss0S(^C7K-CEW(75VO(eCCME915sEi3-)^(ORN^W#Y-? z54{l`(omD4f6%6W`FYn7xi_J@C3t|?%+KOV3+2{zBUaejy$P%hQuwIns|bd~_)2C` zj-f2+(FK;fA4;V^d2m;!Dv!Yd|=SQdf8>4EaeF&EHQ!+i*U& z(H}cZcg$A>$Lkf=;tAA6R*sOd;xymzRML`!8}>Ucelq~bXJZk|phbnPhy!I(W|Ihn zIU2&Ks@QaI%MZ$7PbSRCkPk@pHCOy`mN^9~tB4J<>lQv~sQNzpG}iFlpJWcX2LX--M> zeU_-U5-W>O`Z%Oa*&M$EI!LM|OkjSAr4pOU^oYo76ak+tbOCnz?VL}0}`1&LeuO3ciA!39Mu zN9}(c$#q( z6M0PZ=KW!L{XsbpdS5}VYJxBMHtG}?t&;szHI4&IhL?BMoCdGa zp1wK<4MVlUiLaPpt#zQrSANOrkxaIQ&?6*yBf4>@sM6z~DR3WImI{1Hh~l!tMRt0( zJcBW@!>ZTv)pbSdYCK11I4aZ%RzvoKS@uNs*c-`332SWLs2iokA>_O~rd#d(!VvtS z#}`HU`9j|B=-Q&3EpQbVf>P3|yVoV??OU4k^_c0)z&FR!qPSv6Cg24$fS|hF7dN!- zB6k*QN*UPY52f7lp^p(u;{$>u359o`bF`i3lN5!vMKr#4C%QxNf3o*xa3>4yq8#oX zE~4=6TRH>bP_%2;s`MXOlqoMa97_Z#gVm=btrIRq$35<$2l+uV;S=V9_L{W?Pl*zHOFh3k_h3s zGwk(2ReUoHA1OY(jjL!+3dktsT#XKjZ(1{*#?q&yNfEwNr%i7sN@@F{o3hOhSTG457w-UWj(|og6utxjKauz;>q|CF?jTg3bb72XY$Np0ZXs2n8ccnfAYIYH$CJQ=PKEACdYg-S zF};6pR3kA`@Oq(k;T0-9I&-%(l6b}k_b@Fc!VKzlPfTjRUqP|S-?;8+h;M1$>v|@N z4B^5@^F3c$1%wBS>CZznITJE&df!FByNgsVRMFso$$AB+yuodSlyt{6mzAloIHJb?y-TT$#kOaq4 zLU}ia*MoQ0R3+Pe^C&H6f_j46ylrEtsw6lF;Z9AgbVcKFf9&!jf(=6b7w(e7V)8rJ zQ!pL~`UOQx!C-17Vh?qWd+l4l{H_y29K%U;;HA>*-bV~|pip_XQAMI#%CV=j_puw% za!K;fp(V<8k`O6LB(=eKa|qr#E{4S38tGlzdF>U>0Xw7pjUX~v1D2AwKiS&zHY*xs zYN@X5AWxr!tI-{(=II!+s$Y*p?5gkw>AXest(%aVFbTIM`P?ZUEA!PBdQ)U{)vr;J zVyF^WxOP0fISO<#R&uO|ApSD*!jOg9Kuem3frCwptBVYa_i&T+E$M@ zBAJc2lQKQ7>F*+?E>WYbn$Zy;zbu&OqvW>Y`~87}bHR!`W|x)-mxiI@gIu3VdqujV z$*)!%HXBiseTrcd+-m$0-I~yTEcR;CwFvb$aS_<6v(9`k`(7yygih$Yvc4~HuHHcm zWHFkX9BYPJ(=dDl91>GQg(pR*Ub_eFW0xrPik$|Ov`Q^1=t7MEq^2usOdW9}G6aD-U>eQZ zk>rZENxq}It3Qs?4uu%g4}(!fjRy%8L`P3O1vfp1ccHp$h!|z4vrMzIbOdCCVsR6i zcx6K(id}!g7*U~mwBL0F;u5GtRVF)6v8q2q6HK1RZ4u+6eoaK~e;M=6kBxLnf(7d| zOtUkInPZ=>SFr~NpA0_)nT_4HxNaMtjr_WFZHKPj!oos%U^1=|*+Na^7?Ed^fEI^E zc@i5r1rH1JKtwM1eDc>zdq}WHew@tU=L2N;u0M~86=H`Lnbe^2p(Vd!`WZrsV|pSV zTWe~10zpw2KIU5~j>w8zNCvBuwGq@4{sixz+m!b4I~5iNWOgll-7%LY0W)?mmx{SruIo& z8dq3$1L7FnY%49&O{D}2h?MA}_=RfP8WqXYw#fBGW;C32IU}Oj*KRiH(5GQrWQ8UZ zk{RzgYzpt9x%!@dZ4iQ%^o95`X-9c0Eo>=KcKLqASUFPtB)+3W-4#QIKx-{-7h>+P zLn(N};9kJ4^^+B>jtEO>I zkDQ#tHA{(ezS~HHL*EOFlv(D*e~Z}1J?=`mrqq_tl7sB{#xaiFJghJ(ed6v0MPe+5 zM@A}2Qg!m7)i!dU{o42OPwGPZuNb08l5)#Dt~>{D@zMrF+UT!*obGy3pxOLLR*yPd zkg6ULrs8pjZ6*;b(K}JrJ4&dRGq}ij0>05t#k!UUF?O=zR~wJl{=4!8h2ch3dD(6f zlM_!d6|-A8g1$~-Z0~Cv*SN48My}_zFZQAnyiA#1+z}U>qML-4QVGbF30PrOiHER! zn-~=KF3?=BmbD_l>Pli{!Z%%{*YSt}eu%dui_*PI7q*+Zd-SM}hzzgo84m8cq$3~A zL4t!jyQK%G^dvTNL3t4dsymbN51R@7e%cHJ7;ymqDSr z1@C&F3$n(kDT-A<50>!!M{ehQ-_hYkEtzsT@`%k<>bqV(f#s7z2Yk;Dd~xU<1mhDQ zch#?P;pU#7sJunlvwEa%_NK4^|Kyit+))|M3u$inL{~H1@|fBtSrMxmE=Tr@$h9}6 z*a_Ypp^dK>zeP+9yd_*X7Cy|nr=a|Li0m!H7Uq7|j^)6L=2U1_O!aWt@i02#cd|@f z-~5X-hKNL`WVx7rO;srate`3NT@)W-EPBcS9{9+AmJnJ*TaOJ1#l)Ou$2$6PBX5Se zJc3kcd!C`Ib1&U^0dh~FA0~t2{D)Rh#%%GmT`Gw0$}C{`i~0ufC3tbH$(<{PrTCg6 z)qN49BpW8)-F-yZm#;8i5|yiytn}lUIh)I@k6QpO8=iRyobpIKKD2OjJ0D!IUAZrN z^Nk}YYmb6b*<+yToqNl1;x;AQNi1_NP;cm1_n=TfMRgrP9~O7M8%On#HcfYT))qGI zAO=>>=U%+a_r4!Ocs3;IUv|ud{0TriJNDW!Ra`r0!4QkDw6qAgz3e$9){DF6*?eX? zY2{3+f7$0i8?StzC)p&64Lw~W_IeWkkZG)ATB28WmlM`;z+)!TYonY6L+cbar|ZN}-rUA*3!GB|rKaDi~wYmWq_ zMQUt&OvB)(wUK5W5Pf94O(10RRCWHCfcbxW-}>d$o&PDSc&Oco?z5A|qB9 z(^2TKJz@t-A|6|bi7C{53B{uSOr)36%i`XLh)L@z{MzMMafWx!WZDOMUo^zaCZ0z~ zXVSdxoUcQiJYD7IY~x9ft>rbvyc1wtQ!THYB)Wwo#>x28GZYLoD1OP+cQaa@dbXJSB8d@wOZ-bN19$4C zzA3c2M8H7iS?2n9c~3>C_$#u1ECFl~p=V>S?ZWWWRv8S%%qqMEbB5##ST-p|YRc;4 za&vXQzfo@A7&EZ)9xZUQ`5!NUC#g2dUb<=sZb`&6lc6622!gcfq{|EcP5=X$5ooKL9$udQ_zlVvZY0P@45U&Y;0E=tk)eP7RO+|0WB(5w^S?uKn!jPGkbmeV z5&xTU(F0G+W&K}r2KFXhRCsxlvv$AwKI^5~*paux-bOP*A8cr^9oL@o?-4uIN8}S_ zjt9JNF`$Lo^8GjALMC7Ea??Mb3u_eFp_2S}t2aa53bI31hE3q@@FLGaJQE&Ddpj8)hpvsZ^FnF?VYtNs@RbU{h6p%sm^7!gUo=@{p|D|q~oN0 zFKW-xGBhKH#?&P$i4nEwJ3xc<9R=&faOR_BQ`XKav1M`f=Lr4>yM04dHn9)>||&DaPeWM!`}j|?xEh9Pkn z&SZQql==?IyFU1>LcU!8{Jc`R^#S1Fp8SGCCa@vTS)A5~mdmYX3PyV&p&^@kUbK${ zik=W-Ac#NY3bQXhN^c1r0O*oNEb^ENOKc!3lfcq#?bf~)@+cQS%R4{F29DxbGa z5V1n1^Mj**6D(ej^9%Us1KB^e)j{S)^QEK&82@Ud6g9IIoRWyz`KeQhgiP({KA4n! zEcvL|(X@p<9);eM-zn*TGvCCB>Dg3%lCjvjumU-}Uq zknA_rLA*|WGflo<*O8Bia*lrcq!P0ZKYjK2POjIdJE{3jMq6&BuCHQ5mm<9;lgr=h zyFc4P$J>OZB!bGd3&l;n*O1VJzPTi;3DqJ=f%!4H`*jhn7xMzkV3Z~kSo04=wY+c? znm0#q>qcXqS7`liot+8Bv*y~a)}uf1O?iXStLMwWd|-O7(1pV9SJyV6LuxEyKvLTw zm|fYPF)$ltktf>+vOAkR0C)|0|7r!e_78;#pi$|G&y|hV`kjb|=1rn+0EVbIe|SeX zexC<(J5IlsUpt)v$dge18u5kT#ruD=EQfh)xsqZ8c^N zc|U6hs62n2b)itt|8ZAsupGRC*>CQ0z-GJtjNEOIUIlYfVo(l&yz3)T)>b#jV?mkN>PSU0wh$FK4sH&J6Os1(mfv8q62Lc!tD}>}*BPoiC`usIX z9V&0XghK+{`CuYF=U&*=>5rGivDIAHK+*XZ$DDA8m%MpNRBHqfzfqR~N}C_Q03&c1 z&vq^fD46+wC>V5%hu1$F(*=JwE0FKQd6d}Aefab+bei%jH~@z-#58Q8$W890<@wew z-3{;WqLbv;F64UFBpkd~Z=@VDMQ#y^9v*JyIu51Zig*EeX?5--S_oh1$TAq%(cHjl zaawG0`LyT~A$|3Y9*Dp~SHO&4hLvWbUnu1}CtU9Z=KHTuLf_N(X*Oh%RxCBlCOSZ!Hqgp8k;}qMI-)3xN0sS7pzkDzo z-i=A}KHt3rZ><;8aKHApQpFH8s)g6FtFZP8jXPsR;auF$Ac(*@UDBesAu^)wB-+S; zhV;2bnPd`m4eg;*`*r`Z;DKm$P9MDNHSaI1191kPz%NCrgP&6vGhDdRQ1%6_|YEBmP0Qy=zbhIye<;D#nmlz~T&J_!n~k%l0W53KU=tVse2K z{hir28iqUw15YlJJelSbKEFO{9tbB+%U69N=m&4u0-)~q7jJK56=4H+-*q8vc*wol zADfn8!(%b*3JDQZzJsvg%x!SyY-%ZVzJ()5<6S_aytas-w9W&Ut(a_AandbP37V`C zmgI0G zoV5ij0v>9bmiTTo#zC@>e06)YADf zUlK`~-FZc2&Jpi-du$&UQ8z~vIIST+T#pAH3i=Bod*8GVCjT!wKq(Psy)CX!>a_7iQC^)L_FN?NMVKO-PkTjt?_0Q#%U_R z7@J4~$Y9qQOh%;K$?Xx*9`HZh%?Qk5A(k@T)c4&CueR_iK`6Pv@W!~t`*-B|ze)QsC>D3w$m7qyh{Bp6$;zy(|mVyzT+O`^E zKLyBz{(o7VU~MD}RMgb1EV~c2v5!mTtYm^@(mI1~om%$FNKc~)k)%qv(U}8tR1Fh7 zX^+HhoKHS?+;MKC@D+rV(L9;sw-Pj|6YiZRfdmokG zruk7xcuRhv=Two|8(5VtG!&9J0+vA~nBnfeB{-=}T@oMOgch3O>D0Hd6o%0WO^^6$S0LnILjKasp;-i z&^No0w)N8{wZ_PeRRuvAC50)9`_vaiYb^Y@?vc>Q8lK78spY;$GE0P*dsno-6RsK?A?hPRC=ecFE=_shjJGaevlW*tUJv3BZ z5ZDj!*@7{~`UKBzqEq!hJ#1Vj;eDB#l5*1y1GtvRcreeWM)M3q5=p-%!FAMcbW0^T zxiJ%t!8ebpXGq+DCf4*I?w+iwA=@l-=1oScjFISZ+Zo?ynjNiSkDj5J_7oV&V^s?{ z(C^N)DZ4^fqhO>pKPvjMdmV``$*Sc$kR-TDZ*_1pfP)~AC;N79N^T-zmvsM8+fz)t zQ}MAxo>fFl&t*_2a12Z7``qZB^YPPub^k;7r1@FI6przBu0`biU&6;Z ztPPDiI*RD5NQ}pRdQ9&D;2M?_hZ3x^_HFlNBq~-$ zoIV*5Av9F$3B9Q(dD&9!_S;f|Ze%=2=G45FGvBjd-BXHS=^hC%fo;!NTx;G~;#SUY z?)z3f{#^QLXq#+*4Wf24qY17qK_5)-zCj@s(DuI50?QnznLoi7x7ka=Xf8Q8?ZAtW#oWA50g}p|V$d^}}Omg)6nfug$v-b8^zk^A`Vq5v23xX#XBzM6_U0^AuI_U!>2t_*o1Vb)TG1B;pLA50hdVa$$yM3nqi6y#>Dh zSQE^4|T(yShE%)Qp6<$-6B~%5F*sQzX_4#0WiS-$%w?29vG2~ z5S&Ql?Y->#(T@$%+S@yrIvB>p0idyqUFo2goAIO`mzV(*Gh#W!>0M!Y3rcvrsc5XC zs8|WdQP08>Z2cR34C1oI%nHLSl%>SFnu3Tu7;l68aBT2sZ>ud5N;2!vl~7HM3|2)Y zTz;%P5Tn4|ss0`N3OLjdHlC2H&jj%ELEc;q4bH#{2Jp|Sc*aL0Hm zFFw|9GWkS+@W*T2mv&tVe~;bX<&JaaBXD8KzC$2p+<>>hEX>8M66;n6ux# z&D1j5qghhnw+KCppcP+sgS2HaV&< zQ~e+D@H^7IJgW1ChmALZ&)dIE7lL&Mnp-1jQ; z{1?e{-`XQ#2_KXb>@mgh<1%QRUfbf(BU6$h6Tvza2q}xJZS3*7gMlsUU?eHcM+k|G zFC)C45qjtT;NsG-X$Fi;chaU#Pi_AOO4gM_LOX#^QZ*dSOGoLsBurL-iC z0}Q}dB)lx-PH&#DtwKn3E&%W~lwm&1j!Y#_w;ke*E1MO0Ldt#n?Sc>Non|Yy;}C>fUFv?pj{|xxQs8ipk4{`q39~Ud^(u`l5OolxX_v<~He!QNuHuepVjL45yj;8XZrXdZw z=r0q7=z^QEdKDyWo=Q*2m&)!k%DTZJiS8=wOO2~e)M~@n_yu3J;_*uG70wJ@Y`ZXv zH7|b~-^0)xCt2U$5ys)oqo$==M4{Vh zvR(X(I$@)0ozoZrCAaD=JD`%R70TU6*~xII)0w^UH4(qxBjOy=Y=FK!Dc`0)V^wy> zFDR@@QjLYcK(IoQeKqZW%BWGl(T_apdh<4Y9TSyuo@j4JdMUd!kM{U{j)aV64ZEHe zK;-A0!Fg^d8PspF7ste^7y`wyeaHtRt_84s4F~M5XK&pVu8vXVtm?+S&C6b@^PA zZZnaTB}h;Vio(r@eKQ(P;?!Y%Fk9*P27mgG^e$Xkz+eY|YV9DcFbc_Ei{6s-3VxN%Xy+y8NQ%Td*O)qG;EAS z^-2vh2uaS@otCg-6C<<{{dDZa*Dv5#ruU%Sd_hTr-{4zM^UH>QOD!#}@ycp$a-Rwi zg#{)(1P}HthT+iQ4e3~WMFjU}5D;gC?CZ|YJIE^gr|v9ZpnQh?FZh-uX5y@9&P#&jyL8xd?hp_jSH)_gzVxg2hDZ#Jac$K_OU};m|h1dTxVC zH;gL3rSjtkKTkhL#vy{ORtiq{(=bxwlEp2W9N)*R9{7Tz@dcufkov+nT|&O6*>!EC zx^`u1{f(*SSyCB;4IXt-%X9W%MwG?;H>Q$t$5pYXmsp^AiC%vxonQ4UeyjZ)k7Qme z{+lSJa@aSNV_>EdECE)bD>3i$#(dtI+lMrB^kHf9d?-r}*+$1Ow(nUueCcsY zphAdjV|oUo-lSkfk=N-ql9guJOiRRgfA9Y~T7uXxBqgB#9oqY@7b7%sJaVw7RLpVw z`y(W$k`*Sw!G6RSL8dWyAPo{vY-^hJBLoj@_#7|oQMNO{t`Z;|xKG+0qB7N#c}kkT z&U_En9!ypQqvG1LHP6ZkM6-V}`R$7TgNkz;Rp}!SGZ0S~K&sgZlh*BU?GWULauHE$ z-Z=Dl^a^CVz06>?df&9;w9)n_kvrs<`bPs)4GP`hBF#zexLAv+JiRk~F4c8h`hT#i zp-C2B8z}aWlT6@!F~KUZFH7d&bB}xFK#C=-!2qS!$kXw~S1sUi08PIh5@OkFruGc9I4WgJe#oMxkwtBeB zde##OabW$AQd_@?>oKn5)uk@}l-4(($vVnJOAe1SE$|DZN)#j@5;aNoevSOc4tKoc zU0`MhSd02i<6hUEaqNd*dn04f&AtJ8erlMso0L5B=J?AJZEr>H0HbtDrE}G}pSXZK zYW9;}!orJP8GZtZyCf?{Y0xZRoXNft=nC zc&W@}6IXd?EMTl78Nn!tl8?P+%cO&&h&vCHWb6M}$ zBbtszq@?DkN6}wr_|X~?-PV=Lx+1hL+6SM%Es=e}?nzqM!035pB3VI+ z5$Bnhl3Td{lrAQiU#n{IF_KFMCJu|l;&9qN&9C|q^m%psMXVT(dvWV6J;K&?5R^o{ zFYBgZr1ODpm3dw0{EKesUib?teJF=@P+Ab8bg&=UQQdPl8ls$+g0$Y`rJ&KXu88(80176VIREe(-o^E+>#e%waY9uq#5 z$Vg=lTuA+z#GsaRx;)d>1kr_HnEa}kz6kDA-g@3h4_#RX|4}Dmncp+E9;55gVs?cR ze&C7~Ae!}F;n>HS7^`wEUlgVyy*KDE+V9J#kkG9Hi|e&Bt0j>sTBE>RNLieA9cG@e zhiy$5PU>#*?EHn02vHwTHxt_>lgNLGDM@2jVB?9Cdz-nsNy#ysQrUknk!z@kB~0Yh z@5#wHfQ8$m5ExL3PKfmQ0&*eYmv8v5u$pdZnt|`2R=R|c{aen$bAJlJxk`M8A+X7!N6xNKvnW@Vx(J^|SCP+_>mt7`EV1$mDV@)sFy%+rg)v|5|*5H(KlfwOJl0) z>#xer4`~3DLXN*&{nB&WPUTrnX^t|huORZ)J6)o+cnv$!vdPf*9VvjRABX96~ zgn&7LSUK%&yI0bk(o8D%oeXkLTSaw#vR>%Du&*02mkYI4U3AHbDE}p|E#9%?urU z&U+ZS&GDO-92edn==f)VQr`iqety8b`um}BG8J9na0Rq@{N=h|YBq2H2z!h162f`s z3syJNhUpG7nO?FR@*{?V8|Af)ztoM0UYn3g*Uo(;*Up7KsG-lu07%ua zn-%&p!qAqn(HSGpmV9F{t0jG_od_6c+Gf12M6SZWHl~GE)&k)O43xv=IA51v-|;TlPkaVs z$ECP1fG+1w1k&~i9RX(gVfj}zYGm4=_h3@Mq-FaZ-F=HyzwjZvcdlsOv}yjlRu6rp zt{Y$eTy(^By5!X;=&814rPT3NZ%@TffF){XASUx&=bk_(TjU#iO*Y9LOTc&hj_~*7 zqR~!85%i45f9fv?Y|HuHQn{c#AC^?BW$Skbn@cH1iD24rsG|Gj%5rWN<9rGj$OLGp zzgr|{=J&8CcKMdkS)kKjf-10P*Z1u1Ygle`q)HbMwz%KSlM@&CGm6l|u>_zF&11Ns z7bHmA2?dz=S|Y(LufiIml}z=IR~|XV)HM?(zT&sb7phFL<|| z@7Mx?TmK2G^VK|x?`|Ey!uUia#p;6ZaDU)U^>00d!_)w?)rF`SKr7rD^P5Kjy%$x0 zGO2guS2#w=mPH6uZpB`A?wtAv60^TXeRmQF1%3+IfliDG@4QbIEhV*{5%vhB>TSSg zrK$5-cp5Gj$9_BnKph*3_hlD_t>2!f8ZElvF5r?NhG$ueAz@8G-bbw<(0QScaV_Vl zek0;@eww3P_Z?8ymmTI&V)zZT$Qx@?J5DKUc$v7yug&O0Y~HuY`69;a+u%DMXbi07ZGa8^Bt zgT+gCpHTahbn-}o;MVJ1X|WHw9`LD_ocV*9aKLR=J}G(#CW9|kI5Rhw4V||03V(+Z zxqt*6;W+C~Ky6a$G9thgIlmnp_N>w~Ai4WDHty@hA0f+sfa>wNyK$>Ij6>U(jE&|N zFKWmJ?N7tgk>k4X0htLNjcTSr0_>+^l4j?SSbO+;V$nga06VrxwY!R$#(ZAngztKa zlpIP7iFIVFPo|YTVX-Wjc4Jb+;q)h2>RI9durPfmh_Vm3fjs@Atg6S}P5~%AscfG= zns5*O$~3bK!MFHD^0n1`_2;@J$KpGH zo8@_TkuV=~XIv?`ZUbhdgu#tWx7 zfymgY2Ly8qBnK&?RAEd2Q_?G`7i;Y&^hjA>Da=$6yS@l+DbQ+G8SR5Jk~R7V?k%gG zbyPvdokVr24e(z~0pSp-*mFQLE;UL3fd@bk<0AK;c~d{CUR(WdwHcF(Un_a|dv;%7ay}0Y6xG z_4&}ZB0gPZr=boFI^o2-QM6lge0rZB)n!IzqeG=wJzF{SwjBaIdQLg$wJ2n` z2qQL$XV4OGHj>u$z}-ozh0N^l#4rUGTa709v(n&oh=p35)LCZ~cpgbEW- zHX-_Q7!F>NE6q~)fNA@yz6>)(g7VwKfUTN-gVR+210>0 zV-+-JpBlw$3ei7m=j98BXurOyPeO;y92kUonbw4hH9uZ^#!1__m7u-8Z`t<{XgGIW zFTQdfnq6>_=Cqt_eKp*MdzTPt7Pw%knYg@tW%w;Hso>{hX9;vMLhx*~2Z`D&QR~cR z&JoiW0;|_+JAs$SI`h3IIp$=n8XWGkU&QGyA96WZf&$3*uvl*!{<^#7>Fd!fMB^jE z-o2GuzST@NG>vMe`0+k*3lr01v}e5Z@k>`CWY!=**I@qdr|)17D?uq8Uw%6Eek4iL zimz|N2*kq{`EapyRPb`_fsh)R=$@6kKXn7AL0S(1WAbqEH$KhulnHxaq%6H4T1ng8 zAAxyhbiY#lc_6SIv3mhhxn6U+bv7?mH+~a8RC6f=m~JiTrgQ8iwWq_{$*(XVB_!FJ zQ)4YNh2wc?6A)xIVIoO5(fZzs5we$!^Umi0N7xSSIKh=3I!G(-o~G?u6{+k+bD2ek zD&oO;8qzTm9Z2x5wZEz0EnAlI!vgF3Xc_?iTeLr}9lOpn#r!Zl7eK>{q&W_YF3=5X z#C0`285bE|n`ygq=;0t(;6YMaxd4)(a`ZJQOX9xJbdNc~^L@j9Z2>(Npr5cmVYLXY z`4POk3e$UeJ+zV&Shi!gXxqoQZqw=g$SK-BLt47J zp;Sp=m}U?3W&Re(3FCy&keE71@h_6A?J6A^K%RYtMI7uzhec|bkP^M zKk1@DR;HBfSAD5(3_cigHJF@9L?PojZKjDN$$aPdK~BM^@=V)=?lYJ9FNZ0stevNB zV!gkB2VgoV7jAb+@fbN4s#Ay7lRY3xTIVB~gAo}F)hOSmS@EqO@;#@i>m}^mmTa5L zs7SfNUEqZ}MIcJ6Fuq!wbuq!(@Z2#KN7Xz2E@~?WL|u6G%cz2y zGgoPRyhfdB{aS1Y99g@Igzo9^M?jw~(Dm^MPzjW%xuOq0k1q&|!S<4U=q=M77qX1k zLzGU6eem639Gmul3sqL|IYH#7Ncwc2zKX5d4gdS$34R;0SY2Z&FAPz9INuyW$yRnL zO_G*&a-qCxM$MY>&3HKQIGQLVaXFQ)wNm5y00U|#XaWjb>-w^PViWPnP0*RdS0$Q- zqu3YTp0|5o0u6eRe}g2!pMu534VyoaJmZN7DxWj8584f#Bc}(#!}1wpUVM;?K6_v9 z0mYUSjmD+D-D3Bea|4oUf#xWD(nkyGDj&m zDoujGc6wGxT#x2`-TDieL`p~;)3jAlHy8LikJv-XQJ zz4|uDwbz7myy-2;v43x^9_>y}sm(kzOBB+G%ggj!P|9l_!9s3sZ5|q&_Yoxj+=l0@>vKgpWA)G85Or z&^ja=AdL!<^C!Fc)(|-5IT=@9+jL(Urz!T>6Fhf^t&#cV4zlwG-#k8RJ0bFo$`xDV zcr&&R*v%tl#PRq3q(maiTibwKDd@812|#=Dx?P$PD%@QRuWVwPq4+|TTO*+s2*y{KdCZYSxD z0)B55A=!#6N6$qz{@nDFg%Y_2{AVdX&6SINbPT|_ApGu=!iuR z35&(p6#K}4SIG#(9hSdzJ)Ni}j+VbftF#>NYFyJ1F7ZANx) zGbUHi&?>^SL!0SiEd{N`rR~`4={ z<|=19}06Pqbcxh8Kf)8$`s(t-wh(8>s%$M;v6~*qqgh&^oe+^V+^Enj61@8avfd! zhnfCQ0^L+Pfi^GFi0hS1Bt~#4@;v3(57)+Zco;|y)gsWP~RJTIqGdIH)5 z9|vH$!^KV&#$K67rqw(RE|5RicnZ^Pvr)=v4ZemltkdC306Pj!>*odOZF@~q$;Ex> zkJck-I3!d&RS}5Sy|U-}l-1$ln$N_Ore@?KteCD^mp~#ViIjqGl%ly~?l{A&PzMjjSub#(+!APOAPy&*TZ8gBDaW#I^xAfyRz3d463pSppRnFN zU&x4?nc{;uB$u$AF9ZndmTcR@Ctehnj4x1^E%g$oN!8y5hK_m}b5-04lF zfg{Rh=f&*1cPrW9s;i9PIY@1>yv^ASlu+m?u4O9;PRt65-fgh`jeq5Gd3(n-U2lpL zV)a;f`A2d=YAvS#w316DYx?l!e0@dAYrH%|@UrI< z!wSmwutdpU7^p^l4YPGr*HPCj#I(=F*LS5zn2mIqOIk+t2suA_)G2HRF(>Oj1mTD~Q>)niU}PW0ori@v5hswi0S*EjC|jczPa({_MN6 zzjXaEp{L6VD&v#dubGqoo=ViELcA6FqIZaA@WhCfl4E~H`cDZCo&BV!i1UZBO!@OC z0#MCnR&g)O_cW~puKh`~#%8Ht2KFTiV87W_PbEFZFhORaHB1_akb!T_~c z@XpGWky^y~5?0{Y=0P?RPM^ldq7rD>UB9ttJTTv#K1{=9Ka@0*TWXV2IFkz@hQZ#t z4C9fiqRVTF&9NC%ms)Z5qBlur1l@CV1fVOEWC!(t$NhtwQ=PyU?#D$b*V!?GRK3zc z>aXCV)&x8q-!7$5&%KXnyH+`)*=9~xjpw06@U&mi9$Vb&w`8WY<_MAQ%jdSzvqln> z7gXiavCP_?QJ0PO;SkYO>VuG^nW3{Zv6%~ZHetNg+4ry8{DEX6Iij9T`Hs;zDwk7sX=;vbp`3j9Cdq4tO5<~L8!ZRXF=bAQ z$ISp-;6~EZDvJ)N_UI{!Mlijo+Vj+WSpOiQbUF9o+pKZFWllZh?mv*t z$9{3Qv_XG-HJ&;54#7mxAP?_;e76v$@Ua}a#PBgVHG+}EBozOGKH3IoAM~YSP%7}2 zeE7z0r@=dcAWhg-+oksNxjgv$ysFp7Rpxo9f1&dlQ+eoDINnx7DH;5_%ahpfcO5B9 zpJ$i*eX3w+C;d|fKCMMCWtx5jfQ;v8F3NU;bCdwltXzsZ!3}GOogca|5%D@iepN+yyhG!Wc67wo$aZub*#!!QKcSyFj zdKuoe7lz|(x}b@aFO&&;iMa}WZW+lCn>#COgw!6m-ZhWF+^ihFESYiLO@@x0BTM6P zsGxC;_Y1*pM|<*dZrYZ2$cDHtzIE$vH9ay9r5l=IyxlG)a58-+x45-OR$#RSmtTxu znM_A;H+@li8-xh^UeCH97pRn2P=&MpoLy;qF9p$c>#BVweC48jCk(l5UirCyweHqd zFOA4ECa_Jk#-%Jw6bf;j8ahCTK711w!i}@)<%kwRwk${8@o7GY3)GY^*X3nd4W5(3 zUgM0$j5vom>qces^2gXIa}r@t@lYfq(fzBDvPcLk2i>{`D1h;(VrWQe@TRY-a6Itl zJ=}9&+D5wc0xqbq?WVZ|QnglH=A~GGuDUL??tVIn>8 z_IX2$^rzX)#!AfyYnaAuM;gBPZg_*(JK8#Bz|i?qAr6GZTM5s}q)kRVlSUbr8J!U2 z@56wXqmrvCbG#F7^RvO|H}qat9`T+I4hMZH>-Y87=DwitOwaYPLr}|s3+Yz#f1X_#H_5-fWJUvv?BOc^?O5PtJ4NIh+T*Y8#POI! z|8qTmU*1Pr<$kv@zkeLEf#S`wbk=%f6si&02HTptXFFeq7GkwBh}+G))>Pxu^DuRo z_D@vnOXJ?pKcHsPivRqNb5p6JS@PFL`ai3x6o- z?lMkwvb*)nwmdKHQR1(i-?)q&S6UX!V~z_X{?ALLLPP&0@>OASqWsMjoVl%kPqX!z z=;i8RaU;@?_jjmM3}#cf9|+75?KwWZc#KrnN|*2!(r^_{EL>suErwX-(Q z)6dcQ3dL;AYrRlFXGjKh;kxRna%#I?n##E2(P6lkq(&$sFc!3}8%%cvt4-BvGpbsE z0_#Fd*e1e4@9g{MWYdH~w)$0{&o36ZV+NmT_21u&+^dy)-R`rBF#`EW_brLnK;7~y zym6+b3^AmCH{QfMM<6D7150dkuf2o0d(6Tu%wesFsq(V)i{>4;C=iUK|N6P>>rX-p z=Ofm$jVuvzI&+bfG{@ zF!e=2z9_tK3QOA7}oE2Qw2p7xn;12N7}poD~w@jSPP6Wq*Sg|CSLwi-d~mSc{I8BYfRo39L?J}+bk z2f8-H;>#7LNVgMzn7Z%D>(9^H5Ugdh3tyYEyS?w^xR)9=6WM={Q3r?&_`)yLIx58u zSad%j@aP=XlYth3W(*l8GWtN}b}c>J)CTYuC5M`^RgX$c0cq|D`emClcRVBBzw8bW>|ek`s}6A8bdRe%E6j`J z7i!pq<`T>ZXT&Z$a6r@|fPj`k$g(*qI`-n}%Eb|Q@tA)j2RdlQS9C;4f8t>KG}&wX zjJPEIEP-?iX$pCYWk*+8Y5L;g36T~Jo*z)s$~_i#=>$qAqEADBJuh3EaLgGHmbeNm zGqW23z-3lUE@|*B!?u@HeTG{M79GjJNZqY9{HH?tv~vL{D}A1T@Ii81dY)^RBz*W> z2n8B^K?oqe3G_f)(651Ic*y6;Bkp-Xv90PIX=*2sZk>vK4aB{zusQ*Dhhjy*;Plpv zDtPmDlSwQ&Xz6SES?VBKp#`+O|Lhaxg1r&YjZ7)Pu*lQ@(&N}ozXqtEf-*)hPa)0kMJ@6J2_DLaQf zh*S6>U>ID1YwtL3f))^js!Nw~y|@#jBnXsSd!L9^`<8BnKIG}>>4F3~rwz(%I){PD z#kXpYZ2hiCv!$pJAbq{hS?&RFk>=weu(pHMdKzw{h7{%g%OuC*rX!f$j6JO3;ilO@ zdUEv@Pq|k5!=Yh@=UK`M14Wbl>%rZhy1K69SCnm6zimJzPng^lTyafPHzqRf>|xZl zN+-&XoRzP}6I|1{=cy_FDOP%bV<`S{MyPw3w;OHTxaCYchOx;T+uATQHtg?aIsgvb2vf`9jQXVr zWN*yZ$3Z()Hcf-E{U?kwNeCj|Rr{)(RQ+;6=o!|;y1G@H0(1g05~vL(TI>IY5I##D zu?#2dnk*5*P8Xl>W1;UCe``WNzv`xn_)z1fFF@Ek*d82&Y+MN#1l7fE_mtfBQrl6> zTp0QtJf05iT8t2vj*nzr_6GzpVIzIDI>+{Vas*o}%`iVL?_a}j3=V7mIN2z9+ zuYtR!#EkL+s9Q?%!7~cncIWDU%_TPK?GYSCwE88VX{8kdFim-dWJA0bY;1ss7@RHs zDk4Fk&Uq9e@@IDW>jola`8-dp7YKcfW3%OvXKOHl13d}^W7NihHqE@UZBn3`_ab3c zqYY)PGqpeyw3qS3uA`>Kt~k~L<`(klsb~IUfaNNIt;>AQG!cRT?}!3OP!m+@ZMG@( z$Z5f<4jd9i0p<##$=d!90ok;e(7X`4o*T)-7z1Wj;(>`ueM+*#!Zoa=x43&C?JJ%#Cdt37^zUvx3I5Uv^VyCHfz4sLhhP(scb?lX|9e}3(X}e>5*~GZUR$O7upddVg-#aULGQm zTSn_yQf@xc7(IQm2w+71<@$&mFZ|7Yw6;wTrXdMJnpB;J>x4R zxAT=U(i7wWJIKjq5WUT9zbr4j%2rDBG*~GoSRsvhbBhOviz|5ahyWxD7FXMQ-Zg~h zDk`4mdE!f*!#O#bpA|&GZWh8obN_{>7aseG43EXn1?)85bD)H)Jj_E(aFqH|fR9ZQ zj2mw5ppod`3o^}8N=V5)%s-w(ZX7BbOBC;woun75tzlH^A__H1YINu`cgc$Z^6fa2K;K!E|-%Fcj}I2YN&~A?RLEjX+45uKH#-e6N`QU~|6Ja{}gyIk{R`ClhUtcxNfB zh`LGk+$lD#Rw^-aV+7sn_p|NI7OR^M<3LJBnX4PlAII|=7}|eC{S?WvbsGm)V5bLP zQVyZx+PC-sSX-g#fvsZDS|&7(J+5W&(YI~O$mY6S&bZlRs)8`v5?E~+L(s2|bxQnp zI@u~Ub!X8VFGtlqk?zh(%Clk->%d~gQP1+Tc1l`m|4Ibwr*8ef=o%?|{_<%P!F5?) zHM^GA#5uJ&;x|K<<+T6mJ1XDdHw>s1GP2kuH26o!@hPWV{SVE=L+Vj&!9`I|s+fJ# zIM;VEj5b@n4OT-!zQ|dWRK5*)`C)m@`Q;ji4%GJ>ii>m-c%Rv65{uJ>`I*T1=(&Pl zvbZ(*qf{oQ0e%R0ajF5hf2Po%j)$Dn27Lj0u6;_tX=~(Y{MZZlcophMioZO9B)RH$ zzVIJFMYH}AIA$$J?V;?&CJY@agoYQe2BMrgv;q>8d^0ExiayF{WEUA=L zfAcm_?B)lZtKEY@;cw2iwP44e_HJ6PfYk@3}7LV@@h51c|X=)^971L0eQ=X`h+}J04gcb1`e` z|AK3Q^xjPaaXcVBI_cYUkJcyBx2U8{zO_u_Xet+J8&y;O!vwX%4B!QRe)&3(q9y7A zv7hwHsQJtsiX&2oeYnTwIf~@>^LXnoi+*P&p$1GXoHLQna(~eyvR_@bT5hn*XlXB0 ze*|T>{kAp$k6?RoI6!IA>u>HRdb_2X>)m84=kz4hr+MNUB=q}V;Oz!n90wO%r9Mwf zGdN!GTf*cF-qvqkn8_^$If*_9Fv2&4+iqd$Ec~6@=5_JBw*hA~aFG0R;d&^_E3{~b z-dbi@Pp^vZ388Snt2%Yg8z&J;s!w*mmjl|&$@^eX_6}(u;sPddNR$-hRVt$ z>{O^Ok@V|B1M`jIBurI&D8JM77}To?8-AIs{juc2z5;iK9VWKjn7-)a7GS+v=y#J# zy2Ts7&!q>kj+M!bPo>LbhokDOl9)~oZSyempeZy*`GLup|37eB0f*V1B&?vh;E`{E z81}~iEDXjg7qI{$5J>3XaoeU;+!(C#jxkH*ipuHG^akcQq3ahzyZLBMHgPkze2f=P zc!mC>Dpk-2nOMb?IwzHY_FOqlfSy52ztMWUi&-ebu$tenH>}E(A6a_#J2P%TV`0W# zo9xFl=Fr6hU=PZ|j!p6#)UwWzyY*6`q!Z5pZob>px#uTHq6vz$M@o<}w4PLzLR@C2 zbG{GTEW;aqI#9zk3IayuY!Wm5h-#gB^hY&maqp*miyW_(q(#oLnzGn~f|(Wq!p1^% z&Y!2{?pR8cXpl;R>g<2Y>c(3QO7NNQ-PAW++rWa_Z;Q2WNY2I+Fp+1+CG+w+P#g=te-pkzNYACd`S{-jZ?XtLL z4Fy>~Q(J|bO*_0U6LOp@<8AqUhJ}t1;!ksgNYdTX?rbAzvot`Fqw|p?fET1dn~ql;)En>SZhP4wvomYL)O2H&?(||62H8s~BTW+jbatI?k!2vu#xPEMj+(heZStBk|r2xM!aM~xFlQc&~YbBo@=8^#_T7Wkxho-dKcF2be`@Q*_kwj$bz^`7>#ko zVXx#v!6;>%cxqE>LU3zKE z`M*T7F60`;Fti;$=cI+}ny5G7fm{@5O5Y|C(x)DHpMJI@#~+*J^^kfQCHixe#CA%P z)A40;-5%~WPrQ+(-KtAPZmHF|fwJ&qi)Bf%`i%9dB<&+gZ@21>UR=(!2+UKqa!|g%eEd?@+d~nbl1BOK)p&{e^Z_L921?+pOdV5gQ5Ov2y%5wDNYj zP;l6Yg8s!44hQ6Vg&14syD;nzD|&oH$lP~k;~+*xd0{`}J-VdmzOgrq34^Koag9NA zq3(1%akJ5#wAN6A@}EJ;%%2p0Ntxlj2K(9Jsi?@0$00Qs^Hhg^N&R04PIA(l<^jPO z{0qThn3XQ!Q5NM7wUNgS)}`{kG>_3I9G1f}2?a(_*S~ct}g$xydXA;Lz52yXNbUG6dtM2&Z2*6S`=huy~0&t{%pLWN5jGM{T zJPKak&zSu=Zj}`>C^VdXPotJXUm^WojpAh z>FaW?RnT9#n4jhyAt%IfL7g%sAYULiR!9cFCpBM?TP4kop`bSXnTVK9s(?jdtr#rC zHA)%K9$~pxYHNf^HSD0M;KV-j^i3PgwJ7<%`vE?g=k8BCvSL3RkTmPp(r4V6KqA!p zfvpwGc{b#;^cyy-#?J?9j5$M&HGu3;Kg}rrvCnVf4e{>UJ|O5{AnnZ1UAZM<@8b|P zzw>dVqFX-B-7tg*fhUAklInNj;B89I8|^r1oRW3`tMe5#9z6$`-1U~4So`^T-OGuK zRwwhF?`9tuZQ6V|zVLo*Cgflu2}KWmd$~)tD1OT9)enW zT>FflPVQlKWmC6+z&Cj@n3+qpvDyW`C=jebdi7DB^KX-wF)7QZJ~_`Ba6P@czaEm6A4QqO?}t+ztb@vlRrkPa_+ zY$J#d;p{OTyaUFIUBT6R7L5Xb0?#6L(=BeH%A3@Gk~TUb7QSa4UCH}DsBST9t81(P zA~L=uNCo|P5J1z*!PT4p&dyy}3Ac~I$2uqmQ^WtnN+$Ic?G@C1(FvRYL4oLP10(GC zD-Q%T-&^E*wsvFS;jT0Zy3<)XYv>IPwoyoJ$&gI6dq8-2*2Qg=ND5-%W4yZ0`+u%~ zKB%2zgIi8$AJc<9JykiF^=5fU&FV1Hvz|daBVUwTZ>V-Ah&6}?di9srnC$Bk@!(~K zHdc;Z(ljjuK-kY7G4U(>wL5rrlH?sLYKVPM?HyrjfXh!GjW56A8{+l6VH=fP<3iU9 z0#AfYAF^KGA6B@*bBJGR9bTl_;M|nc{nndmlC`Jt|B2#jsj}|L0E5J7pghxPNUD66 z)eIy1Brvv@FRc9zxwct`pw0drI`pAM$PX zwt_0==3umewBlpP?V4*#Y1wO8?Yq2c+tQVv^d7e)^)ux-FkO?pfz_v3^rymOw)ek# zyKAg;q~=wq(58w9))k$0ZP{vxXiIdn+9qEE4f`S7ZHCqk)6Q+1^h8A%jWbUDrulSm%R@Hn4Wl;amSU8LDo zEo+(3A8yc=)dsIFpt>4XOLLmCDoTIP?nL6H&7OVO&Dk67@-Hg&OJl1E{MFlFKx@y!b+aY31UjBV#jp{uC15L z>_^e9d$ZuhTyn59bSolD56;!pic_Q++aq%4rlgtn{b0MeTUkRHdtzazc7T{DGRTRc;T zxgbPU-a3Hw^447-ri#5Qyzw(5HPJCnIEx#Xm9HKfkG1c#nZ89l2Hp3y+?CT?#k&j_ zQOkDU2CSr|!p>Fa49B&movRaa0t;7CipK#5aq>iP| z=Do}R3xjcdMmcb~Se`HFCQ23mGaLzR$W85IN1|WiCE%FNh6itb%+XhbXG88T+-|UB z+4JW&?)tlM)V)YU$LX*Ca1yipJN5f7%63vmjwZHCe^hY*J9lbsA>(@;?v1siWn9S#b+|`_q0+_j2zOAxi+E1LH-I42n&Cvr z*+fBj;=w#nL2dfnVWafNQvc0+{|EmTO48Kx zBOUuDHYLbMHut5I1qb={HwE3a`JS~H>%9?R|Ic5X5G6pG#<@NlN0cg~o+$UC_II~G z&3*))Ee?#ze6aGL-z&*0B=PkpW4_gWHO}Mr$a-|CAI-cKiyr}hvQmoRG6@5p{|9dP Bs$T#A literal 0 HcmV?d00001 diff --git a/alf/examples/dqn_breakout_conf.py b/alf/examples/dqn_breakout_conf.py new file mode 100644 index 000000000..22dcee671 --- /dev/null +++ b/alf/examples/dqn_breakout_conf.py @@ -0,0 +1,36 @@ +# Copyright (c) 2022 Horizon Robotics. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: for lower bound value target improvement, add these flags: +# --conf_param='ReplayBuffer.keep_episodic_info=True' +# --conf_param='ReplayBuffer.record_episodic_return=True' +# --conf_param='LowerBoundedTDLoss.lb_target_q=True' + +import alf +from alf.algorithms.dqn_algorithm import DqnAlgorithm +from alf.utils.schedulers import LinearScheduler + +# Much of the network and critic loss parameters are the same as in sac_breakout. +from alf.examples.sac_breakout_conf import q_network_cls, critic_loss_ctor, \ + critic_optimizer + +alf.config( + 'DqnAlgorithm', + q_network_cls=q_network_cls, + rollout_epsilon_greedy=LinearScheduler( + progress_type="percent", schedule=[(0, 1.), (0.1, 0.1), (1., 0.1)]), + critic_loss_ctor=critic_loss_ctor, + q_optimizer=critic_optimizer) + +alf.config('Agent', rl_algorithm_cls=DqnAlgorithm) diff --git a/alf/examples/her_fetchpush_conf.py b/alf/examples/her_fetchpush_conf.py index 87a2f6d00..e59092b2f 100644 --- a/alf/examples/her_fetchpush_conf.py +++ b/alf/examples/her_fetchpush_conf.py @@ -32,7 +32,7 @@ alf.config( 'HindsightExperienceTransformer', her_proportion=0.8, - sparse_reward_transform=suite_socialbot.transform_reward_tensor) + episodic_reward_transform=suite_socialbot.transform_reward_tensor) alf.config( 'TrainerConfig', data_transformer_ctor=[ diff --git a/alf/examples/her_target_navigation_states.gin b/alf/examples/her_target_navigation_states.gin index 467a05a19..6534f5d3a 100644 --- a/alf/examples/her_target_navigation_states.gin +++ b/alf/examples/her_target_navigation_states.gin @@ -91,7 +91,7 @@ TrainerConfig.num_eval_episodes=50 ReplayBuffer.keep_episodic_info=True HindsightExperienceTransformer.her_proportion=0.8 HindsightExperienceTransformer.threshold=0.5 -HindsightExperienceTransformer.sparse_reward_transform=@suite_socialbot.transform_reward_tensor +HindsightExperienceTransformer.episodic_reward_transform=@suite_socialbot.transform_reward_tensor TrainerConfig.data_transformer_ctor=@HindsightExperienceTransformer # Finer grain tensorboard summaries plus local action distribution diff --git a/alf/examples/sacbreakout-lbtq-Qbert.png b/alf/examples/sac_breakout_conf-lbtq-Qbert.png similarity index 100% rename from alf/examples/sacbreakout-lbtq-Qbert.png rename to alf/examples/sac_breakout_conf-lbtq-Qbert.png diff --git a/alf/examples/sac_breakout_conf.py b/alf/examples/sac_breakout_conf.py index dc753da49..8c754d56e 100644 --- a/alf/examples/sac_breakout_conf.py +++ b/alf/examples/sac_breakout_conf.py @@ -18,12 +18,12 @@ # NOTE: for lower bound value target improvement, add these flags: # --conf_param='ReplayBuffer.keep_episodic_info=True' # --conf_param='ReplayBuffer.record_episodic_return=True' -# --conf_param='TDLoss.lb_target_q=True' +# --conf_param='LowerBoundedTDLoss.lb_target_q=True' import functools import alf -from alf.algorithms.td_loss import TDLoss +from alf.algorithms.td_loss import LowerBoundedTDLoss from alf.environments.alf_wrappers import AtariTerminalOnLifeLossWrapper from alf.networks import QNetwork from alf.optimizers import AdamTF @@ -50,7 +50,7 @@ def define_config(name, default_value): fc_layer_params=FC_LAYER_PARAMS, conv_layer_params=CONV_LAYER_PARAMS) -critic_loss_ctor = functools.partial(TDLoss, td_lambda=0.95) +critic_loss_ctor = functools.partial(LowerBoundedTDLoss, td_lambda=0.95) lr = define_config('lr', 5e-4) critic_optimizer = AdamTF(lr=lr) diff --git a/alf/utils/value_ops.py b/alf/utils/value_ops.py index 2265381b5..443cae9b2 100644 --- a/alf/utils/value_ops.py +++ b/alf/utils/value_ops.py @@ -125,15 +125,19 @@ def generalized_advantage_estimation(rewards, td_lambda=1.0, time_major=True): """Computes generalized advantage estimation (GAE) for the first T-1 steps. + For theory, see "High-Dimensional Continuous Control Using Generalized Advantage Estimation" by John Schulman, Philipp Moritz et al. See https://arxiv.org/abs/1506.02438 for full paper. + The difference between this function and the one tf_agents.utils.value_ops is that the accumulated_td is reset to 0 for is_last steps in this function. + Define abbreviations: - B: batch size representing number of trajectories - T: number of steps per trajectory + Args: rewards (Tensor): shape is [T, B] (or [T]) representing rewards. values (Tensor): shape is [T,B] (or [T]) representing values. @@ -143,6 +147,7 @@ def generalized_advantage_estimation(rewards, reduction in temporal difference. time_major (bool): Whether input tensors are time major. False means input tensors have shape [B, T]. + Returns: A tensor with shape [T-1, B] representing advantages. Shape is [B, T-1] when time_major is false. @@ -179,6 +184,80 @@ def generalized_advantage_estimation(rewards, return advs.detach() +def generalized_advantage_estimation_retrace(rewards, + values, + step_types, + discounts, + target_value, + importance_ratio, + use_retrace=False, + td_lambda=1.0, + time_major=True): + """Computes generalized advantage estimation (GAE) with Retrace for the first T-1 steps. + + For Retrace, see + "Safe and Efficient Off-Policy Reinforcement Learning" + by Remi Munos, Tom Stepleton, Anna Harutyunyan, and Marc G. Bellemare, NeurIPS 2016. + See https://proceedings.neurips.cc/paper/2016/hash/c3992e9a68c5ae12bd18488bc579b30d-Abstract.html for full paper. + + Args: + rewards (Tensor): shape is [T, B] (or [T]) representing rewards. + values (Tensor): shape is [T,B] (or [T]) representing values. + step_types (Tensor): shape is [T,B] (or [T]) representing step types. + discounts (Tensor): shape is [T, B] (or [T]) representing discounts. + importance_ratio (Tensor): shape is [T, B] (or [T]) representing action importance ratios. + use_retrace (bool): When True, uses Retrace to compute advantage. + td_lambda (float): A scalar between [0, 1]. It's used for variance + reduction in temporal difference. + time_major (bool): Whether input tensors are time major. + False means input tensors have shape [B, T]. + + Returns: + A tensor with shape [T-1, B] representing advantages. Shape is [B, T-1] + when time_major is false. + """ + + if not time_major: + discounts = discounts.transpose(0, 1) + rewards = rewards.transpose(0, 1) + values = values.transpose(0, 1) + step_types = step_types.transpose(0, 1) + if use_retrace: + importance_ratio = importance_ratio.transpose(0, 1) + target_value = target_value.transpose(0, 1) + + assert values.shape[0] >= 2, ("The sequence length needs to be " + "at least 2. Got {s}".format( + s=values.shape[0])) + + is_lasts = (step_types == StepType.LAST).to(dtype=torch.float32) + is_lasts = common.expand_dims_as(is_lasts, values) + discounts = common.expand_dims_as(discounts, values) + + advs = torch.zeros_like(values) + if use_retrace == False: + weighted_discounts = discounts[1:] * td_lambda + delta = rewards[1:] + discounts[1:] * values[1:] - values[:-1] + with torch.no_grad(): + for t in reversed(range(rewards.shape[0] - 1)): + advs[t] = (1 - is_lasts[t]) * \ + (delta[t] + weighted_discounts[t] * advs[t + 1]) + advs = advs[:-1] + else: + delta = (rewards[1:] + discounts[1:] * target_value[1:] - values[:-1]) + weighted_discounts = discounts[1:] * td_lambda * importance_ratio[:-1] + with torch.no_grad(): + for t in reversed(range(rewards.shape[0] - 1)): + advs[t] = (1 - is_lasts[t]) * \ + (delta[t] + weighted_discounts[t] * advs[t + 1]) + advs = advs[:-1] + + if not time_major: + advs = advs.transpose(0, 1) + + return advs.detach() + + def discounted_return(rewards, values, step_types, discounts, time_major=True): """Computes discounted return for the first T-1 steps. @@ -335,83 +414,3 @@ def one_step_discounted_return(rewards, values, step_types, discounts): rets = (1 - is_lasts[:-1]) * (rewards[1:] + discounted_values[1:]) + \ is_lasts[:-1] * discounted_values[:-1] return rets.detach() - - -def generalized_advantage_estimation_retrace(rewards, - values, - step_types, - discounts, - target_value, - importance_ratio, - use_retrace=False, - td_lambda=1.0, - time_major=True): - """Computes generalized advantage estimation (GAE) for the first T-1 steps. - - For theory, see - "High-Dimensional Continuous Control Using Generalized Advantage Estimation" - by John Schulman, Philipp Moritz et al. - See https://arxiv.org/abs/1506.02438 for full paper. - - The difference between this function and the one tf_agents.utils.value_ops - is that the accumulated_td is reset to 0 for is_last steps in this function. - - Define abbreviations: - - - B: batch size representing number of trajectories - - T: number of steps per trajectory - - Args: - rewards (Tensor): shape is [T, B] (or [T]) representing rewards. - values (Tensor): shape is [T,B] (or [T]) representing values. - step_types (Tensor): shape is [T,B] (or [T]) representing step types. - discounts (Tensor): shape is [T, B] (or [T]) representing discounts. - td_lambda (float): A scalar between [0, 1]. It's used for variance - reduction in temporal difference. - time_major (bool): Whether input tensors are time major. - False means input tensors have shape [B, T]. - - Returns: - A tensor with shape [T-1, B] representing advantages. Shape is [B, T-1] - when time_major is false. - """ - - if not time_major: - discounts = discounts.transpose(0, 1) - rewards = rewards.transpose(0, 1) - values = values.transpose(0, 1) - step_types = step_types.transpose(0, 1) - if use_retrace: - importance_ratio = importance_ratio.transpose(0, 1) - target_value = target_value.transpose(0, 1) - - assert values.shape[0] >= 2, ("The sequence length needs to be " - "at least 2. Got {s}".format( - s=values.shape[0])) - - is_lasts = (step_types == StepType.LAST).to(dtype=torch.float32) - is_lasts = common.expand_dims_as(is_lasts, values) - discounts = common.expand_dims_as(discounts, values) - - advs = torch.zeros_like(values) - if use_retrace == False: - weighted_discounts = discounts[1:] * td_lambda - delta = rewards[1:] + discounts[1:] * values[1:] - values[:-1] - with torch.no_grad(): - for t in reversed(range(rewards.shape[0] - 1)): - advs[t] = (1 - is_lasts[t]) * \ - (delta[t] + weighted_discounts[t] * advs[t + 1]) - advs = advs[:-1] - else: - delta = (rewards[1:] + discounts[1:] * target_value[1:] - values[:-1]) - weighted_discounts = discounts[1:] * td_lambda * importance_ratio[:-1] - with torch.no_grad(): - for t in reversed(range(rewards.shape[0] - 1)): - advs[t] = (1 - is_lasts[t]) * \ - (delta[t] + weighted_discounts[t] * advs[t + 1]) - advs = advs[:-1] - - if not time_major: - advs = advs.transpose(0, 1) - - return advs.detach() From 12d704a49ffacc729876472acce1e5e8aeceb2e1 Mon Sep 17 00:00:00 2001 From: Le Horizon Date: Sun, 15 May 2022 21:25:30 -0700 Subject: [PATCH 7/7] remove unused conf to fix test. --- alf/examples/her_target_navigation_states.gin | 1 - 1 file changed, 1 deletion(-) diff --git a/alf/examples/her_target_navigation_states.gin b/alf/examples/her_target_navigation_states.gin index 6534f5d3a..b20d0753f 100644 --- a/alf/examples/her_target_navigation_states.gin +++ b/alf/examples/her_target_navigation_states.gin @@ -90,7 +90,6 @@ TrainerConfig.num_eval_episodes=50 # HER ReplayBuffer.keep_episodic_info=True HindsightExperienceTransformer.her_proportion=0.8 -HindsightExperienceTransformer.threshold=0.5 HindsightExperienceTransformer.episodic_reward_transform=@suite_socialbot.transform_reward_tensor TrainerConfig.data_transformer_ctor=@HindsightExperienceTransformer