From 70b02ca92256ec0f687275d242a89aad5a32408c Mon Sep 17 00:00:00 2001 From: gecheng Date: Tue, 8 Oct 2024 11:22:31 +0800 Subject: [PATCH 1/7] DSSM + SENet --- easy_rec/python/layers/senet.py | 73 +++++ easy_rec/python/model/dssm_senet.py | 182 +++++++++++ easy_rec/python/protos/dssm_senet.proto | 27 ++ easy_rec/python/protos/senet.proto | 8 + .../model_config/dssm_senet_on_taobao.config | 286 ++++++++++++++++++ 5 files changed, 576 insertions(+) create mode 100644 easy_rec/python/layers/senet.py create mode 100644 easy_rec/python/model/dssm_senet.py create mode 100644 easy_rec/python/protos/dssm_senet.proto create mode 100644 easy_rec/python/protos/senet.proto create mode 100644 samples/model_config/dssm_senet_on_taobao.config diff --git a/easy_rec/python/layers/senet.py b/easy_rec/python/layers/senet.py new file mode 100644 index 000000000..5715d189f --- /dev/null +++ b/easy_rec/python/layers/senet.py @@ -0,0 +1,73 @@ +# -*- encoding:utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +import tensorflow as tf + +if tf.__version__ >= '2.0': + tf = tf.compat.v1 + + +class SENet: + ''' + Squeeze and Excite Network + + Input shape + - A list of 2D tensor with shape: ``(batch_size,embedding_size)``. + The ``embedding_size`` of each field can have different value. + + Args: + num_fields: int, number of fields. + num_squeeze_group: int, number of groups for squeeze. + reduction_ratio: int, reduction ratio for squeeze. + l2_reg: float, l2 regularizer for embedding. + name: str, name of the layer. + + ''' + def __init__(self, num_fields, num_squeeze_group, reduction_ratio, l2_reg, name='SENet'): + self.num_fields = num_fields + self.num_squeeze_group = num_squeeze_group + self.reduction_ratio = reduction_ratio + self._l2_reg = l2_reg + self._name = name + + def __call__(self, inputs): + g = self.num_squeeze_group + f = self.num_fields + r = self.reduction_ratio + reduction_size = max(1, f * g * 2 // r) + + emb_size = 0 + for input in inputs: + emb_size += int(input.shape[-1]) + + + group_embs = [ + tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs + ] + + squeezed = [] + for emb in group_embs: + squeezed.append(tf.reduce_max(emb, axis=-1)) # [B, g] + squeezed.append(tf.reduce_mean(emb, axis=-1)) # [B, g] + z = tf.concat(squeezed, axis=1) # [bs, field_size * num_groups * 2] + + + + reduced = tf.layers.dense( + inputs=z, + units=reduction_size, + kernel_regularizer=self._l2_reg, + activation='relu', + name='%s/reduce' % self._name) + + excited_weights = tf.layers.dense( + inputs=reduced, + units=emb_size, + kernel_initializer='glorot_normal', + name='%s/excite' % self._name) + + + # Re-weight + inputs = tf.concat(inputs, axis=-1) + output = inputs * excited_weights + + return output \ No newline at end of file diff --git a/easy_rec/python/model/dssm_senet.py b/easy_rec/python/model/dssm_senet.py new file mode 100644 index 000000000..1866b08a3 --- /dev/null +++ b/easy_rec/python/model/dssm_senet.py @@ -0,0 +1,182 @@ +# -*- encoding:utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +import tensorflow as tf + +from easy_rec.python.layers import dnn +from easy_rec.python.model.match_model import MatchModel +from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config +from easy_rec.python.protos.loss_pb2 import LossType +from easy_rec.python.protos.simi_pb2 import Similarity +from easy_rec.python.utils.proto_util import copy_obj +from easy_rec.python.layers import senet + +if tf.__version__ >= '2.0': + tf = tf.compat.v1 +losses = tf.losses + + +class DSSM_SENet(MatchModel): + + def __init__(self, + model_config, + feature_configs, + features, + labels=None, + is_training=False): + super(DSSM_SENet, self).__init__(model_config, feature_configs, features, labels, + is_training) + assert self._model_config.WhichOneof('model') == 'DSSM_SENet', \ + 'invalid model config: %s' % self._model_config.WhichOneof('model') + self._model_config = self._model_config.dssm_senet + assert isinstance(self._model_config, DSSM_SENet_Config) + + # copy_obj so that any modification will not affect original config + self.user_tower = copy_obj(self._model_config.user_tower) + + self.user_seq_features, self.user_plain_features, self.user_feature_list = self._input_layer(self._feature_dict, 'user', is_combine=False) + self.user_num_fields = len(self.user_feature_list) + + # copy_obj so that any modification will not affect original config + self.item_tower = copy_obj(self._model_config.item_tower) + + self.item_seq_features, self.item_plain_features, self.item_feature_list = self._input_layer(self._feature_dict, 'item', is_combine=False) + self.item_num_fields = len(self.item_feature_list) + + self._user_tower_emb = None + self._item_tower_emb = None + + def build_predict_graph(self): + user_senet = senet.SENet( + num_fields=self.user_num_fields, + num_squeeze_group=self.user_tower.senet.num_squeeze_group, + reduction_ratio=self.user_tower.senet.reduction_ratio, + l2_reg=self._l2_reg, + name='user_senet' + ) + user_senet_output_list = user_senet(self.user_feature_list) + user_senet_output = tf.concat(user_senet_output_list, axis=-1) + + num_user_dnn_layer = len(self.user_tower.dnn.hidden_units) + last_user_hidden = self.user_tower.dnn.hidden_units.pop() + user_dnn = dnn.DNN(self.user_tower.dnn, self._l2_reg, 'user_dnn', + self._is_training) + user_tower_emb = user_dnn(user_senet_output) + user_tower_emb = tf.layers.dense( + inputs=user_tower_emb, + units=last_user_hidden, + kernel_regularizer=self._l2_reg, + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1)) + + item_senet = senet.SENet( + num_fields=self.item_num_fields, + num_squeeze_group=self.item_tower.senet.num_squeeze_group, + reduction_ratio=self.item_tower.senet.reduction_ratio, + l2_reg=self._l2_reg, + name='item_senet' + ) + + item_senet_output_list = item_senet(self.item_feature_list) + item_senet_output = tf.concat(item_senet_output_list, axis=-1) + + num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) + last_item_hidden = self.item_tower.dnn.hidden_units.pop() + item_dnn = dnn.DNN(self.item_tower.dnn, self._l2_reg, 'item_dnn', + self._is_training) + item_tower_emb = item_dnn(item_senet_output) + item_tower_emb = tf.layers.dense( + inputs=item_tower_emb, + units=last_item_hidden, + kernel_regularizer=self._l2_reg, + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1)) + + if self._model_config.simi_func == Similarity.COSINE: + user_tower_emb = self.norm(user_tower_emb) + item_tower_emb = self.norm(item_tower_emb) + temperature = self._model_config.temperature + else: + temperature = 1.0 + + user_item_sim = self.sim(user_tower_emb, item_tower_emb) / temperature + if self._model_config.scale_simi: + sim_w = tf.get_variable( + 'sim_w', + dtype=tf.float32, + shape=(1), + initializer=tf.ones_initializer()) + sim_b = tf.get_variable( + 'sim_b', + dtype=tf.float32, + shape=(1), + initializer=tf.zeros_initializer()) + y_pred = user_item_sim * tf.abs(sim_w) + sim_b + else: + y_pred = user_item_sim + + if self._is_point_wise: + y_pred = tf.reshape(y_pred, [-1]) + + if self._loss_type == LossType.CLASSIFICATION: + self._prediction_dict['logits'] = y_pred + self._prediction_dict['probs'] = tf.nn.sigmoid(y_pred) + elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: + y_pred = self._mask_in_batch(y_pred) + self._prediction_dict['logits'] = y_pred + self._prediction_dict['probs'] = tf.nn.softmax(y_pred) + else: + self._prediction_dict['y'] = y_pred + + self._prediction_dict['user_tower_emb'] = user_tower_emb + self._prediction_dict['item_tower_emb'] = item_tower_emb + self._prediction_dict['user_emb'] = tf.reduce_join( + tf.as_string(user_tower_emb), axis=-1, separator=',') + self._prediction_dict['item_emb'] = tf.reduce_join( + tf.as_string(item_tower_emb), axis=-1, separator=',') + return self._prediction_dict + + def get_outputs(self): + if self._loss_type == LossType.CLASSIFICATION: + return [ + 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', + 'item_tower_emb' + ] + elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: + self._prediction_dict['logits'] = tf.squeeze( + self._prediction_dict['logits'], axis=-1) + self._prediction_dict['probs'] = tf.nn.sigmoid( + self._prediction_dict['logits']) + return [ + 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', + 'item_tower_emb' + ] + elif self._loss_type == LossType.L2_LOSS: + return ['y', 'user_emb', 'item_emb', 'user_tower_emb', 'item_tower_emb'] + else: + raise ValueError('invalid loss type: %s' % str(self._loss_type)) + + def build_output_dict(self): + output_dict = super(DSSM_SENet, self).build_output_dict() + # output_dict['user_tower_feature'] = tf.reduce_join( + # tf.as_string(self.user_tower_feature), axis=-1, separator=',') + # output_dict['item_tower_feature'] = tf.reduce_join( + # tf.as_string(self.item_tower_feature), axis=-1, separator=',') + return output_dict + + def build_rtp_output_dict(self): + output_dict = super(DSSM_SENet, self).build_rtp_output_dict() + if 'user_tower_emb' not in self._prediction_dict: + raise ValueError( + 'User tower embedding does not exist. Please checking predict graph.') + output_dict['user_embedding_output'] = tf.identity( + self._prediction_dict['user_tower_emb'], name='user_embedding_output') + if 'item_tower_emb' not in self._prediction_dict: + raise ValueError( + 'Item tower embedding does not exist. Please checking predict graph.') + output_dict['item_embedding_output'] = tf.identity( + self._prediction_dict['item_tower_emb'], name='item_embedding_output') + if self._loss_type == LossType.CLASSIFICATION: + if 'probs' not in self._prediction_dict: + raise ValueError( + 'Probs output does not exist. Please checking predict graph.') + output_dict['rank_predict'] = tf.identity( + self._prediction_dict['probs'], name='rank_predict') + return output_dict diff --git a/easy_rec/python/protos/dssm_senet.proto b/easy_rec/python/protos/dssm_senet.proto new file mode 100644 index 000000000..90471d268 --- /dev/null +++ b/easy_rec/python/protos/dssm_senet.proto @@ -0,0 +1,27 @@ +syntax = "proto2"; +package protos; + +import "easy_rec/python/protos/dnn.proto"; +import "easy_rec/python/protos/simi.proto"; +import "easy_rec/python/protos/senet.proto" + +message DSSMTower { + required string id = 1; + required SENet senet = 2; + required DNN dnn = 3; + +}; + + +message DSSM_SENet { + required DSSMTower user_tower = 1; + required DSSMTower item_tower = 2; + required float l2_regularization = 3 [default = 1e-4]; + optional Similarity simi_func = 4 [default=COSINE]; + // add a layer for scaling the similarity + optional bool scale_simi = 5 [default = true]; + optional string item_id = 9; + required bool ignore_in_batch_neg_sam = 10 [default = false]; + // normalize user_tower_embedding and item_tower_embedding + optional float temperature = 11 [default = 1.0]; +} diff --git a/easy_rec/python/protos/senet.proto b/easy_rec/python/protos/senet.proto new file mode 100644 index 000000000..9830d9211 --- /dev/null +++ b/easy_rec/python/protos/senet.proto @@ -0,0 +1,8 @@ +syntax = "proto2"; +package protos; + + +message SENet { + uint32 num_squeeze_group = 1 [default = 2]; + uint32 reduction_ratio = 2 [default = 4]; +} diff --git a/samples/model_config/dssm_senet_on_taobao.config b/samples/model_config/dssm_senet_on_taobao.config new file mode 100644 index 000000000..9bee96337 --- /dev/null +++ b/samples/model_config/dssm_senet_on_taobao.config @@ -0,0 +1,286 @@ +train_input_path: "data/test/tb_data/taobao_train_data" +eval_input_path: "data/test/tb_data/taobao_test_data" +model_dir: "experiments/dssm_taobao_ckpt" + +train_config { + log_step_count_steps: 100 + optimizer_config: { + adam_optimizer: { + learning_rate: { + exponential_decay_learning_rate { + initial_learning_rate: 0.001 + decay_steps: 1000 + decay_factor: 0.5 + min_learning_rate: 0.00001 + } + } + } + use_moving_average: false + } + save_checkpoints_steps: 100 + sync_replicas: false + num_steps: 100 +} + +eval_config { + metrics_set: { + auc {} + } +} + +data_config { + input_fields { + input_name:'clk' + input_type: INT32 + } + input_fields { + input_name:'buy' + input_type: INT32 + } + input_fields { + input_name: 'pid' + input_type: STRING + } + input_fields { + input_name: 'adgroup_id' + input_type: STRING + } + input_fields { + input_name: 'cate_id' + input_type: STRING + } + input_fields { + input_name: 'campaign_id' + input_type: STRING + } + input_fields { + input_name: 'customer' + input_type: STRING + } + input_fields { + input_name: 'brand' + input_type: STRING + } + input_fields { + input_name: 'user_id' + input_type: STRING + } + input_fields { + input_name: 'cms_segid' + input_type: STRING + } + input_fields { + input_name: 'cms_group_id' + input_type: STRING + } + input_fields { + input_name: 'final_gender_code' + input_type: STRING + } + input_fields { + input_name: 'age_level' + input_type: STRING + } + input_fields { + input_name: 'pvalue_level' + input_type: STRING + } + input_fields { + input_name: 'shopping_level' + input_type: STRING + } + input_fields { + input_name: 'occupation' + input_type: STRING + } + input_fields { + input_name: 'new_user_class_level' + input_type: STRING + } + input_fields { + input_name: 'tag_category_list' + input_type: STRING + } + input_fields { + input_name: 'tag_brand_list' + input_type: STRING + } + input_fields { + input_name: 'price' + input_type: INT32 + } + + label_fields: 'clk' + batch_size: 4096 + num_epochs: 10000 + prefetch_size: 32 + input_type: CSVInput +} + +feature_config: { + features: { + input_names: 'pid' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'adgroup_id' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features: { + input_names: 'cate_id' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10000 + } + features: { + input_names: 'campaign_id' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features: { + input_names: 'customer' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features: { + input_names: 'brand' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features: { + input_names: 'user_id' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features: { + input_names: 'cms_segid' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100 + } + features: { + input_names: 'cms_group_id' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100 + } + features: { + input_names: 'final_gender_code' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'age_level' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'pvalue_level' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'shopping_level' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'occupation' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'new_user_class_level' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'tag_category_list' + feature_type: TagFeature + separator: '|' + hash_bucket_size: 100000 + embedding_dim: 16 + } + features: { + input_names: 'tag_brand_list' + feature_type: TagFeature + separator: '|' + hash_bucket_size: 100000 + embedding_dim: 16 + } + features: { + input_names: 'price' + feature_type: IdFeature + embedding_dim: 16 + num_buckets: 50 + } +} +model_config:{ + model_class: "DSSM_SENet" + feature_groups: { + group_name: 'user' + feature_names: 'user_id' + feature_names: 'cms_segid' + feature_names: 'cms_group_id' + feature_names: 'age_level' + feature_names: 'pvalue_level' + feature_names: 'shopping_level' + feature_names: 'occupation' + feature_names: 'new_user_class_level' + feature_names: 'tag_category_list' + feature_names: 'tag_brand_list' + wide_deep:DEEP + } + feature_groups: { + group_name: "item" + feature_names: 'adgroup_id' + feature_names: 'cate_id' + feature_names: 'campaign_id' + feature_names: 'customer' + feature_names: 'brand' + feature_names: 'price' + feature_names: 'pid' + wide_deep:DEEP + } + dssm_senet { + user_tower { + id: "user_id" + senet { + num_squeeze_group : 2 + reduction_ratio: 4 + } + dnn { + hidden_units: [256, 128, 64, 32] + } + } + item_tower { + id: "adgroup_id" + senet { + num_squeeze_group : 2 + reduction_ratio: 4 + } + dnn { + hidden_units: [256, 128, 64, 32] + } + } + l2_regularization: 1e-6 + } + embedding_regularization: 5e-5 +} + +export_config { +} From 3ed7a46c30ca37dbafa70fd58b7dfe2ed6e8fc95 Mon Sep 17 00:00:00 2001 From: "eric.gc" Date: Thu, 10 Oct 2024 10:24:21 +0800 Subject: [PATCH 2/7] dssm_senet model implementation and documentation --- docs/images/models/dssm+senet.png | Bin 0 -> 83458 bytes docs/source/models/dssm_derivatives.md | 77 ++++++++++++++++++ easy_rec/python/model/dssm_senet.py | 59 ++------------ easy_rec/python/protos/dssm_senet.proto | 8 +- easy_rec/python/protos/easy_rec_model.proto | 2 + easy_rec/python/protos/senet.proto | 8 -- .../configs}/dssm_senet_on_taobao.config | 2 +- 7 files changed, 93 insertions(+), 63 deletions(-) create mode 100644 docs/images/models/dssm+senet.png create mode 100644 docs/source/models/dssm_derivatives.md delete mode 100644 easy_rec/python/protos/senet.proto rename {samples/model_config => examples/configs}/dssm_senet_on_taobao.config (99%) diff --git a/docs/images/models/dssm+senet.png b/docs/images/models/dssm+senet.png new file mode 100644 index 0000000000000000000000000000000000000000..af57ad2e47bc0bb6d442077d45708fbcd7a7963b GIT binary patch literal 83458 zcmeEuWmJ`2*Dh=}se~vXAh4Abqy?lm(jZ+TASlw^T`D4qbhpwWDJ@~qB_Se@4;i~Z6;CZ4itJR6PEq0q9BScova>jJy17DO~ zhaamho*ye4!IOBB0~DD^{S1<@kk)B+n$EjeBCd>e0XPb?yEvM^1^03eP?F;1)RqSY z6R!FfN)&UsYw_FauGb#c;7ql?5?Oo3XKFo!>W))bvtr%k_ce^0LR>o{eq}Xn(mX1k zKe5FA)pnqBD?`DxKbH$bm%gtFYBMgA>hd%$d`T~IVi zZb-UIdJVIn@Rr6q_;VNvik}Gz{PY-k z@Cy9|gJ0;JlUKCR7buwEKSJ>9oD4fV8ofLj?d&~l8GMH#q9Q6S4PI6BZ4C@8?ToGL zk!3B$-~()HDNQ>R6k=NF7gbuB<~O*0pNZ;y`}=bD`1GwTSROsGdThYrY+(&u2Svb{ z54^Q7uz!SbwlKG}<8v0gaeM?Hcn^KddINEMh`pKMjr($n2vI9r0|Yk<8w=YFAshq( zAz=H&kWX1m;`DUzpWqE+dwXj>R#qn`Cl)787AspLR(4)qURE{^Rt^qka0IiRi>3V| zXJ$(~>XVC{T}RBoPT$tV+TO&<5&>QJ(PJwId%+twpc|e4oV3%x+2rq?EbUIa1vVL1|2>oaHuvp+ zp8Aik{`aYBb_TYhRu-6M*etCML04vn>f0*LL&c~kuLkr;uu$~W^5RMj3 z^#?GH*G$9|RKY9oGUx}@3H)I=d4=9PGS%SKq@ti8QKZF0RGm>5M=*!4d^_0wQ`mLI z+5Ba~jbuF+1o9mNBN-RN5953HE)ozu7W>8!`Bm(C{e49xG>Pw-!k2GJfBh-0l=4o1 zQR1fZt@{N6nea<3hco<#MjUQs8c9r5PDvisW1EgfMuwwyTLK*(R-093a95>USO{+v z*nj*(b59Q!HjGWqUitc8jzH+av8>nbB2iG`1pn!eR4NMUvKStDg2cZafv9UU`?v3q zh!7aO{4%e(J=wopAL)(4lmBnuq4V1zdP8^<>CGSV zMIz(Uf9f0b8R$IOzkNsW-_HMch<{>?Z38}0vXA^Y!A_uqB? z-+bf0Am{%V!t=QYlUs}H-PpRtT@^RYWyDzT2q(xaKHUN0K|UO)fIo)~!%D@`7W62* zVBccCES<3~$NbsuA1=2_h^R{-x$JvtYeXRIuyx6weoL*jKX8KKJ3agz`aXh(vm)h9 z|Eu%v2Wx?jSW?TJ_2VKs&CF)Bf3aLgbOps1)OWI2lh+X@vh@!)m-d{y`fDa9+rI?d z_?xUe0!@E}pLqSmGRV7db5p{nL)fHjmHtQ!RQ|9yRa#Q-{(!&tQv>%?r*yn=dcrmI z{T>?5jEnyIBy*QxSXU5iyYB|(BYfLlX%3?&D_uD^QXLk-*tGB20Qg@nK(#cqqPLu# zWrKylxYe+W%c))ZU;e$1IJgh-HOI4t380(15?bCO?fCOn1M3PS(z{BLz4`BW-Ur=K ze=2!uO{su7%M?m>4_ZO#$1ef`Zv?A&Z#l?=H2duKSYn_xfyhUo#mAq0)$&H(|JdO7 z^SWXH%F4pyo;w)l4gcqoDHV_6Q}7j+ltKcL0gj6oyYe@8c}ak%qwx7mciM)RW#CZf zApC04P6OqFO#gI~{}IJE*Wn{^T5d0Z4gp9Irpac_R@7*FpLO^OZY2 zl)wrc`iGLr)ep%I`k9YTP<=$ldj8GLeU<|D4eNIei^LoKztQ6#>7LJ5YyzyD;o$0>={VEgY&Fw`_hstu zACzQkf5A4Jt35i%e^6g?(j_Zj|qUy_GxRnu*88LsQIm(&6*G~uRG9tl%d27J%Et%w0! z&H$Mvb2N|r>k2-Hu(;vB-*>Z8*rqiu4 z*}Ao!d6P?Jqj{d&U#MbdzP!Ly>CRAa+GDH^eV-JuEUW6qzw{x@b!RH7Esxi3I#bAF zSBywsi`eIZv#t@Z;MueQtqGR~0~({3MSSY8&Gd>Fd&vu*Kz*kYdTzRQ(xqDbU0rzlbBPU1NY^3m`|nI z@Vn(S>QP3zssSO*h*u6<1v{>+^NBC$*A#C2DK7iD8YsATEM#-NDHq5M%=ey<5 zLX?SmKJ}6nfsK7%DN&w&lFb78##UjIGFcvJkYBy2Om6W(NljbVb&fWSztU;tr%adk zc17I;;TUnW`9b)>-f~sG>+esP23*XFR@{+0!r(5L{||RTB62xI2{KTe@;K@05@Z<; z_gr)vyaHygpUq$bNzgExYec)xPyIm?ef%1h+jNXktNBEqu9wdHs^!XTqmD$nr3Nqx zSNSqJgpY<>?;MW5W(`kenCnWDHTF_b35yl>su{FxVeVzAT70}&eXuz+pXj!%bby6 zn*&B|xsEX-EDcm1>)%Zi9DcBmwDojkB)ZB-?{q5&>$D{b4uuPD4coQ3v|Nq}Ve#ye z9H^a+HH#%(!^Oo-H}CO!%v%4r=;&}yl1O?;z478Q81la1ewD$HKT$TO=H{x^t{WT& zvN-|HU`b|V+5u`BPEo}-B;X3i^U)RxjM7uaLaoZer9BEiJ&3ccG)YD)|8z<6+OOszZu{RnSNR=tdDSrkp6Tw7llb6`rAP4p`Fi=8Rm~3F zd)!)zthES@hu9GnOQl2FI$%~_SPG+85ZZZ?jF#quccaR-)M~Z-69zflYqvY8D=-ZA za7Q8j4*Q+8R?hDEXd#bE{vWQ}2UqNBtM4uJY1_$JZ~yYED&n(Vo|n_nEPO29`BoHD zWAlC4PZ?&y9rp$bC;Nuh7*6`4fI}GK>K*H3rl>*P197qBXMG_J&GmP6^=$*TTkfQm z(sno~3I?yeX(4Rba?m!P#r$g{L)3~1feUB<2Q&DTqkpC;&ovsYM7)E$Z`>M4wiqh4 zj7y}T;AkfjC`DDle!MW;pO=`4oI)qj4(76EY>-0XH7#6{is`8{FBO`PX?Tk)Skh1} z*jw>Ba(mG2Il{kKAUe4-0;0!qd8Q|eb~)SXogtDvXW)34D(UxZ3~0xgbjie+y$%+1 zb$A7HgtDb3OEV95(4p{$>=<`)7^+DVDmCq8EZ?2aIS_4gWzT2FUfkO#vl?Uy%q)pMorvjLPI+jzbnkhJ!d^ z^0k+S9r|=WC5t?JsvjiCO*4b(oMI@s2l}ckGj`wb`l!d&hz~q~XULqhFP-XZkBP`r zlkW5w+b)tS1=zBZh)!+7UD$Av=WbS(R>mXjK!Mf9F&|7ytk}3XG;H^U57#X*DnI0R ze|X-mlej%&wZ~B0OM9P{v(iAlN;|X>NkLB>4V!2XTJzW;voAcbV=UR?Y4_lS*2v3<-6@A>I9=C;p@N)cPhiYw{Q|-u#6@ zlgt1xSTW+9p)+mxI5E~j%=3hPf|2oQ=So!1Fz>9WLyPfXu>di5@Y8&6j_A~WWm*8d zY%2PmY^i56`{>c()`&}U6pGU8q-pUPRt2VaYkq!)BMZZmCI%c9jX!FOJA{Td zYoNmTW=&wqC6Bx3pi7Dbz!b~R#AnE`0yzwO1VU=ig?lmxyWeA!O`bap{g?Fc&ptIovh+E=^tl13dJ>=d>t~C< zj?W#8&v-)HdL;Kgh{#m9yc-@?gwsc^MFy6-LQFBcTVN5b+?>P+8X%7A8?LP0`swy|+_$Da(PcVDuX^K$-I8oI zhY-im5dBZ$F2< zB2--*sdAcq;eYqR1>h8F@np!7PXj#-tNTZm5?Yz5^^y2g+8Mc5}Z|8<1VkRwcs zpH+lj`XC8mk3v^=LSY#WIvmyY927_UYaLaNG*+(Kku=r$6OSc?|7JQ* zp}Y}x)|z-|$^$6DeUtYcqIPd&rBZJkIPStL8o|1y(J_tEM>b$mI@i5Xy3RBJ8$Ns^#4~LDW$aG_l|s&gn*HBKd$KZ@ zyl$>@(N)yb%g0~My`Ie z0)o5X{_FDj*jm4VfvF`*%a(Vp%|kS_WWq3VoB*zhl&}kJ&Q@@ZatRqq7V%+p}%KHRo4IbHS+1& zw_BhOQ~;+I04!C$`DAlj0RSqba$2Lg-I2o$kC2)S7BHbxA7+_#%kkiP0$hO!-Q#zU zib70snR)SydzUnQ8n;@>jE$SF2JX`rf9s1lfJMG=0sRdYM zK?t?6c}w*-e;Po?vNqa!#Cis06akcBljgn7vrx>#xc=;Awc8e6x?Egj`fLY6Xtj~L zdsB8ED@)Rly7q8)!DP?i3j|}IMtkmFuKi7F6#uJPY^u5ahkTw;dy=s5_9uMLvH7UO zJdpB!Djl*U9|n2Fr&Q^%R-u8-`JDXS5gglI3dY=|Ckq+OOb+Q7?+r@V0m8R<{hZRk zuvl0fpn}K7Ue^^?=SIn`K3-a^TB(_qi1e%qjZM|afA}fsa6KbHj(C?0jJ%rld{#y* z>9D6((a$`csZ=!lOrF)9mzLd&Mmaw z6n)%lB$%UFy}#Nr52gHFH>FjBC(v zm)a#V*eSFpmeH-~I$dNsQdweZY+Ml6O_>(ZQ~Rf$JY%V)SH4teusOALxWX<1FPl*? zjJGFnDJfCM;&oMn`+TXd-9~rhuqdpmlGb;w_5*nUu*l7^$xI8 zjN{#Bs2LHg2NG6nN`cD2<7^69jRUxsz7Wns(|bOYx*91)_Nu9%a6yOI4F{Vl5@lGY+i{DgEhRUwi zH0fJNe_b$l)@?8on8iIk3lV$;v_zWp=C1!~UMTO4j84%*_ldnU=ApwW+T?#u4-pZ8 zP@rT)K0QiN6yh}Pgc?^(f zyg2FvnWW(UT%6(YXN)mIQ3>>lXeyokblNcZ)NVsNn-hG6kIylc@D$Vxn{(i{reM2t8ar52`t!J_BWW|ClkM1R4sVdi{ z(_gRCk#-n7$G4`Uz;+e^)J8rhzBPi~uQ@EaGO9-5r5IJo?@HFHa-R77t{X8qilJ@a%7QniLb`W#Vd{Ef4^7iGQP*<=eH8(1y(Zx!blM8! zcfdXSt0IG|j3w~IlHUQ!-9kA`H9YZbcu{;}0m-OiC)$jC!fz^i!4B_P^15N$Z!HZd zZLEk_KHGKUvaYuw3ML&Y(st^5ywUvGsxPL3qVM}V-E*(81@MI4D0lYeX&;_k3XUHR z%}X`;S>{uGOB^B=)rf;NhX{x1W*j3hpH81od@{f*>!3Yrym{JWfOMX4m)&BmsfpLy zI=X*fGCURn=vzdHOSkFTxkb?j&qrb(3zsxiA zt|uH!*jO@o`M+HF|GZQt6&QvM&L8o|okHSbH{ud;G9Stj)6OvS*tF1m1GO6@!V9%& zrK6)~znbA}#$u_;fLo`fnqVF`%7+k3O3$tJ>wrryjZZYFi?rq3Iw^t_T?nYKcN~0U zHZ`-@KEhv#HdS%Uy>M3Q{&}C8SwJrrzurD+Ino6E-P2o2%&qYpG^U(G$20U50mI~l zdf-NclPsU3Mb7{lpt}K#J*T@>l@LjY7Emhcd5Kn~UKn*erFs&PLb`0LYh|1D37n_y z?&Ev`R1UVbJNZv$$_?})bWN;0vCzUxpfkpW$UN4o(zv~kA2XgC8V6*rI&7qvCqtvC zgNj|C?bR{8CGueMX_afX}Pm9GX3Pnag2%|e^L_%=fRZ{Pp zQUS!Sg{##a-Yb36-$PS$@U1V(&r^PgN{$?%C&RHvw=mB}UFkOH{RzgXC_*R(;;_Yi^msS4~AATH3GP4Po|P7AUv8~*0VAavmmWEswCRhKD^^RwL5RfggxEcQ-(g6q=014pxFaZhcd7_8(lyRb2v2L%(&9q zR?_!S>%Ddb$0eWrck;o{ppFpY^y^9Lcf4pMmx?U7nSJb~Fa1%9EA%-C?9omqy{QBk zK~zyqR@7y{h)P3iEfaFuOT16ZN)V!nOMW~sJFucQJGSz5)VM$~?&h{8GPm2$H`n0M zHIhrw7fIq{EEP3}j5-ER3q8PTp$P76R1GipodgLed)9sT#Ec}*2e=e`J?8TR#Tl-E z5#Jv%^cZpIQv)=l+)z8l@w7?9GG9BHAivS0+m_4M?0}S5;JK z9_)=Csjp49sZe?C7tZ%&g$)`+>8Os@coc71etShU|BXafRZdQhj`<}#0VK^vy=k&h z0Sz92a4`kdiDRxD;NU9L(VFV43Ba>(ReW~aT7HiQk}}$g4=PcQVZ#9atFTtDhwf~F zi)8{V_fT56%3&e>hqhf?4_#};N02TU5|uI$esb%0?Vj02A-|K+x9H zHW6?lUTRy*BXU()zLRvZuZfvCENWJpnd)`+9fxfsdjZJq8DHM4 zct+v7hEbpR7IH7U#3ve1%pzAr{~z0hwX+opWzE#by);%G1okg zaCkymr8Gz-;9S6DwNM(w7fd-@_mO#-vTVqb->G$dt{YHw;+SJPF{1~|PKPXIo-+wf zl^lS8w;Nh|b;xnOQx{RrQL4MjpwDg4d`0W_QX!}|O_#SP@MH2JDQEQ@GbsN9XfONF<}Jz$wJ&d5v;Y#$JqLG_41`TKA;V zhHL^yRdyKnr)Q+dRS$hS0djmLfZo+YiN%y3^&51hO3$Z;$#W>sR=-R%$;IQFMWh)~ z#Ho^h@xynjD8%QTB-jA?b2p_a<;F$Ju)AMIGmUQK6u0xqHsRi&ec!PCXF?9=Qf)x) zOxU|8Q_T^kLl$LfD<9Vad9`;vfBBMRqoQGC=GW{tr#`;aur=y6y4dIA97ZSBE>0}C zHW(k2YgPD-Sf(pty9JO885nriX1SpXZoOHU9JkHT69cG13@SD01Ke(#=Dq#>#y@Bi zUanh#a@ur7;>deC`FP{@Es%+*Md^B0@BxNZ4u>R5tE+bLL$`c-?tubkuiUFEd)kyi zeDe$rfZ>m|XSHQ9O7bYJjdK8oMS~)ryu)@UOLn28C`CM1#<3njkp%R-I}^+DJ}R8Ez@vwMP17xLSrkERhQEI!*>bG`rnro1VS9+@viw9(l5Aw zf6H-r(emIi9!*>Q__x=h1{{J-Mi^T=Gf6AEWo_I;aK7Mp^@MgcidpXuOc~EU) z7k_sMv?)_iKm6?{Tt1O{w7LSQJgkWCJ0EmnRJDX@TR=FXkgAO8nF77DYoN0EdMT?4 zjsMwDX+GMfi9tUA6&uuu;tvLq6r4-*Myl9N`}E!6BS#}#g~dj7efA>Wm}4d~rSls!;nnOWFu=&~@llBV7o)HqDL9wOYK&NY7imB!q3e*Fx#G_-k{3Lz2=ZRhOodvtB1g_h4S* zt;a51lx0#)p^5Dl{}-+!7ML;oICvXiF4-}a@kGa>;72tJ#`mwxH*A>oA*n8x-N)2N_PZ0X0h7m znQ}QU_BYyUXxsR9@dV%AYv*4PP|0VW&0q)#CkrK{8Ku>D6$&VjyK^6VWIA4Pt9gRb zC~1dy$1ROriHP+0)b(w!#A);7RW0*z*-lEWx%cTfsqqz9{-8bZzcVK=Nk{RrohWZT zz^Zr&L_W4TozyW95>1rfH|~n!3#P8$v(V%^2tAY*HgK6siyCG(e72Re6ju|H5&D5H zl&_w}^HnGyA4KNj)CP3`_(D=(^{djzt;NbQhC-UBXT{T3j{R>O0pvUn-931-3dY&4 zuD?x3m*<_j61J*5N-eA7G8JLERMMB%v<3pPM%&IDo6@Hl`Z_hQ8rjcO>v@Y%C6RFOLI+ZL00${4F_mVB=ch` zZQ0e}cNB;tnWwd|N6r zQC|;mD6{zfPW;nkQ`m@I9OfQyeWxMM{Rt|+`S5U>o$?)nO2;KKDLzuKKPus~C}p*; zp{S&U>Sl7srPR#cn-^l?5k9cVY5B3_?^9o~wv(`@9^TJ-ofvK?q}6P+Y?SCi=iu}> zQTAhQlSjQLR{HHarYdi`|`drIiQz(C}W_gZn9t)4Nq z?PEV&zV-qzq!$}T+QG5L+kIp2s~GXlv&M0v~!3_{8f!&*F5@~8yIL1~EQlfg{r z{p@?8jaHv;rtBv5-V=$B?ZSEK@(HG;BY?4#8KvpUZSk_I-iYktNx0ns*v2#fy&wN5 zKADqqH}1FD5|-NWP=WtC zH<&5!0MSFj<_?Nj&%27>w_Q3`(zt|>?XTx_^Z2UnV!SjAyT+qmf6>cnva;sWTRw1u zUQp*W+gt&E=Rx7>_QLLq*D|B=-*c1PSDWU$WLe+jQV-SJVw9_N2Yv)~t8B3NhKz)S z-w1uC9l1PGl>-RL=gm91cv1!0upuq;Lf@TlZU865v7$|)G(|A)d^EUJ1udcgt2XQx zMId%}7xJw(o4i6-zjl10aL9^biZ}2*5zZ-6!{DW6PY}GgP~4Y?H@4}}qqUi+-%Cz} z4J&00*`1&| z%51+g5zx5YlxrS$wAU$$uXeQEAl%j5#Pe-!rh{ilGKliMy$oToCx8nvtf-Ssa~Vl_ zES0l2ighyH*wjNsoJvqCYIQskj#07_G!S^AjiUp`h7qJ%q0~q{eqMd?lU(34Sl#g4 zeqZ#7!~(S>VNjo$eznjo&o4K3B|6<76ngG^Y*meNE!b+;dLD#pv&UCLg%WLt;hvt< zCq)tWehdN@TMejalwViQ*rpZV3lH0VdXK;iz<vK8Kf~ z9furB_m(|qlOCn%S`2mf2B|DuY-)Fi@JJbJOiziK`EWLyt&o-ju z-rXUy{OTJmjj6osQ4>y}&z*2?t+)LzeeacrHQ8;}=sDI3G+%T`m{ygJvz@BA_w}-y zf7J)GzIN96A1RAd*?&M5p#g+)G8@YyS`?1`k4!Y&t8f4(;?s=NB8ompa5I`;{fN+RFAm5Z z1mgylx^D`=1epewwln}4U5q^9SmTM=u$>-@#IQ#WZ77Uh<=~(i7(Lu9Lw2}3O@URe z;)7>1#)0HBrj$`4Lmk^#ul4Vec^|C|S{hR(X96EUYpur2bet6c>Q1Wc#d4#!Ik&eb zg1R@up7FnVj*CU}P;c{N5L#2d;Ow>^T9ZlhiRK5XjNaIm&=Zm}YD4&2SZnaw@G9xg z(L|GxpQ6o)LGFf8duB}@rSAesZyPHnqg9tW&njVNjj$662k$+e8_OLsT0#HisvbW*X&}-nlIwDVH?NS|yHnYHC)y zIt)6`TD3g`8Sabb0s38*4xhw>sCsSGz(q;8S3U*hQJ`o7N{8g5|Z5AQ|a5NPSy@v zby3Tpz@^AAdVuLl=U^On?@1W8de-Zs9S?w-aMxRZciMOk4Ov!MBrdtqGLgpSh)QKOM(tpZ4XQ`F|<^fn5NC zK2 zt@ZbLJ^5?a84-VY8l$jx0+M2AM;K|bssO9u^YC5x2~hiTbCVcvMXYAt zcw|Q<$0rLv3Pv8;%IRLQjhe2R)I)q z5N+g3OmjN6qdSbec=0`~I5hILuQM0-P0c%C|7fA%@hYu=|2Y_|Y#l812$eR@dL5qx zE@6wuM`3DOsF(A;)OO0SBXMc7Ws-X??7iqXetzv?>Q~ z9OFb9+K=eynkNOdbZ3!3d5=_s2BL|DGh#}sCKH(jkK6*4M1;ybu8HzYD$eAPxL0ZE z!6&BC>G?P*iNoKeBL{u8heWf(%FCEVg6foM9y?Q4H!Q}fg9OGs>solyeQ&L{d_H`9 z?T)qH4^ab*l6A1OlT327FiBuQzJrR2V4tx!xiUnJ$Av3KtYbB7JjBze{3*a6hx|MAs> zrz4)NVtI+Bui=dcF^gHKh82WAsW3Hb0VlEeYg%ch+tC2xggs?KfUJO@qdF&vm}_Hx zRn^T#=c9AsC>)Ti1k`{_rM-GF#`l(w#t!|>dpJ6iel_qEqxN_c&r8z3`A5Z@oAEaq z>|YV?(9$?Gm310{rmHUw^qe#f)7{qT39`I zFu1#KaE@%?GTb=YZnO^oa?+;$rmIo}y6#2HwlOj5lDF;yz0E`b3vfkwUWq%w3&I@riYfoje|h5-6(C%!8BajU8vvCC@{pa z(;hiDP;69R(@z1^LS;W)%`|4zG;|E`<=pn$tUM>0=g9eAY_{Z@4(0dslY{-Of$B7!speIn72nN7EsE% zH0Y{kir(d`s{^5g-$1&NHIpC{-v`~qS<-fyj{Dq08@=78i~WTr22Y;JyS!b;rD+pN z;1->8yIWzqE8#cqjfTh8;qXc+O(mhy!%-YFKceRIpg`~3?OuS=7g@G4jXanH!v+QV zwpx{431fyxN>+hD;-<%%Y_V(ynTFkTD>hO0<)!eNm8dSB&f7y9ldMou?AzQ=@2XxI z%J<~E$%Alt{Krl=6Y9m@0O5;3A8&mGUT&O&V1=Bt@WvzuM3OKHlFH4IjvB_rzpkkq z3h&;cQQ_)mlpZ8ymDyEfZwb z)R|FGxE7#i0qASZk#-`|Zl6`9GPQvaNcr&drbH~QP z#~y%J^EXGmyi_Ef9=URtbZzZ&bZkVd&~4{+`^UPj_fE;QOCKkyyDySuGUV77eD^a{ zVi*&chOdPNd0cl;0UG>fCcs~Rtm!!>v=z}&g~~Nh6X=eYO+wnbm1Z$ql7#mK$eXZX zKAuvL;j~uGbfvemzxM=!;!6Eh>>_pQI_BaZTLVT^-3ke-Vr>cheH*-{-?rQV4+g7pfCPt5HePnidrc0WFURbILBt9z9}2q zQa0H&Bf(LbwRXQKGjS7jdZGJ)X};} zU3kr0s>{!8HKr1v&L~b?5{^+@-eRKKGv6r0M};#&Ls9he1H>O8z;+El?q$<;IFfp2 z5gykx($TSO`__Nk+KOXcAsq7rO2ZLz!`ho3-C}801;*S7rzzWp(@aUCLU0Kg&}M$1 zUa8(n1a$BkW%b-%GnXpbR+Uc#2|tvJ;rJFF zr9BYZp+)RrC&i^#hq3|Ca3)YI_rd#ty2ztg;ZU~i6U7FtFH`&@TB^N^enq6H@N5KV zI?n9v=^4izF#70;3vQ<`)2aBn59d*?hWQwdwYzBwK3N9merw{}4` zHf(2YCQdH&mDoLlV--AED6|bg$Nsyx;U1u-lk+O)8H-u6^p>kugo>As=v|t=Nf6)M zYtcRI8cnbhq}q#*Pz2-(BP1<*+;jVQgM=a@)PvAeO31ZP;h$mfiZ7G`ISM{two3Cn zM#~Llgs5ogXQ7&3IP`Z?f`mBsXmI`DPY)`WM3~fb6XOhwVDD4iEPHnj0?*J6kyA0Y*p0QtGBd z3kOKpOJ)+?21l0JyGbriGz4h?txK5i`scMJdEK zHLow{BJ4#r6zN&t_O1K56cW^}&~>`xjAR6M`FykrW3#9RHTrchd0RUqPTeM2;5 z7zlP*IX@tcIT)_-a4P}ZAvoP!K<&cBU>_IMAfY{BhLIJi6hOudil%QJD!!+rnupYQ zt~SwIjsivId!Xv$w@ZUF=*wcw@ zVX3;;WHUg#!!Zs(ff^u;GU6;OTUtm+yBerKO(8wNE2Y_e(LxX+n>Qd$lh?)BV{I1l zxwo9;jjBmB2mPR>E;af@dr})mb22>|S^|9z11b2kS#&BHTv|U&>L<Hlr$&H__Rn8l~UM(f#(aBdbH|kbaK{5e$3Lk&{0SO?}h2M8h0P`Kxy+`GRMrb_2w*rGx zPOkMb!uw!^h3CbzY3_Yo+=LTdO!747Yu`SB-BWNGq&wW4k8m!;9mR6-J3&(W_sgfv zFb)BON&Lr#?eu0>Zi0v{bqDWwf;8%a;>)3W_=#cxF$b`En9$R8Drn(E@p%HYi1I(2 zziWa25TM>(dYMbx{$vB9PbfCQBnv4m(V2zt0IJ~i?mt#wDVCp$F&e*VmS2}V?j=I; z0=T@{qbt0aCrulNcCq}N5M6=p^%eWK2)B)>gx|4=2^exvyvV~NqMA?s{U5qShBw{^ zzX$t^eSj9?Mfigg3uWF!-79-KY#f_YWol~7OL}sPY-k5@_Wo@MrqSrurBjD+hT!ME z5hM%sQjR}UD29uBBb+qvXPs51pO>O1F27&;e-ks**L@*+dpk( z0+3F1X(~heixo+LffgP|^PX;|0FtHPe@K>ez)s}6*N>)Yz?nXg7tR)y=fg3~JZUFt z2{?h`E3oFDiOG%q!NntIZEisqmxQhUQXz&7IozI&2Ot|VC=ns>(d)*sr_O)@Im1}8 zt;~r}otWGk6#lU6>1}#uIZsaNg`^xhUJoEW(TU-Xef2jS%hv5`?x|(~>`(@S`Oh87 z$GR(Mj*R@BBcZ^N_+us{@=t7WqQe0?Iug*VHPKe=lZ$me2G?SyOFiq=*AJ>zTUeq!T^9kUY`hK=5@h(1m3~1+v$6$0meYk0=8M8kPt@z2sSk zUj9#6j88PQ2fPW{3Gw6q=(W#2Ji09*>2lhs|6qa^Y_K)Q18(oscE@<~8B|F9Br+9# zau@?x_MLq6uKX|iy$k~ClnGAa^*A@xvspdyY}~j} z_jsslC5vxDxEP%6GYE;K$R`>#A@}5 zXD=fN{sJ(-KFrhF?L<7AYRXW{$NW`X@5itq*JhEG%Gz&Kc9xX8&~^=<&*+O0#=-;n zugM(Pw$vH4gQ-L5*ro!-=co5b>}Vf!VFVsRX2iY%w7hj9DJ@t%amP;=$hc=3>Pqg? z-iOn#O1m15C{z0dNdTn+y^jn8HQv!04bPgipiAiH=8L%{Fnxc(B7L%zdH^gLM%|C7 z09CA0*Y0?uTxa8LGMXAOkiRuH=uG(>uM^2gRN?Zr2xIhdC4oOMZU4ya>yyU=2R*nz z{wv#eY52TK-4U|UJY&}#*ZRjJRPsqWDyM#5BgoA0P?u8a{MQ)1m)Szi>%!1@7oiec z3B{OQXr|Zdcf?Q=m_au&&$y3Il#r8*Qk=~AQceXr9~%zNhnLBEOB4A+?J6f^{{BU3 z20FVR>;Vs4>g;5P&uc-g=aTlQ#k;d-k19;XqrsiJR{%X0kG;7_<(r%5c94f{~uQ*qS(8-*+)|`=1o5&X;zi3 zI}D)Xk}i?(cAQ`wo0*ulIw%h$Jb*ZxP@pg4chQ#!x|FHulfKht3Os*$Ke}3{_)DvQ zgp{|YBJI;uGN=h(VVfBn6;`>m5DfUkw>C4Q_6&HK^}sn$}}N?+@Et@yrvG9iRPf1TK#71Q=j~hi)Cq zr}9bx71o};nPQm2f>snp{B7{mgy5j6t6V2FYF53Y$dK5WY8_Wy(_CiMJh7bje~v4i zKBhoKn74`5DFnk}I6}K#oZu{iP>l9ILJiSW^c_%wboUHt-U%Tx)>YxzJ!&m5ZZJac zdRH0jETQHZ05{aAk5LDWd4}Rr##vWD>H~xCBeqWF{|i`PIs+8(kh18DQ5znW3rycV z!7ws!Om6R9LQtJ6hr^-qsuYRTogBa? z#Y`U&tG3+Yh0NJVr&pUxzv9SEyLmmh*1N5tEoyxyPU)CU0PzZ_TnT~SjKlO<65huG zYU}9`QAWlmzDorn%k_!4rd>CO7jxxbBmfzx4-O7wlBNh_qURY|vBV&j#Zo?$$AiW! z2iT)%RhbuhTFkLMeFWy=OyR6o&lYo+!>rwBc+QCvSV-hy47yHTX);yWsT>M|-0V-CNssq}7Uf|3Gd~H4N zK>+Vv>a-h&h5}M=zJ5G`0EQ3)@$HUXbMRffGyrwNk#!gqe)vo8B(C+Bz`2ysq0V~2 zHf3%rhxLs1+tv3%lQh)0liduh-A2iE#jW}m%yDeXzF8Ez2jIy! zSu)Y+5#?`FzJR;X0(2U1yCcF_jSL?9-VK`Idr{ztyTBBG!PYmdZ7QsoCDi~#145A! z)lCh(IJtR3_B}XX3$|%QmnCg#rp+5SiM`6!prn;GuWr)p_^wDg4-`dq0&QW3wrct;FCuH_IwaK zUZ+L~#RJ8Y-Jv)_MbKh&uML>k9F=@HvRJRwxTqoYXJ4fELHwr(;}WjCcxb;4&?Ewg zM>OdSC&0o6uIy3CmD!ZBfx9#T5*+XZCime#q!(NPv+{?qz^g?nKf0PCzwj*&mNhNe z$sxk*%^I%_5}ooevhpQxe{3Qs`=q zOlkih(XaU7B!Y@_+9JRfh-B}C>mYz)U`DBAW243e#W8(1eSC)(8(JV}!1&63mZAk6 zypixvy=mL*dH>5@Sw#rlm$ef zm2Mqf|4lQnn(R7v9SuT(VJVQ=2%mqL_~?2PP@O-885R#w?6R77Pb<9UBHuIs+<>vtc=^XKy%zvFP-_Zi>w z``MrOdc9w75Ih(C=)qR2V1c{CMHX&_ap!wHP1}Ii$fM%Xy-pj|zyNry9E3`CWg77u%ffSCwPJ;K2=og>hX zqXwh&SAgx$n?7ZZCY2q;((C6ZT*wKu^&0({Sig8ow=^9%j50Ix($b_5?4cfj(X$Zt z6xv}u*Sa{}VUm4g-=TCwd$Kla_HC|g1R=D6WG5qim5OfC!6v!V@pPT(Pa?z5z(|kMX?V;(wP* zD=P=Z0T<(PHa6D2JVF?a*2Pjj0L9b0TYcUq7Df%b=9L>g0=WK_=K}ze^|EAn6-|Sy z5=_ZljlAMfr^j5%6yk??Z;eX20NG3)p!jDVZ}kjTeacGAL(WVihsVvIhau^37lxiF zWN)Xjq%R*(8{>XMv?)3YX`h;0WvPmaN*6(SbH)8}QQWJFM_AqO>lJX5v0vdWJZ@5> zEhs1$&(nI?=c^Q8{?#6<7J|PPA@FShC+|A)In3|9Z;DhcD6(KCW~Dxat(i{_&21%+;<6aBW*Dl4!MDK4#jD%#{ncR z!mD=b(W-A}Y}xR$&Vr(w96;ir^rD))(B;{C{D*9KZD!kx_sd(ZF`ckcW?gBeG4naU zs^s867@$-Dc*#JdCg%%9Tlm^iZf1q-hqUIzNCsrGh07=g2)SHlwu)l~t6 zBD1-*+((HcN*WCAl+3CA9n}VM-#aGM?%)>?5ES?4rYJ# zkt%Yn()WP?ab7{cHaMAd$er}kO`d}pbP(h3;Re9O-Ufd_elC28gIJL8%e8}q?jw)B zJS9H9Zyb0+K!UXRVy>=XIB>7N<>NGM29zZr^yTSRU1>gRuDYnJIf<1F=c2TKV4eQ72y)4`tYjEkI(- z6S1NhH`x;giLcS-?#tt=^pg~YSAna2bb@73SjkqA ztl>;zlqDcL7venThw*Mv2Xk0K1nrnk@<4G>mC&>}=*25Svq>n?J=Jj(7X$wv`6yr~ z@WjLgPQAmpbdlY{jlNBIG}Q;ZqV9;fm#lopb>MP+Mj{M2_f9}dTkZ!Ak+EJ;mG_%^ zd|=sHQENB1+CVZgXzxN|O&w8`9QG7-&UrNY-I@Rl__9At9z@$AGJF z-!AMKXTC`v<$Q}d{q;$d)Zouw7iZAjhtjLZ3JEsAq3=W+S-jNBK7VSehWo-Oss$${b7}>?gZBr1$d$iE7Be*kKh)(&)m;#oJ7@Tv{sVuL&ObFhQw_Up;`Xvm~n4t|JmaKc=PnVM-ST3dLF; zscpC1{ZI&t?8BGH5o1qaf1A9(-q!?(uj!OI98W{v(lmN^S)u1GrNkh1I)k{y z1cAU%G9I*C{;9EJ6i$Hy8v;d3@lExR^tR?1yy^Q568LTyF3&AlJ1M0>h7OBc!Dz8& ztcA5xM@jP_*NakZRk)kEtyPrJ1XbY0H5KlhwSeX6#B_SJ+i7))dw7&DGr!)q0LZi) z6u)w-g0rh=#r3JfM|!Mkj^{wT|UUT?wmkc0RMvml?&L=SXH z%|ZzO6fyewLUxIfIWP&$hH|8wP(xW&D{ytnkpe*|$n3-sS!(4gRG`7R% zjDbQA@;trQ7VNF;l^T{K%I-l{T+S^&Um(}Z{0edPOd#m6x;@vMc`K^HE3B<)UfB3j z@^(pqo^GRG5!}n|HGQ;j!Kib=1#>Sdivdf{ZZ1JeK6?8@<4{2PvMQ$2JGK(|z8>|1 z$W8;Y1{F^V_K~X!3X5wEJ0@*i_tL42%qoF)1_E{;a#1#18n+?bt>xY7&rautHGaIc zOp-_~l>gJA;G5rzPm5TL7RitWWuO^7#iIL|A8u@;fFBS>*Ob_Ndtd<4b*w~R13!i9 zGkJx_GspdW<1wR)Yp;;@er9V~Ye&7zv=y$7Ys7_p60s}oM4bsE1udpRBPW6E!VDpv zN*PjjiFE=?!6-b&bNn7lq%!ztnIF%-6S)ERX1*u+`r!SQJk%Ug5f9xO*>J(DX2Y-Q zOgF4{O3TmB#9cT$>lRU}DG1*?QkI%0wC^NHZ!dy3hZ3<>OO1I>WD-maRFm6??*kDx zp_JDgAbokgqMA*u!6J9tt!F;W%nTTLVzyQu`4=KP<|-)0LX9Sul8dBt^Nzsvo>$z5 zoK}B9S)y2e+GgwXpZ8ihzVIB~H5GvBP#xdhL7`&EDlr$>_zXX7#^iz3FkAd=zX0wh z^`;OGehNjalWoe5@ewihUJ&n9sgK%wE*4-1k`Z*m$L1q{+Gu%4z($M5)49i-3Kpv&oUWwA-Ag|5{*7{+$ddAPSUb%Pa5%{2WzS^J|6og-KQTSC5qd-|)mQ3Kl;k+F7 zWeISCMUZ$_B=p97PxJ!og9Jcr3sgW7bH0%MunhHax#YVfB_-U*ZqHTBd#VOe@yK(? z0Yjjdelalb-7(Ph^XcFe)o(04*eTe1;BZn&jv3m7+DzYT5&r+itgS*oZKSAl#NtD4 z%=Z0dunT_ZUu5S<5p+7lk))|NW-mrTU46l>qA?NUnCglzqS?u1XXZ66-09P z1x_JvTM}+ra+KKZW%@uxqIE`6STzMNAh7_(>0R+1f8Eye2Pio)4=+dm2%Rb`Fs(Oi z^Oa;R=6!LF7j)b_hDse%WJJOlAL%=Sb{Pt7POE3C!Wp0JL9Xzpd+#h90jCnUXk+W82^w`iNVX9mIZyoN_*`49$&U+NhmNSL z{`9T1=PHjOLHiwr)q7$S$U-UtlILZd#0H-lHHRwV$;qG+46XIXS4;&^(hC{-` z%9eX&QmKYW|1Aec!~iYcFuGA8(%-ln>5tU^86|nXpM^n=X%M`JVJlx>iePQeHkyXY zI@PiR$A%=65PTmLxn?W*kBHPI@;)5Zx;W13lJf++&cobfzG-FBkc@x_;6pn1?$p_3 zLfDq;W`j#hY>(f|_mG9;gLw&O-B>Qf55>+QI0zfPnsP7^cY*ib##igBUXKut&oMnprHGY>XA1Pt{q`<<(@NSY2Q^93!Ek-t4 z)b(w;fHb5Q=X>@0GiP`1)Qee_#hduPkZ zq_7p*#TukXRt@o#c%)t)v9U-yceL;q!s0x{cs`7tKF^7Or&1D{W0GMbeWNnj zd;E2lE2Bx>7EgASC&i)w1}nr{JUq2M6`|Evrp7q3OzCqN7t@TLkr=$}KttT^MtGfY z{meqK&Y4pN4oLJ~CAW|1A5aL8WWyy{P_&8X36u@=lD@LocA+oZG1=DV{&_46KO7A| zyiAQ*-N1ZB;be9d4~NYH)&;AZIGQ5U(Wb+OC9ov8QeV*~cCl`fv3!yI5M z_8xB2i#|=6DNMoS)QHJfamY^He{f`)kdh*9N8f)j*GMhB{*ywv#kcrlfmd)RUNA70 zU#%%V{ym()OrlFp#{~~99g&1+6|_mpbRsf<VXg+wVJ#Q5u}Najg}@^E?`DRXVuOA!fmHH4)4VK5t-Q70f!1Mm6s z=8&e z(Qg7PuqUfuzvGYlhp)n*V~?aZ)yJ@QUi;@i50bx}G`-bf4xb?GQc?G{mtht`>$^)r?Y|#i> zoNybLhxhPdXDF0iD^#6($W4DPPoNC$>XtDBt~nr z?7AKrG2A>Vjl&$g8Y^g78wuc1mWKthXh>R<4IFVZ9jBVbm9BvE5{)woTi_ejt4)lC z91hbiuGl-v3SUdW)FzuJ9Dhr>cJ%GN-3cL;Co5i6P|##RZJbDeopmWRA7l?E0<$L8 zbh2TLYr)_tM9k5jBbm(Sqa)L4QE$v;V55*}=?TD}R!OhCB+U9=D90#%(s$*pYu^RA z!}0!8zZzgt*K8clwzOjh_&z-BKzzxmmJGdO(jRjbdNVQkx8CF&HJ{fK5qCu=An+C5 z$!fWVJ^gC%*#zsi{%O{2tEKCz-_pW|``K%zVqQHx9;(g{{ASk0C6?eaK{Epg8Q3QS z25)sVMJ7?}yPb!&A&s+`Bvc9rXS^ zi!Kw4d4u51)QYnk)$W%iQ*Q)Rs6`}oMxp6|167NU+DKJsWPcRe#`NLSRkgBina}eFe7mdI%Ldv1ed;ed$etcPExKhZ%M!3r86oCHMvH z=)&$BFOe6wUAEv4`S}b1oF_s;%|p8bJ)YY1Q~WT!l!N8HlaY^_h6e?qC2-7DmerrHFg+oQ0lYJA9icaYB3Z;fxap0tvhln^2Hz##*ARg6e z;t)LT&-DR#RK(}B3qtUkQ+76I7UFm8glgLygldPn9M3#hQxWP*`K~G?wlo!QP5sSH zeIKhN=D+=@TVtuyf@-_H!{0^W5JvvbKL;_|PD`>(xa9xkCr1e$dVR4<Or!@IwP)S^k7;<$EQ#ky@7<|{(P4k-_>uBl zrpLXsGEs6SVBY*#f0@qx`GWoBWB7RgIXUu-Fp|F495Er-MS@1#+KEW6RqyNyqe~an z=qhaF3kz?F7yhJJgIZ2y$(3q*@nWnL_s&O_Z zQR*ehi{0s0V2wjbt35W4=6^gaR7`9V%X&6GI7-r+@TIts_m3MY$X zOWt}IdR6xNgPn6S*_*sCmMq^Uyvucby(@DCK5-pGbFO_OAb-lwTV}1DvRxDAS5qxu zVMuf09AzX{o(UIYToH?-O|5`oefbibGwJYg;o|$*JT3u$bQn6of^4coT=+Z_n|aFMe1zov0MQG*%#W z$1DahPJ5(V)b?I6I1CLjkRIf7$kK6h?bO0&kR^jI*{TK*EhKp2;xPr9WB4CutazMY zG-ZR?V8=Cdwl13(nzbV%U9-FSc~o{~tjxZWwr3C>ipCv+YDEXy)88u5@&>+x0UztW zS;DF=fO+TY(i&>gpT%I*ue@JZ867p7BTTxZZ$d&H6npyxVuG>U(vqfMVOkEQ91Vsc zJt3rb&GqP6#T_df@H`GFu}p!sq^Uu*v@r(wgfUW*1^3r0pX|HRd3JYd*M02J)bcCC zx*0CX{)83mt890`;aYx!I!rmg;*oyE;1pT}ipiyt+hTXpVb!#QVejfRbt=lIx?$*_$8?8_ptnf{^t`ZJ7BGqz z$80)SKk6B>fSz8$uKR?ygy#NRbM(*ux4HjeG7Db0oCbUiDyXF{VBM{ah{+9I+Smqc z^9!>e11jq_5Xx0RROf@>p-;!^zMicxZwjhAwSyNET<);`xvZzN0%cWhpyc8TAS0?# zSnPgOx2TAMrFjr6RyA>MrpkkWsh7=_KrFWcteRr4r2%W?VOX~8?Bkq-u-rAU3F3;j zQ%}Oi{KNux=ElZ#^1A`djZ(@Uj@N^UY4QN^W2BOFs)Xq}1gY}wvdHBD9pMR%F5rq! zJ61gzlyo`x%u%TQG7E{PU7?8EWdMbvuaYRPpp-=W$|0-Ogq^A5{@-uyRvw!Mz>Svs zY=7GW)>Z2h-=SKXHjrHoq~AJ5TYH*XeFFAT!I$lfWmfeNoQ9F$*01OY4H7~(SHuG5 zIbd(2ZXRTAiH4*BCsHkzuUt8+sn_;vUT6s~iIU7EzB~`KKB`^u0&-hW$$-5Gc;;sj z(Z?sJMW7kfGJz=;c}?W25AOc~AaZw2zqaUzh=m~4yj|ggpS)ocRLpYBClMZ1M;~_S z1`(Z&k$8`QJWS#~57ERVLbUbM6Nlts>+_?7wD+VHNQG8PxBvwP#_a6G$tIbyQoQqB zN)hEI5z$3Lj%2r}lGYn+f=>}078Gq7$kE8?PrG?I5AsN96>M)81#KXJ8;fjsYpAEh z(aJX#UV|?k{DkMBURM?>in10Zu4dRoGUwDENSb9=b@toCG?bcx7E|G|}a zq3kb!N8iH4fBja#9lPpTeZTM1uIa{jD*)5;bccxY^z)m(PD`~mkpGyUrS_mGR^Nhf ztyCmCNbCUNaaLuR&+E8lSIBe7aQ&(-W6!V+v^c*Se5gRN0tMw{Y}NM?R!tf8XRnHzR+ zo$-wTv<$AZ>lCR$Vo79Lk&-HC<4}ASd{hoF16{7MGHW$KhtcXNOBbqsXR1*sp_O#^ z+@>4k!Zx?yg_)&lQjR*6VLUmCwReDUs2|Z|6_k!Z#gyZ^z7Q;y%{^m3d@_G|2SvJW z9AI?l+y12HuZe=jTb zkx-{IMcv4;-aV4DE|v`AbH?gn#*i}XxKcuOc<1*P%DOHowLoN2?flZ`)M{JKYR_RC zj+L*^4$wJ=agCfaJxrGf?}!*{uw#Iu?E_Nng>@^rJQEqn7}g-cm0;DRQyM5?x_zkP zvR>I|_hu@+^4DR>Fc@)Q&3o@jM^We4ur;VX0+CQWVQvQkEhp);F({L;0_4YU?0}K0 z`l-u1C>rRq(|H?Px>vp0UY@HD{SH1vH8c2w;` zZ6wJhZRq<++a7Ai*T4yUUg*rzmN4?vbp`-<&Q3Hm76LEhVVSm0@T!bY4`m4a`EZ zTRc3Gw-a=@gNo{lEKuD^t{$WV`+hnR=0=)Ri%N|j(4BQ=G4A(P>Fgw%^b~V7aiF7z zfr({>W=uK^x+917?!&VtVkM&NKQn~=sXVY#OS^uW|$R^jRaoM&N!n3 zwwo@3ZX(Yd$ou2;?AtrcSFz=#uZ>*01jV=Xd%pJ_J6nfJcNqb}ffOfHj&e~w?0;y{ zKrIi`;k0x)vr^&OMtLozQPC=4e6(IIO^LJjyI6{Bj`$Samm^o|`-M0TGeVel4nVV= zS@ELowyLz^%;nBGw*oZkA`}89=373$<}4HiqS()l;w|*vJ-0cB>}U^7Ooo)m zfpe7hfe!(1!Vpc`VcDGR!AHY3JgZ>h z)P%==z{mE1OFP%h=#C?BC)z@Vq8a{PV_<_2A{?zq-H)}C4P|?>ZwQ^V${!C39gRWS z3Oh{Ps5n}5OX&5vHo%Xj8CKfG>#_BkZ&FgSN{l|WX|72?l#?&hu8AC}*FDQqs9kCB z%s6HfDB*~HT>q$J%ie>aDdi;rPuJ%l1E23by?uQlh(zJ{{0l&`eewc5tKZaJxfoON z?8(?EkRig~`Gkh;%DOs5hH>sYxM^58SwjyfMd0Q~bqh$8UqE~!K1+1z z4lVdsra+IAa7<#Z3s?}5=MaKCw+#|sxi{NIJ#UF0&1XNB%u<%0T2vlHN7d97qKtVB zz;S`|JXw<-eM(O(REW59 z>1d{(cYUehyjur6(0R@@L7C49pp@0MTP8>%_)Vyi}7ZXudepOXvHl@oqfbvn-xw#WlbLIFZ3wcT{?w7A7<)#0R89>HFHoX&sA zY}DhL928Ll)u#t!S|Qx8v%urg5#9G_@-QH}OCFwmis~-zmNOnNJ&K0NRe3=sp7-_nSN*_5 z^gXA%M=>~*N%}ul&)Gawqyln*-e)ZGR9kl?WDZx_ zUw${~t>Q@F(0yy`8L^Mv!vlULJ z0My@EEyx-F=`Mb`Zb{X3LC@)juhabuC%LAgAi9N*>IeRsIS|+BWMBx;a%3F|%%gw} zIoqnG@wR5JC?8cD1Gs|tGVuHy+Ij6@p(izOdO-Arr4uBnhz!1R*ap2YM%ZWF35mKH zN`{A%+aM<*&R&J24iTGcXCyyu1BPts+{%}U34OmGEUC|L1|=~Y!L8D%>t9v@O41h( zkB#Ap-s)q+QMyjjix1?)H~B1Y1yqKw0{uU|TJ6f$Ye9LeDvPW{w@}uu;B(L|9tGFJ*bl&PF1_PxA%DEQcf#vi zT_i6ui1HhT3}GgE310>o-Jv}w&lG`Ok3ADnuW1rr9Dk(1|9Kwa;25uAxx6ufc%YV{ zO|mtEY0f*K-)ycXGO&ujsQl#Yrzz3cwI^Wex6~7Es#q+w?Pm~`X(ygrKt|K2w&HJ& zgi0PaK9hkJch$F>8tPkt75CBFpNHpddRN6Hs*_6WaH-I&G#`k$g?q+sPQ4~_$h_C( zM>hWfDh>!+yDG_G4V-pB5oysA)@^6L^nQ=~b27OcAd|@v9MqRSU$6ogZdzjdg3)I; z;@_O=BZ0Jl6yUch>tqY>Fn@*0aU@dt0F`*qFqpfeQ>PuB&|$cOU*-jiUjoJ;106Fl zQ=lcB+UR|Hv5Fu=c3{&L?3}SAmS;kK6e;aeXI@aAG}PtNGI=wQ>%k>Tar*?Guxamw z17yw;L1%*~N2|9R_Z&EwDHJnw9yiQY!%E3;>X{!0aPJfvm%adK@v9j7<6j@Xh5PW$ zSCh`Y;A?%(3U|A5K&L830q%BouF|9TK{XnOC-lFF%YRp3LVk%R<5DN8{DB+$8S)U* zQ7{R^7A~Fg1&P~GZ6ufNzHu*OA=Ftf0u8gIZcd73F9@UZj_4%L&$vM(<>yVrflD&! z)DFJB>4mb-P+;-7%y|L(`Kad)I(=?LZGd_sPP<+zZ|MweAop{M*J6pRn5J^#+qas$BV>M843k8ofl_sY@Ee$kw zWapq@(D^+z4|P?EWMK+Lak-roty}0}R)(11J`&o9FJuLq^Z~fCr^_y=lFP>ga8`>I zj-4fuh*wOKIbY@k_eTx0)BL)D?Xt9h&KHxB92O5pBViXU2VtTtRYPYzDsdhbuzdM&<0nbz-kfuH=o#SUTHp?o z#jsICep%~dskxtY{}2p8KJ_j-N3Hc8l_=H|1pT|0+R87_#EdC_r%G`27Ff~{pDQ0P z>Tt+-$x0d@5Po&tLV};49fs42gIiJI&R{avJS)U*_y|LSYPH1XQ|1{VKtRD!2BSGi5J>jkDsP0nJN;_q-cM>m*T6DjN+PCWvlkyp(o>B(*kZmdsj(6*Ac^3a_+w+iOpVj|}P_98^USAHgF`?e^Mg zYj%ROSY4!W3xncQlvtFW%KFs|qyIzvnR~v%mdVkbZq+ zcrE?vh53Ytg&_;ci@MN+HpN$-O6H?uWtlY{ob+($!@%R3oZg+v3H|;1&yHH-qmSD= z2UAeYQ}R7|>eb4Yd85kkzc1LoMneIx4q-%`Be#%8MjZByk}r*hB(zD1{GrXs(y#vB zyZ`)5gmm5$bS?iG+x+`)9>&3FxfbYQqyN3T|NdDWJr+E4;>N#B$-g(s)C>!v>ij|d z=%3&-g3#+PpOHM1P~cq( zt2~VZotpKY_8GU|)Z2ev)c+s)_t*V^ws{j_qqh;JusRrf=O3DpM0^=|o`qKfR^V($ zO12Y*tW<_5@Hj9xvHYF@$qb6g!>keG&+Hg?ecJ6Bl7DR> zLOI}S6}X|$YjzI?|NeK?*&0{l z<{`?7enSb9CakDd%79)Hi4({)XP^}T_)U%Gla1{_-{1?|l5*Ucr5RJGQ&5dUmr_mM}~pn|GQ z-tu?U=`OyBvDvdo@oWO^Yb>@odKr6gV<_(1&D_K-v5F~KO}iuW{Y}~ z!+{QrmZJvTc)y-kB%NH(CZ|L|gWXkRMWJfT@1<_L6M7JnG%2X8SIYcn>|}&an-pY} z(Z7x|lm~@Ki1d0v0vgk7C=SP!m42g`d%s2~pGuBBwVjJS7c2GaJhqWT{|%n}l#H}D z;CYh1dv*kWj)e?{en$q|Z27A}b!pUN(lhyJg#*2hG!mMMuTWnE(fo4&Iau7={@hY` zFaEk$BumkZ#2#1Ly%KIP!}HJF*yupFgT;g&(z`e>B*>)r4*hDdA0c7{ulIm5hjt=? zQ;?oo{W*g_OTaA^@cx0Ky5*r?-;X>BuA%7Zj^5qp&%^VtPnfjb_;ahtur_{ElTlax zeF^{j^e0UY{9jGKGy4^=&QraiJY_#bzKVdMO6 z@CvXrZvi#J&3@CX^O*~eKcr|^ML+%PG7caO5!k-V|JSV{9}Bb6ws`Rq8uiN)*mufb z;ogiJ2_X4-H!}q)lfhToC^>~dm)_mWw(_uVXrEdSps!4Nr6ip8D)ok?bNt@1Pr2E0ZdG2BE>jR^)`M_U+6Fj%wSlcd*%whK zs4Rh5m-neJJ0DS6fS1?E9@X<~48UdggP@!)DnX;tLGk-6fOMvV)$JQ(iMjb}REw~? zlY6e~p)rD?aAte`!a%L3;|dhkni;49vyf-K&=joLe#C9v7`SI3gp6geSLzi>@Hadf zS2e^n)KD zb9KzqEW@s+tw2ID1sEEn5RQPIC~O(%z&E0Ydep;@mpmt571%}m>`uNslM34QT8`?} zzXn5q1kec@gwryx?VXApm`iI*mo^JZcs#Qt=}fq;C*<;87=2pz$g zUILAaL2Tb)@$$i^DDqyb7QB;a~34Yj=`+Zme8iZ~y+N;6*zc=?WSRhxguN1pdD7f+CtBe5%;M>jy zt*qE&CtJ-06GuVeOKDUxZ59+3x*W8GV0}i)>IDXw1k{EXxI2fX{N5g*J$dLwcp;=eX z4I!(?ko*leF9u*O@?jDL1)=_71$cSz!&pA0%5TFa>jzh5V)TC9@vlP>C%~sTH(^LQ z3RKZr$LqcZ8J-WOQl{NZ-7=P2XqmhKil(!)w9_RAy z4$;5$iK%@(`*wn~#;=PGtOdXqM2@w~!bCqzZrn>}XD$?Y`g8HKe()U`s~cDE1SpYm zrlp1NCsOx&wh}UzB||w1w*JW{m18~)TdRo(I1`~nt{)19_cwM?<*Zb~aR7HvZF(Sb z=UJGgUFU29_>GZ6s~^gV+Er*f8BroE->J)D@nnfL@!m2C$L3~a$MtpTU+ zekga=q%4MuCQ)j2?sd-;VRT#av*k!FHMZ8I${?71*z+J3PZ96nbhmRGK*qN!O1Q{9jakvdjH@gp&FiNg7>x3pQ7erAoRlKBVIe)wvi&3cixr|&C-Ma z%$im&T$9&V_yTSjgf;Q{RQ#3&>c@(?fO9pzi|cLXSvsCDUOGtY!=pjqpY}=@qYJ>2eQ?d*=X!iB9Ph zeMURBiiLth^<-&JqOcl-la*2Qp+yBTC#l3sY%NFE03ITO4_Amwt}#rs_4Z2uB&>Xp z*cnP%`l&aAmnH01v>Pcn{J4c5SC;QYWc!}K>g^JvSDO;&HTs!nuGX8bAmc0~g@cG+rl9KcGBEzAdL9{U4kq#|4I`H=1MEpN3) zXF3#EKCQ@(ri`bKWQ~>fWTb@WgLFIk1<*#V9aX)GJYI$d4oj^H+VPO4{n=*C*=zbN zd%b2)pM$*e-AJXWeWDN>`lKoN+t`TY!o0Bz58M?}sqNVE;M>Wmb|7HK!w(6BW4)ir z9tgXjY?xY%Fe&z;_I&d;mmQ*29Y1~?1NM*<4`PdVRS+XhqHdKbpv3dkj%UK3>Wftz zN4!R}!Tf$u4Y#y(gKD&wopY8EqRtun{6SQ@TF&7A-jm?Zx!DLO{jEI!w>;?6c?;`7 zh73JBMR4IjxKWDFqJq{kU|}vivQe`C0@$!cRq+*VNARkyQX5#3MN_}ehnjP7%SAy( zG3~90Tih#0uIRiER)jU?NFRz}wF86tEqpe?eFvHDx9-`4xsK|~ia~<8xcxJL10i_WnluMNqBSIuA4$F1 zcgs;YCSy`|o0C5oqC=Gq%Crbg#EaxA?eL_Y(*)ehBHxOc))el$6_@V*WNZ|_WB zqWehA`8cLD%6JqI_yS7H{esnXsP^$Jw$LQ+%^VaeO|L&TI3^VJTYN-HA!$+EH)ILY z!i{ttf*ti*sv^yOXQ<~{@!H>l2GT<(=3#d`5~8irX< zKe~jzL80HC&|~acFXnsrZPH?H=)~k(-W-Vn%wl&fAXro$7teU?DTq6POR2@Tn0$Fft*T?+3ZHO3Pjz26u%Ov32%HIUb} zGmcM)J5O0ix{)wu{KyvoF4+-rSO0~~24tfO`$|aQ?&h^skd%nHSIMmj1q1c+3rfQb zp};9|ScXYpT)E-n!t$VMb5T3d?*M}0vct*c&0Ol`HLg~-5#}J8#s)LRTShA=+=;lo z{7{t_@Q+r~EdAGf-muy{o{klG%~nWXl18z3X}Y>Mb|TP*mG}4AK}JCEzYDk%9l$n| zSDXl@`-<1`kMgY;Ni>iy-g``~zuNh|ODUxFuW9pL zNacA{r^$e%-kCgQhw}%G&bunxnT9$uw)o8tMyJEhdIR?vK-zLr18y2i1rjYV zao7DeRCd@T@r1vC69FlV@*rw&Uk?rabHn8+OV%oC0b9^Dgr=&2f}g~LMZ%8S!;dOL z9{}pZcBu5C=`dhZx$AdAAD;m|W~GW)f>Wu9L5IB{rw@YEK>N_$4=ZDory?{YW5f8? z7D9TnP#%Qub?#*M9JlFD9e@cJ?9ab+m1a28CgJc{ju{{RV5vZ3R~_lI{KXV zUw?Zu4>Va3_or&^V0`pLAh}@KYMgx`C*!szDhHz~y58t8R#VYScO@T!xOq^VTKD{> zRwuqN5rK;axej|PlkPOAn~x1aZ!J^?j8HM4Ns>FQ^_sRoC(i9o7E7E%wH{V@90V4x z|2mocvY=W1f-Fd4rlG^N#x9+IwC70HKnT|7f{Ht&9oR(!O1sBErM=M#L^pFlsP^c~ zN*mMxn^q2z$Eea!IVw%|EuO^l2vYo?Q~|n_kYWwA&^%yVyI(kYo4XGNQ@H>1S4o}& zLida<@1E<-ip6JeiyS&he;4AsVNM$Fo*mgjcxv_T3uPTty82Bq&Hlgc&PX0i9?z7-}d7h$Y(knXT}-j4Vo4^51u7t8v!l04Y0XIpaYX5u|W>=mYqKs0x)f$VaIw%`fu zQ(0tmCf@iX6;L9Ab!PUEDmtzq3NVv#IPrfRYQUk9@*x*+=ZA>rb#x8R6+YRR#hcrN ziY%0r&)+DQ0$E|p{YnYG7KuA%16`b5z~1R~L1$iV58HQu!4;$e3RdId2Gr1{+8#7^ zRjGhlHVNtKtDpzxB?w-Q)jQdqEPk~TfxW&l?4NVo`%5WkJp#5_*_$~#0i-F%NSRMv zkDm=08;ONn*gxl2KJ^UYS3G~;Rl3ZdKD$yh)_P=?^kme3_Shn3DNlwq{cUU&H`av5g`72(Scu8x$lt-iPXcBfd zA;9bHs`b`Wl?J=_#8ov5ibRmU^S5cvMGI+LlYRy-@JK!c4t3X6FIT;6FB7uuyIoFR zp?>8oMv(pM;_`Rf_S%V@R|mSvqgXRP&U75R6^G6sGuHb{mRa?D>}TRCQZichgyYn7 zSv*6Lds!NF+(-t~?Dp@jyreDi9t{^MG?=q+be11e$a$mc@2_!|94GMdi=NW3h4XuA zwL62Nj@MrCht-u~CS89T5w^hjktpl!t>&Oh2cvLDr6>H@WJS^N$71G%%lrtL0wobIw>F|dAZ$a*KZ&dBh$HQeOGht4I&QeQa z;#nup+~WTJl$YnklIhXPzWk5N9hjo@j&_DdkHI~-b^TX$cE*IhHTcaA_rYuSw%CF?@<<^nE87tvhJ`x6^|$zf zHodHP-{HM91E6C@t*gcJpWB7i1Mn*knEBYI3vWJ$f`XdJEr^sBkfZgpQd2dNd`>H7 zRIB&$h)+@SqY;UmT4yH0^)zp8yF_ULXZsUNK;YBHj-5G)hEWA-LpDX~C!ZDd7#jd& z1&KkG{AcUIMPmz@6@@hcV65gFLed7ReLf)g+BgGDzy^7KV}Sl4l$q9VS54QowtlD# znyL$B%%V87D9p}y~J1nrY@a^ z(-Y8)Vp#{lk&}w(=z`w^i=o33LQ2S{Nq`BjWyRZ4s2j(xmby-pv9~Vi6xdcO3K1_W z`R}qubqR)t2;5YWIYkZS&cOd(M?9;BbI4dDO~>C?E>xkBjZ`o9`F)dPU2e$|OPvBD zL<y1Y1qf#3h`abk3h0{)|<0bV&fxAqBTML z8&;ihX;68(As0+5LAIz&y>P{PCnZN>9-jE{XE~!qNA+T5F#|`OIy1eg#_%&ArAN~k zC0*0W&$iW58>AfF^)5QUU?BMlrh*Cl=UZ{j%&IGrasJY)^xyAnqamv>^>%QjwYVL6s;$yfr{>&2C00En}lYQ1XK8{Fe#q$k+D-=!@yn!{0J3MdX5^NNYM1rV*Y$u+-1WTz_CK8($^iDa z##%|qywzGDt~yy&7*!e?b!%|s-oy4%`P2=pV#&LiTZOZ0YK50O6N2cj-iPc15hA$) zzkK1HQD6ga7yuWLy+b0{oE$H9JoHD3pCSfMA+LrX3t|}`i&nPu4y<%klaSoJF- zaGgh7jbe+{KT>)EL3hr+cDju32E6;(w^SCbZ)?WK+XL)cq?Row-B!+mvom(A3z%E zl9p7F2I-Uz=|c%fNOy;H9GZ6@ulru>`Qd%m^ADW0&Uf~nJu{z}i~Tf$a9sFAa^|kp zu-?%YM6)jnX6kvpQ}73DR;>VQTshAR#P1Z`cE)-dO51s51kA4J_`|wSl>0*X;i{aW zOmS1Vc8^7g5jRB=zHkjH&d0yfp{eRGe2aMitN{xF@}bcRXVX=ao*V+}L)(7F)b~A= zTN)cFkEdDSq|sf_jdD{cT>yno-vVG@Co=S*!Gch*2#q|==oP*NGhd%ZKOaUv>xdch z`L6sWhkxO{RxS9&G?zOVDpLrMQgEj%R%YGrnLZRR5Av`q-%I-E`~rkhWphm=7<2WD z8qjz^HH|nbS8#p7Cfr?cBeka`et^`x3uucvKz>oz(@gO=u-ARg;q4k`I%sA|;1WUJ zb84_tck*p+Uii{6fDiP_fX6+wSfP9^!iOj=LP~K13FCjg0Qmdbz~aNg3{f9cgBbaJ zEtm;%9FVNd*N~mVS*~z-RXIA5XvO67{2VFWMsD6>UWkr75(a+6^#|3pD9+ok&N8G@ zt*S57gx=HY*Wm4VzVng4Ii4?R?N#Qg!AYpacheIT!Q!5!^zVR3o-z$K)BB7gd^xw2bA=r5Z z%<@wRObZ=8S170Pt1JNPU^#xPPG5QG@HV}#{JgrY|MkquVk3Bz{s6^MNebvS{s5ua z-VNpXvd>G~36>v4!lcd0$$BD!nw4%&biN0f>)8%GQSl3y3B%bS%7q^ZIEDAg+G%6n%1W#_+sA#60Y}MssX@VC0)f z!%r~a@b*rKPpo*|`01+Ubolv)wK@$oYLQm`yfQ8J2G`wDMQ=5=0sEHqKZ&$zYC*r; z&eAACK4tux9UN1gr?}1!D7bsSm0V;T<_SkthJq8pPHZ2L`20)^K1s)##bz-ZXJnwS z=gXS{V6``|qs{+r&y+tVLuPufM-@JgINk1lZ88SOAiA4H;FXdSaC5G|k};vI-SAW; z%fBds9E=yJnb7QfG*O*NzE>}MtC+u0{>IDdwaP3Y_a!hsIlrOgGp-)nOerUFxDY_M zBq;N($vlR$l1aWiCKa~H&nz${-|8MbOz|Vzw^&&YllLdG``g}Sz>=#&h}kMn z-4%e}PRHJb7JQVS(FoRg4|iD5dw_=X%YL-bGJNTIcLReBi^7c5 z46D7>Yv?lT5~{Q#SIIDy9K5V5){uie$c!?8&{N3ybASS6Qc(}mx|G2LF3KLrH@iEG z5jv8w3aP?XPx5n;;xI4>7s~NHhxn#y&Mi;Ab589CQ0*97*A6NVOc0M{)L9MK&<{YQ zi(KqVAnXK8U(K*)LYgP`*&28P1LzD_e+%?6b5XB1mHZBk?qqq@5hKFuHA&2$QHMVG z1wm3(;GG?zmN?-AT*~7C7MpHXLO&qu!10=bGW}8)3PtA@D@lKgoJeIi~ zhCm|66Ir~Rl8W}(cjYHJk41E0*1niu zYBov=%7V+jd83#PlXlT80iluz&q8%5L2Ud`g>P7Ppt!^++tqJokeAa6IQK5fQ^=0`S*hDXk`*m=7Jbc!y5UJS#@Er~-lzQRk`(zlm zt)yw>@#EcYEL@|m%!?g|L6&-P0LK4<$b9g8x)J;598=;(h~*wY?^V@#`wTH7y)`@o z2_pZdZR`^|Vlr&V2|e)=nCRVCV!i!_isN?!273M1kIHwfGw%n5bQpNOfBO=21e`#ltCK5@?T2i>DFem_+4RK`q{qH*Clyg{asV= zTNZo5fp!Yq95Pr*neS5KIU`!@PGQv}U&7MfoXedn6TNGMsJREjJ6HH#fR z^vIgGbT?l#pRJ+e9InRMXiR+Jg%|Xy^*FbzYXhlAn2ETMh5&+bux`ZB#CQNo<-V#* zyb_4=c21ZJ7}w6!eIg*>6M$RkFtq~+*F8;5+C{UN2D=SoEE2OH(DGzdom zC7?{g-&ip!Uqh*(fXnnL!e5kkKG~`9L7}Kv0K0IoGAy+-Cbrm!J|V8jXt*?6KI(3a zR9X+GqgdlYt}%ta(&RUhvZRd2TBz{^*E^$Z%1LmAi^7tQmEwx$XOat#yR&X_wxdR6 zz1`F>!0%D3KO?MR?u!j=o*U9je_`K{GUwQ^h%PXGGcA~R%-$dlcqb*0$!Er;`Ol#q z-no!LJg-4*2tAjHSUVjxq8w_qBA%sH8&N%^yC@_w$2DtzTY{nn8UcaTOhh? zfjwaH#0gaPU_eP+EotW>|G`hT#a->KbqDDJ&FWb}$ug>NDQf&x{NnW3WR#%42(-_I zDh$U4+s0Syd$Kn1dQOlqR;$qp|H)HL^DK^7WZPG?!!RM$Ma{2&+3bLm{D@{XxopYf z;Tko$M3QF|-AKrT2Mq1Jx?`&mFJ?)yWGJF2je0{$^#V2o$2wlpv9y6;tum&6j+HEPED^EUUeIS}W zGbiS-AlvX4qO(S!8q|Ocp4jQRT#+@oCCwJ}vBT&N{P!Q`IG-J#+KOu7JG}}biq{B& z=)b^t{Y7Hp&~@VdAHpT=*%g!@cHB?2aiRy;VV9r{s+!3PAQ&{dU2K%}dtjh3;b@k; zdBtySVUNt{qSxLaZPEvq{ss2S24awSsZT0G$l4vkE($m_Hn-X4f>&4Wm1nbI&3*lQ zT!uG+!ykLmoOi-_&+h5@7i$fmAQ2rQN$vKq@nh)yVc{^-J~Z8kIQ}uXRI%*N`6U>d z@vcIzDYax-?4*xg*2qVOB2N^kGxAz-dVU(=uCk*t-#fRK`buZ6+@oX7$~TvxDf;_m67G-@5+e2rS zFINzIzzcTI>&e0^*Sz(w<{+75z?Ui7Zd!^L)Fs0%vGRXnn(QsjOtHZ`p7vyR&jkq! z&nE7YfJ@tP>ZM>Ij$Wn`6+nrdrmf1+YKcxzuD(IfY30oKyHC{m5ca4sydSZ$r!ILk zUN-!ltu0{?m=w043i(LJB<1~xjp8t)kJqRbG0~@S+e--3aSDwo>9|r1v-+B~R_$pa zoBJ$PJ4w z!MDU6-UcSO(#M=!=hlqmP`jMh-HIH?Bb0wuC$ul5B(d_#A8^+MKM`=j7#kjQFCL-Q z|E2BuZpVT!_+B6+Ii{vObRQ^gEPg~uQTvRWacnx_iVb@9xM^>dTb&(lJq=v~Eq_X= zX$NPy^${Jy!9q|_>3eo?-ibFHXUcop7MDlm8hwL(ld&E>*a|lICEkZs&+WS!Hn<{L zKFSN9BbXK=pZ-4;XtvHYxFT%)aI8OMUzTKYcgPmx7Z(4o13GB*zD%gV@CSB&Kjy9u4Bs_hz;9Da^dwZgJ&7b8h>H8+@p4Io%V@ zb7}dBeNoo&=p-hd*<>vi)#|@2h;6Err^@PoHwrhTel&_7n{xYvm$yv3G@lPZw}Q)3 zV!O|p8)1CWH4XsxW8q~c)C9&C$X8~ZRD^6r%TP_riO@QQ0HaFfPPdkj0Bw;{-`~B} zfja=K;^-g3gN0mrrSe-!)3;gB=#$av@67 zQL`7^;Dm%W@{I%yK9(Oi_=Pxug-ct7JH_Ptqduxv6?r;#^3Ew#@Jt*u`7;H;8eVRP<@1!C4OWyB7NHRXc>X%d-)T990hb@?`8 zOd!aXRMcyL;x%rVNZyg+_D8Ca)-u(V*$M`7hr$FBSOd-LQ zkbPTfA7^in``^udYK-FXq!~!n4RIuoKcivH(QbUei6H;*YVEX2jzzBwQ2zv+f)g-~ zo@(;)z5&GI{|PH0tvRM63rAJyYg98mF$|g@NZM$LE|<@OHe@72bk4x0t2R_Qva$*@ zCG&ob=7IaFvL8#T{()1x+vRs9Edl@ISLXVL+an8tMV~c!Fi-?gi!4w=RfO;qlPUJ` zH-Eet;w<{>#F`P__(3_&IHbC>v;QzxaOU_)aBo9sK!`o;oQROk_j={FFgmc-OOag|JS!1($-0b1*KT!mw`T( zln3(G8}UBdmzOefPab|>V#_yH2dNnSY`kv_hBDG6`m}6*wfn1w>e3OP{hP)YxaF*b z^ZR^W<>jD`d~L1~8$?k;&e#^cK}!iOcWG5Ogiu+U==X&CmMampDT#E`gYZS=)T^ZN zKd4Y1iG%+Rl^QeA?TFc}63XlDKu3h(ACU08YZu%ZR}&vfB!tmGwO0L&2ikoR$_?ge zEfq^our%?*7tnyUh|3KDZy0Ms*J_U&&8!+q{oJ028ao`lntjfi!X6V`O1PHu4$eFO zuiHgpRPNX}IKJdmt!(`CSkV|oU^f+!HH9XUouJ{FWIK~Y0`7Sf*Q+Xg-jA5K1lGd* z#~;ZfS&0(WAb+(Seq=Y}rB^?tXLMtmE*?8(QTKP9vp#wLZM>?Kn{dL32xKx8L01AYN;@CW|RKnm%jyYY)1W zGLG96^Zu0PIrlP(YO@ZcAHGipJA+Euy4JgxY>v!1C?1F9j~Mv=*X2QQv>=w2bYyzh z#?}pfId_H#(6R6j{E%eoxe& z)9`*Usl(?f3FA@zU&qLh7K>~gA>dqtr)5;+0vwYTT*C8P#LXV5CB=z@r=fW~n3<{G zF<{l$D3Bq48Om$khpa@e_a&9Wihxe<^i@Cal^L54>CSJsKn9P1t*)DA&D{jO3Oz_pGwXG$#AELMW$uqy^o_QAo!V8dHJ>5MEQ zn%*rUQxX!ogF-^l3G(1A$OOP#m+q( zC^>zv`!o@PIUYx6Q(}{c$g(!$js4d!VKxJqEbqEd@NKCw|Afha|jG2_i z^-psmk{hImd++Pao^U2|uncFTry$H^cx{p$ezuG~HZy*f4H72cN1Swd(DLhsgc7({ zk?dJ|t<@h%oimp=?&a#Cy#*b8Ve7ztvn(Sx-Bv8t;*sUdw=7Soxs{ zNKCvD=dQm4&Y=`eHIFR2EtjeJA~IjCchRf|-e782t$y8Gs{9vQLnih@0v!%I0{$-{ znlsq>12Orn)q%JJp`CjC6(LT_zEC&tTwKp@e2ef9khY(zE9*~83ZfV=W4Q>G{*C9- zMTfoume%svm}8*K9E~Y6O$>7UP>#ht0JQFDm#&o7ubmGt+@Au*LLpDfKh`E5wHG5O zM15pp`smWO*0i)+JR!#E&eg=r>}&$&dHB`3CCa?T#5=PS!9-M^UZGWLU!}hNfjH}~ z=6{sVxBdu^Yx&p-5j)pz3u5`wx&YRJqFHSWNv-;9qh$gucw{_atr9{9#jM>?p$!v( z&c;QE2Z9XZep}gA-5;V5{0AWplfbm?5U_)WSzks=>iND#5wPkaMLxIWxB*?c4qSm{)>omj*BV~VSG1Zll?1O!X6mKWhz5)X0*ekcNC zAC<%>uz~~iz2k^m>^%-2C7FWQKNg<%LfGxO#I#gxc@p?Z7Px`MM2}UepI91U(UYvF z1`NGAc0sYegz0xEnwz@=rn3UE&*Ns?@}^z3G#Z$I$?pJRoz17+J0sjE?s_6b^K}$n zn#^!Md19<$;%6I5tNf2@tpoTMYlbE6jQPv%cfa<*SfbX}WN0tuzmho2(v0I#Z;@63 z>YLCJ`%BG~_kx|e^z%*^hx~D5MNHYKEbpLQO2)!` ztB2ftpxVubJV<#b$&sNEyOfVM2KELICvl0fI%D4#YB2A1uFKxSK2jdxabg%1M&DQorux#3ZcJg;n4`#Cf_h^8!Cn6o< z{SRks5kMlfSabRSsNYZv^1%RE^EQAgec>py1n6u3o;*vF^_QBYn2(j~Ts3)mV@>jU z{N_lQ?ql+weLJ?AFCTk&JtpO*-Ll3wFOZ^?Gc1 zjh6$q0!Hk4%VC;RLwf@JkPp7}R3^JAW zHM670im@2W&T!c9Kq01V%S|jAj5Iob%>S}^-OB&vqMriwd(u;%*;nK+^q%U!+M3|d ztapSZls}f-+J0Mg9Ue4CRUiDthbDjq8toW>Z#ocaF;CNZ@JMfe zjn(}4HAD&bNRJDCI8rkdKYfEA4s^AJ$bW|;j3|v%_EXXCyvL6|1K4FKSbUM=W-Atc zLGX7>hSPcV7d+?@h2c0E2gM9H2l-CqdcJ>QB*#r)qU zfa{1F<8CX>JWg-MQ9j5~nEKzHxAe0esMTZlVdY-}{=}~jp{b05@T3jq(o}kRXGFa(a5)^lT!bUN7lo6!dQw| zsLS=*2MwlB$o%N5a8XU-yDW$@!Bk?sozAR`<6NRKf+v{bNSLK0Ba2)9+6Bo^Yh?_n zZ6UrFnBrJ9@SL&-caetlSo+dA2(Lvycj=Zh@j?vPIt9DqMkr<3XCK2{CBtwvp8&jd zx?)>yj+?lXlBJIdZ9qoM%RKI^GCF*aKYB<;LCo*gvxsfW6JVcusl0tr*H76^-t~*I zKx#L`EJ`bT9}h^T1w*w-)1FfMAN4%za5G1@<}n&F7l}>IhUf>0CHs0dav@Ict{TX% zI(P4iKMctC(?|L=$eqPaMb@gEbk%-r{)%_LVjV~^0J4g^MOEBS91C`?&zL>UCx3QD zzzo))%B(e`9-VfZ>&YVw878aeO5*CKFO}h(Y1-P$xjX4#3*7G z*@cVdW_eBErb%AM<|H+HWckdMjtrl&%6aEUjHNirA+#1N z*z{<{avLZV)l^Gx0uwF8=2bYkKu=Xf92i-fT$u~zz9@re8FXlW39?MVf6Jk5PgwwS zR=@5k{G>`j!LrZ$^n2cUvgj^M$hOJ0-lT$5^LnFrj2NnD*h*YR4g0ZJsAeOLoNsJ8 zEflU($##ciYD40Pa_CX^TDwrGqFBpdG({KJs{Qa~hMK{buOz0sC}r`l4Zalm-JR>^ z6D=xMpmSXQ!m@nAOl?#%reQuG6j05bifC6xtE0qO`#SG?_G5?jtQxmERwed&&Ib#W zET^Np2`=U&zo+JjQaE?K^JiaLNg+9RL1{0~-|Xw?6lQR5P3Vj&vX{EL5LuvHwO#MB z=bOGD{A=p)*DT{pLmWe&D^|7LhPw$(&Ij-00#-xzqnOeJ68Zf}*=+3D<&ODB%jzue z#Ag+q2pw~dDT-nkT@t{CyEJrInjc11O_0%!wyz@l%)&b#6Gi{9szgs;kD#B1_@>dV zM5S`fHXp<$(d9Qw@`Hp=>m_y&5_}_;do|lSUf{a#ljyIYP1BrJ)8?mIu;u&P z{@v-Luost4d*j{#CVJ9^${RnH6a0t6po^;f8tsfR9mpHn@q8tcd``^20i8qPXrfGB zml_k#u#^0v%FYbUv`m4tF3C^J){kRBq-S_-B8J9!KT=}H`b5!?@N0*Bg-Po#39Vh3 z`R7MM;YhKBtLqmq#4q_ZGq!A2%Nzb zl%R7xz;2z%AmhlPo@d5aC!P&J+y|U6`&mTUt;wF8TxMhna&qlFP=-b^Tp(r4eI}jHeC}ycxgbcX;VG@7PyLzDJpV!d^hov zo`{eq$9^=$NYQX4$%x9Y_h>FZm73ilrl^JsmN3|7iz=qo=_AG}M>_t_zvzv9uCBoP z-J3n^uB5uo7f4b&LH*p)8yHq99c9MWtow75(H&=RJhK(Y1_&w~KK{cJ{?zf6rX!EZ zBAgntKzRzfTZU<}2oI;La>*Wc{xB#S2Hhme+(!$2U({R7TLP1vq(yON7Zavva1nbw zZ^@t4>zn{vJq6?hBh;XL>1(Ao_sd$GEQE$kA7h%VPPtgo|3@s6Hm3&ME?w>xW?w29UROip!EVwW!gUBQ{sw04XxU5P43HYJ=OB8 z^6Z%>)s_m`92E)gT|)e=j|WQuL(XgA!lLmTh1ExHe%(<;Sc`Qsi>Voht_v`(VRnS|Mi;MeLtd4qBePB?> zM07B-_OGurZ{WA*bzENFT?3$3{|toEQF|4&!Mx*#aG#38eD#+$Zs0|S^`qb7F=xZM zRz&s-B!!=h!Pa$p=yoXeI`#FLf2Da!^g2l*&#EvEEdT6!g1&~M3Yr0eE5 z>cv{J5+If0nj@Co)qEW>D6&ZL>!1^4p>`@^9A^0afLV)WR3mOQh_Q1Hbx^#COf21u zaJTd#M$!Y6b>cUyWS8a&lv2N|-?s9ER}lu^qW4AR$4ZtX1{o!74_QqBcUiNORhrWJ z<2q`cEyp86Nb~-Avz0SvVWklk3@>GLC+6yxCLc$#%x^Bz3HbhGo!lD#mneg3$Y)rk zvQ;nJ9dJu~M&x_mn^ABHqPl$LiDGa$D< z%`zT{{-Zz3WfPl}U%q})@wM}?e$jqXZ`_C46bVEQjzta$or=cT%vTVOw~UK`-(;@SYp$IZ zy`qugU(2AwA;n+n($#=xsnt4TJiIHs?)pdvTkW}?26dGQ?|S_!VKnxvU*h*6A^zIp`QGJPqZe?a^owT=wc%nc zttZVQXg;Z%GxI7Km0ARK_@*$RlmE1e-+@-BI59H?Ir)5?zH2VsDhLMe(m&B^s`*><`xT( zea^o+7P%<$<@#(#E2YPEcAbd$-OY5BZdN!>wyGJqKPT0?% z{;NY2`+HnoWoe+lYdc&7I;VG=?U(!Qs9$~)EyGaTOeITd*J@gUDlfx*XUBX4Q`!yg z8&5TSu8HMK`21fw_KoP)fY_A~KacNuwlHg(`!4NCMOl!t_rtT{1dW>!-s9;Ga!f6Z zgDgWs*Uwb#jL!zDG~II3^(;0}9Y(FtP9=}^zA^^Gt<&d+zOH$rcm*@5O`98>*zL)u z!7Tu)r-I!l!d6XB%;TR*-rUyQW!Ni4iw1#X(dQ|BS&H9P1D=B&zZOEZj3wdf!R6; z5Bk}$z4h@U+&u6n>^izW6qGHNDYIx*kn(XU$LktKr;Py3CrRGnTz!ScYjesX-J?eB z*MCL+#W)fJ8U)vPWU0cV$4sTo$Wydjj=wEe%+w$LiV~%M5uqQ)D4RQkQ7(I#%_y58 zH0RHQ2Qa;uJ(;hbSIAnttyCvMsn9KURDOBa!8w6>~-$mWMB7W&r zeXF0&L5LE^z(VJ_i#fXhM-)TDe?Z-3psO@1tURp(f9qc%uOsCS_e1%u_wOc+~-b2 zm+T}W9Kfj@vTitj( z7)3UDoF!7KDITyS2#W!hU0rmgLY9i&GDCk;_Ik4W zWpdpUv_#U6z2CzeBgUTTu`Bsa;ZqE@tb0%6QTV+=&{gmUg6#f{Z~RPd}!4U~H%#s=5lF-<8E*asW%K ztUz>7SS)@~;K8JL1gwq+jr&81n-x{ts_~D@`5;xEEL^!^H3NIq?Q%2sS+KK8ON>@= zW!3lpzTdM0MUnc}1JA)$8Xjn{kD$Oi@qOxOHtt2ZUpvJ{$0hPq@B!MCCRNH7{_>a% z)J>8diP1Pe?mu>kfmq5v>75NTFuj}gDQr5?=rA^3#MUrO!2KkM>DV5cxes`-qP?){ z$A0FGPgj8UvFee2w`IK1Hm;KMIn&1kzTRZ{c>EbA4oHI4!a)e_D}EMDYXsBSpc`P> z^5g0TcIadn!I-P*_%ya5ma*qZ9Q3V4E_dfsga3UxW$%@^YV}uA9oP>WaCi!v?2Ji) zAIGm_WeI!G(z=L;$<})ImG#4wYHQE|{1GRrp3^-{0H(+esce|NvrNJs=o1plf z<`=HrOtJ14z%DiQ8^Pv`X~#W!^I#}gD5#;POOI&sSbdS1>OZvIz$E3LNXLdA{{6i3 z>P~?4)9~I+-zNbV4@~mcR}tGWSvib9Ppczg>N#Tse#8YMdC0>ZY!2}N5a8HS>8TbZ zJ>v0&O5+*!a1uGe394-nd$>Jy(CSfh=TWjvn-=PLlaa%60kEQKU?f{aLZL~78G4Xv zQHrV3kQXZjFq$qSVA|l2mgnHo9fdFx|9aqrrt84i|GFXbU&j%{E^rFB>!SwsPv^}G zR=FNOf~VS_gPP})jaY>L3Xr6^_a7FPAj_H{M;|gQx(OdIO)o##Bnw2N7(9xyt|4J3 zS`Y;p6aK9Z8Y;gfdxp+ZxQho#Pn8>EquDdJCk*# z^)q3vMcc=CHsucE1I1@aR2OL!#QuO&J>x9(8pXq41KW<>(*clT-G8C%xngY8u@2id zG_W6I)(ZUflSx}~rLuh17IXucqt&oStLEDXnG3xB?R*}5*+*S?VhhHxucH2V<~y40 zrs)6AOJd>16APCMl79lfnr#VQ`)uBUR_cFsd*caHGVTa@i!a*-ql{*Cr4c@h90I%S z(PQW-aful@>x;4Hj75G5ks4J?GBp54=43CPuhMyqYTnjpehlb|<7`fPh8?dt%6S}J zq#GR)d?j^;H2t#w;RHmsPSbbjjhL(raH3_e1uuUxJ+z*b!h*wu#q1~0+IV<&qIjk1 zzae`r$C6YqzxOW-CbOfR!i9QOnnGCu!h<`|yn@^rIdD_rH1- z;A3X^OXso#JJx4po#yR=YhD~gp==ftQzs4qog+9VgEqvzlmo+Oec>OE zfGx%^EKC+0Q{Z2M+dSrRFfTMRb`8L}s=I)VTF43_JA}inXAo_No)W-%d4@omr9wX8 z18kf<;k1Uyad`irY69~MJnGRS`U+9V#bRx6KI}j%WOoHdNiBdY|CI>&Gsiiw3+ZFA zPIf(1TvVj|modsh{w8x+f-N5Wcbkqu2K^`c#2Bv=0z{UUUF$zd(F3B&ZtA}RK*rDr z#pAbPnTxJV_TS2-`N~T`bhJsHa@YgetZK1iy%OavA=0SE5kCa5n|GWWa13L`eZ1r64anKU5h3 zg8hu1N#$$OxM<~XNZIpv9P5TFhyx&U4EK{*|4vVQx-6~f7T!;+4LEv#I>$w&K8b#~ zK5NnGn3YW$N}#@@O}psh`}_J3)qZ)V$<1!gBSAG57{aH=oU%7);1F&lO!|jdkP%22 zc^s0gvqOu2hhSYj0p}1e9GmgZC*9oQr^NW|xM zMZWVTfpGqR1JMTMMn4LNN8;Q683}eMrypazUHapC3P+ZD6=$8U9%FdHr3vmlVMha& zSceZ38NoLi*-KD!VvH%NbAS>to~XK#eDaS6j6N@tH4 zb=ICD^~iyd-QO(y!K!+|!T2HaM2D2r-m$u$NvM?P==!OUEU_rD^${m=+Pb$KIbQ_nAf2@`P+z)I5#wuA@%f!+3! z`F-+TrOYA_Gm2dDa}WX>zAp@U?X9JnsMfxPA)=?`q;snhr}z;Fp_9m3KN z)_&9I@s+wPTN+>EoJ;HIJ>;qyg@=|?L{`1Ln6^}*u93SW8l}5SB z$O*;dm5cn5P!gY@MH*<+bX#t4QWbS9x7UW0yE#2K8X7X zh344fgm#v9IcERejCbwC71Gjl7_Pp?csj}@zOGkQ(}p9Zl>LORw-PsUJ$@nnxzaFE z3gwT_59dE!79Q*l8V|qtY-U^yfACz0>pl22AHMgqe2ee-(%$_qH`CPer>ZY^7(7S{ z#!?60q&X1h=MQ7wnN<5_{5Y68y!}Ltgifd>;i_AdUfNu`_w(pZ59T4^dW8WI@Ht`& zZhz(X@!hw#d@OvMg{BjOQZd7M5=9rI>%K4LB*TC?H(20};QCc3Xdg57ik9QaDe!4f&w z&5z7~2wrWC>-DYV_=*`&!(#pg@*hQSpY5JML zcay`gM9U1}V_z@ra_;5%ox8%*?qWacd!avuUk+mb@C41DZtdr!k`W{z0CZi_zuQJ0 z+bWImbaA`$qjcU*_!f`gp}9?(EJg#@)1iQVTfeBchGf-;C_}FsXANs+XV`%QSBV!Dv?T;jQ?M{s_rf+trX0 zd6UWltoZEeG*0}o1D71U*+%1nM}Bj_Ok@4ztcT05D%)B9{N;X-i8$(j9C0|MO9aK% zYjM&Z5^SOT+jo?HUerK1;~2!X@D%_4Gch#`Y|&ZN-wvnyp!4EaywTYq*!K0ldEnI! zT)K;m>o#)U_5I|#|I31~Q}kdGU<{(3WFkP4m_AU7q1Y^x>ev3~zR419ROa*z(uslH z+FSh$IOG@*#*fDNXzLZ9rSleElO}x4yZ*X@wv!&|p; zcS=onH1F9aKXDwwT7a!AKXcx|X5aJu_*g09l-O8;fbd#wH}&TkBT2MYun|cbrl)NK zWm_vt7v}nv$d*vuJGaITREmbC8F1-xOLrWR{-g$pVFujLaQjm2Y7nFWBsDPxtSb>M zHf%FK48OeW!?eB5xc6(z>vfk#lqFO4TxA}d2x&f+_29pBWz9MuStvT{SM%eY-Gq&T zr}OWMhie=rDf>Qz=D``f! zw!eZ)$ul*F^87dBeJ(8wCCq_s#UYr)tCTexF$7!&l$z9 z@t|!q;-_CmUgQ@oE_@ZyX?_2PCxkK=#MxJjwviiRWvTVSvjsmCi5bAeF zVY}Ctz2XN*Eq=F_C~e|r*|?1_brFm@OqWjedW$?<8wzzZCfVcR^2p}JIP+y)@t{K8 z_Z1|6#E#}v0t{PgKajzIwi?P>@NMneff8v7I2QJ}-^DN)(Oh3Uf7qHt*wpC>cw74u$Un3?n zl|)a7Zl^&V@D=HD7jJW1X%~^2@nSf!)Oy&voICF1tDv)mWva2e^C=F?p}E(p7lx^w zDvIeOtJe3BG{u`1#U<9@V&rI4Mx`e0C%@B$?|pdooqmCZqSX7ji$A)I{@ombU)7IS z=bK6#0VCz0JT$La0M&W;ld9If=sEOHdf+kb&snYeyV7Ri;_7WxdDHT|<_B-it*o*A zq)hWaiHF=j0M+pQfJJ`&)cH--Sn}6g2y%n#@}G0{pEA1!r%A$50nkyMWf!ob@V%sn zN_>$-I&Sp$bAzFT3)+kRQ?FsyzqA7h%qlbyDp(LZq{+&H)Udvz7+29^v@_uFxL%!A z^%vg7)El>cz01j1yeCEZbe9vwFRri)l)me_u(N5UfVm~4!{FZg?p;o@3x+me|B!pz z`>2@wr?jL~GMoZ+q41)mWO6XiIBsM+%Om4Gd9V)dJv4nOh>g#42m!nfEnSt*z165Q@3nLFdjn9b z=ir}ys}g@h>Z9~SAIuqZ_v@HrtGd)bd$ipwK>2;aR<`nn&gqP0J{AGm*v-0QP zE=IGM6qEw)9L|#IFg{cWm64zp;kvHu*_2n@sijU+)N8t8bEG zE2g|Jq0MR=x)wY04;~w^I;7*}Cw2U*Zei0L-FN-3wupK4UH*sx?&W2NHr?D7-8#qm z^2e^%?nDt3XTuWMJIh=48VbO-nw4k?`TnebWST2F8KI@eXc8xqprg9 zNpq(AO)3Hl6Vp|P3vai8VPS4nS0S13aBCHTf&9&!@SkWZxuNM{kzpwZt0mP~ciU4d zXfzoVXYSo!Uz}m>=2;Ax+`6I@n?tJ??qXz?!ZHolKhn7jG+zr3*Ej$3N9J2d37IWb zzjlxI%Y_BB=8o2y?mzKk#~h}EBxaCL z;AN-b3+AP9!7{Y^R)}qXHFkMZ2Ehegu5}tzugKQwHtcLG1g{-dSETneD)U?&6rAeBE1CEK`fU4 zWmUZ2o(&hh!(sOsdj~8x(m!qlywBbV$V{4tQ%!mTUtKJX`G2R1C`bLnuMnn#3;*X` z&DHj_0tRr`T>*OtxA#75D*oNIPw7g_1nTkLUG5^q*Fid)(Z=_V;>YFUO!w1+-F3&O zq1L0^j0T+-7PsH-YoVLCB(>~(w`oh%VzwucbKY34MO}|pn}&?M#y3{%xGKEgl74RL zVeB#a#U9m)C7Sx2{NY|eF3JZh%2}_wo#)xZ% zWfbb>qpAhuhP&U?3W$!_Z&xYG#oEghsQEUlC)r>50IKI+ux~I=S8*mVOiaYyj^fwSK^+2 zevMFiKO8DK!Oc>djqrg&LVRqFdUDjxr@&|&dFpr{v^ueUXxFwpN3UKmdAI+kRi%m*t+<#YHnbnD6rH00uhsk4 zv6sjxF9pxerD5A5PX-G}uOCqGH4TYJV`FKY_S5 z*SA@3f3vbQ^Jac4hj%BOgHwOpO}PF_EnPT?WJBqJ@bcj#we2s`;(afU=ymo%QyKHd zcmbzVeuaDUY`Fo~n&Z~;ak|orOf}pP+SCyd8KF>uk7 zG2s;kUu}hV%8hKIsBQW{mdT>cLnpO{Lb+J0YMegOJbBkF%;ea5{Brv^-}txCPp8YK zRNi`FUBSjLREta^+G-9vy$-YGl>)aY#JlWnb7|CDw+m$Wbiz02mRl9 zy=kC*?MzLf$IrH|;d-ozQyu@bS`jdObj+Y^Q*p?=Tn%1tqUZd3lY3Cc~aSx{o+MjbPBw08+Be}n=fy) zy7g9}UK~oWF11(rxj()HnmnTu_lPg=(y3nNZzSoDIS8@nIUjYP$?x2lH81&sNwd7@ z^_mCcnbImp9wXS*1tA7en+inE#4yaRl z4gMofiwVzx?iqN8-D0Q(Va|_wU~%Ru-`y>`k6C&2Q9UBHp6g)OyZ)&~-QGbpY1G?W z@{;JT-aG2UhSd>iQUsq;Ma&v5j6<+yrlYWpWJOa22* z%@>R!6rt71uoCCvA(6Fvktm=o-sAcevfRO4Z>x~lQmwO^Dir|o<`_&c>2{`fkCyvo zk=GC_h6YnW#*gy_kQmXO;@Zo(#{Y`0wl;fDqoklM>DMx`9dHO*d?an;=%5&#^5DMB z;*ZQ53RulbSnEQoCaS)gR~P!TRaX{;fSPPhG5?ljLbg@ZlK`gW&R@>l2j==NT}SUV zu6K@0$5#cfhzIBq)2E9J_gW95KB;$1T9aLT+1&76S6;1I=v&17%oQIL&R*3@eob;i z0!1K;=2vszDtKDiF1TkO@Ev}gh2KE)a`@f&5oS-NDWt|rPb^KiBEoxLIAI~L9G1b9bu~DH?Sf2nABlM@0N+f)wC6IDr?By&(V}Z6C)4*qKSGCN6m>$ z>PW`VfHYw(P1GEl`x_sIsd~epmV{k$?UU!4S~wpnsDy!OuiywZTjE{{eYah2X6(3> z))m*o338bu$O*8vP;2f}TEWY$I9Q7jprKjjLU7BwZSpk9Caa0b<(0!$;gdHnE=!dN ztr1zIF&OpuAFgS^ba-#_-tfGMZ?lyWZ&^UV+j-mV@IKngX%7<#;@wwRegooUO;5lm z7eR65tEDh#0$0ZPg0yWr1kL2yeJnOqa`_iwB&g@u%dr+LXOa$YG_FZnbJ0fWx$u;5 z=;+W+m@ftqpip*GvV%e<*alkFxMHnzI4X`Io5|MpAr=A!phQ?#`MJ)#U687_qkgDZ zxYNLWFDbd%#yxA?R-J~$WWU2G8{>iOl|4OUMkB9>QXbA;O*wD*+&xu3G(J~Z_IgX( z7a$`1PEzHth>3w1zJ^voaZ1r!r1f6ibM5|p%0SOZ{VOy(%R7%0;#%b>QWv6g+jL3~ z&??^53JLScl87xoS8A{T)wsq^8VI0^^>ZOoT!)PIW`#}S?)7f})&n*o8jke`t2nrl z-c2m)bG(@mtB~VdgYO@*SCmu><}yF)_pg9M+BR^6RIhXd9Ev_HxsqvicIb_ZQsDT zq%=*};K;M3WpiS!Q$bTcfKdp(M)sgAJOT9NUtBX4Xv~{+r4)64&sn|!nq-ZPf`DoY z%>n1CWUc7Cz!iPt2yog4trW#}2xX5lcGH}WyrrWHrZNV1*LxZt0Y7JZl?qMtYj*Q? zlIiCX=u`6pdeJGvrmvoLqt8Y#@00?%S8q4_j-oaksfqX&6X~Y_c7<_#s+6M{w=ijV zvB&FbQ*?}Rt&le!;=|KWqIO78#F%w4!sL5WMIXSR>z=+zFVt z{?K55sDBppeM6S#c+%*k%f>C2GtjQk1quMH=W#x)=R>oPJ;$a{Nc#ZbPR%I-E$Ad3 z^^&P-K9E9!P@L~(Fz3fC)G^P8j)r#`%Xjnf`Dk7ewVnU2F|+^cjl(LWqOi?!t1 zeL6-sXrD_JAi5L)+R1e1MZ9%>G{xAt9|jSJ?6E!nc??nDnzr+XQmTi zgis&~sMp>4e@S&z$Vo39iZo2oA&O$ynF3@N4uPfn>)o{O7-^%boZC>yRUn9GAAt<$ zdJC=n1gGR2=EKiPG5DSL1Cj;yrX=eKBa}nm`ilu=Z#U08b$COF`H6A5Af@;?Q#Gvo zd4=ugt1r#TNr!p)%q_{Mho`hBWai*_8xm!r`JfT6h2NKI_nKM*+}J~2B~qI;JISq` zB^#K~B3gy_;vmr_U8>B9RNd__;R5JLfIrL37vx>HV$iNA2YH;An0O|8^?@JPe={%e zvaN=Z@NixWe?$bDz2WWMrST3{uk#8n#JNeb1myAv6G#L;^F zt(EpiWSiTQ0|%9C`$DH@854|IzJ$$a8=zsm)@^0M_g}7J`3!v==_A8aW7L}H$g~AU zuJ*u9^AU!=+y@EG0!FKA`Mfo!O#7VOYYV7Ma#H%Js(9)`|SOchL+ z4tZjHJ^b7h#FTGM7uKAS((cRQFX{3*r=|2cuk(^?SX$Rfta?w{_O|FJZ`%!Ful?Db z%<91-_WDJhDJ3oPM1Ae!?~=l9R!k+L+>Sx-w6HS5x>ng&Q)}wP|>naB^ za>1e5TrREC@_-W&WYpwr?;U`QRr5tz)_|cr-{g>!=PR*wtcXpDi?W=WPX{)~dfkT0 zNXYgP9IG#m_c?SLbTSYq9IJ}XlC?WY#aS73L^e!ntMKyTGzQY^-FHPi_{nLonMCFr z)V4G6hxj;a9k6Ku6q&W*jf?w-1!vKl72M8PN27=b&M;U71bVa zBj2U^S{;q&)g0m_3JdkUeNy+MfM&eobue8)Jt9ryIz9$Ys6-=n#m;r4fv?+WF%{Wz zaVnM?9P5Mrps{le+8#*7Bg4AHY4*WQzNec0X|^5+V#ntCly>Lv(a`thqopG$p1t_W z4|E;NhEy%Zk+g%u3w-8loQ-)j_%8C#t#j|NRM+&c&HCKkAmKDYL5AjAwQw5^LMOuy ze3gr`VrhAH6_cXskPrc0ONlwj>}I|CA3 zIyCWQaj!ejNxJsQ_VYiAunR`jaiRO^bln`i@Id`-+D}a7#{Any4ed+irFCO>I&_9qrP9& z=*p`Ic^VD+?)3}qo!Y`I)~4C*9M~#qV5%mD2jU10IMeXx%OoBBRe8&ni^KtO!p&5M zbPHTuktR^LIU^duG0-&CKhCf@uf)Ex%1(Qb5SNW_Zivtq5J1P?4IVRIMw0;|xuE#b zFoF%9GCoS1QS``J&zX;szp3)GZ=Gph98_+>L9S`#Sp^#ScLTpN0v0YrX=T)_&BM^UcTCTjE2vj z>UrPjfv!IlVasTvwGw08a=WaEg;}rl7HHSUPe<9s1)-G{3BQc=W2+y17B4UcMHX8w z4hK*bz|-=!7R&1NdPi%nc7q#n#74F*IH+#0C=}5Y{?(>vCL==NRZsofbp9D{C9c(( zr*?VOXz0l7k#(GKIdYAM$OQe9%|3kO8%(+GEMk$w0FQUD7d8(#7ikp*`X=sal)<-Q z_(3s8`6o169l04-I)z$ew<7PV9{Z~svpRaecC!(%F@jw`NL{N(Pw)oHE?+ay@Vt{8Y{6Sc9wmxnQtP$4S$(^#Tm4-(4xZ?6b5~m0&+=Lcl|In%ef1qK$~{J zQKLhw9;>RU2L=<9rXCtilfWHF!C&el9OhiQ6&wjc0gNUc`y;T{h8Y&k=vw`E`DZ@f z?)of2y`;@_EuZ}$O{vdqnXb4s@lfC73DfAHWSK0 z>k*oVqw>?rKvf4*Yi{x8Hw2$XKot^SLSM)6B-*aXLsUEaoBY!X=)b5AZ<4 zz@hC^MR!}3S}^07pCd|9tLu@T)m z5M9xRG_@^Vyd&cVKlQmw$#tyD;dNHND?b>XDz=5$qqHW!8bF-=6_PUGPFWOE$miXlOrWqKXf!I8@$M! zCS=gQ0!3&+BH@Ubapo$N!lV#9!L~e?OPk}w>Emf(u4;({eKUt|fMl}#h*IHu-6NK& zmg@<}zv%K8s^9hF?Rs~D;G_O(>RcAQ*>$#eXbUd;*5ZdXoDCFtR&+nYrh2_9e>300 zCL-GDYcJ?K?M6;ss-qvK;{UGZ#dc3V+tneBvKMLcRI65NA0T}$@hj{4HPjwE z1NTTTw;g@N;%K&@FLt>~y|z1jZZyH1@m1Q!iPa`1Okxo<^xywT-EAcDOUq_u*SPKZ z2$e}rWTX07IS#|1;e_qs$oN)F)qGS%)<4^u|Eg-SIuY8m^vwVj)4H#Vot)fYInf+!GfzwGGzP;@0DIxYhUe$dQ1+MZ<{ zeJlSUvia>&$N-|R_N|JtT&%mF;(cOIoegz=k#N*JX`+tIo1L zd{0!sH+P5QXotMpw6~YTT5aTj`~_dtjomsf_;8KAZ^BBx-!2E+$<}52)n%QNvM6h$ z{dfDUUn_3M@@kUT0MZSP)z*{0^!-EWL@|ux!lBcS>Yd!)s%aoYPAU)D45M3jhsP$5 z_c{$)uxeu58;)ZgProol1zEf)?)>JeO7$up1yAU%nc73&DniK0TvFQ=u1Hl=^;9*V zc0D|BuAFgyS!LU`NlC3>{OJ)&Ud^~P`f4DWy$nj8~>El%by&|9=#Q>jdv(e}!7SL2RZ_G;cUY+%|WA6Lt1Tx7t zrAo&^9;rdu&~wVss;AdH*?~x4RWkpq`Y+t{d$)@s5l*Id_}H?$lBM5eGfgbqN<+NBl@B0AXWk?&Jt&o9u#>K{I9{rkiXjZoFJ zHEB9J6@f9bn$(|@%IHHerWccWx<6Qe-1J6;tLNn`;pf-JVXRBe>lV0 zwr6&)2;4ef4T@yiqUuEb%auP9{7?m>U0342+zw`(cD6jCTKtZ<>+qKzhqtHU8l@$N zvyzN|b|kL@Y|EWIXQ@DuY>`oC_?j4-1zXeY^YsMl(3)BH_jIgD;{E#r$sTJmq*G7V zk}GivF8?_eo|Bj}2o{)n9q(Jx^GRsID{6_rB`-<4D)XbXsXozOLDOsGoTegs7?RK< z6Z~&6|NA4fS3d6D`@~hd<>sM@;CJ^g}56LF|A@oZ$ zjByrjH03XUziloWy2a#09NJ(;u*xuRg7dYASfImRRN+dVZ;Y6>vD+E^GJsXNC{>{g z%$VQ29jxk6{#^S};9vG#Y&shU!?Efgy67YGcRpD#A1^xZjdKny#JHsJ6%J-pgl0_B zz3_KN4KSm{<12q9CJ>+0B50mA=&aRH$jLKj}MdQ))Q-SmADwr`Q_#Y_)7!aFS1jHQu@@*)~{~l&X zEF;r7vR@A813jqE#w7-zk!zV-je7lEAt{)TB)IwS3Yi*-@75B48CAfH$`o9GXRLx| zym#lEHX9q8$1Rk!ZSW$w0E)+x(?ZMSq z2Y++tTm+uqS${<;P+dGPlhQT{z8j6j`}5pC+Qz>OOKbxdhUzcaP0K$r2vnJ4jO%AH zZ2F!{xxYf}&xpP|;Lh=VPkt`6{%xH8W!grq@aS6!|LX`DkVYKCyoMm+MX4P$$HmS zF4_XeL^Yj{;pqn~y9t-lG4sv`WUM#EMgLxp=ZXv;&{gx5%!v|nm&@va6H1|jiG{!8 zV(*ifp_YDWj>kW7F+{IcBI?x*4pD58B?Yn9=e$Jh#hOUYMfKkq>5G6~dPQ}zutW&B z+WLd$NtSF6y zmclT$XY%b(ihP*FnfV@;epz0a^Fq_h-((cI3*Rip; z0ZOyc=v9L!@Xn-|~Z@1kFl`>j*7-2_@2wzd}0UE1d((HY|^q|;lP-(RFX19^Zd z=uVcZWd;nY6PSzNwJInwYR_o*9+>=sMnCb=+D zb|}(aY(deOW(F)h_rJYK?|*oci(y3JKpQ&-1E7@BOSOQr^CvyG(` zbxJ0f#Pk_xQ)6?v+;0O!ps#X=Z1}e#*cSsN$mF<*D``N{E7enw4*MF^3eB@0d=7H? zw%TxdG7s|5Iq@t2PN?sb74Iqz3W6=*D2h3lkLj&l5L~-%@SLZDYvaLw z^Ut?r9^-_{9=%&4j*@2vM4|!s=%&wqqUc!zirnZ4?-A_$*sV(@TRZ{Tki3y3b5CtS z%-~I3d)POC0cF4i;9+iLy)n`T>;9uYU*q*mb~DVW+d$ek@SD8%kA3=}m&0_JxdcQt z&vH_0L4-x&9`MJ?K}o<*3LF(J!4vT-T1=s5>UUx;SVHFuYtb(5fTEi-_B=bF_@Em! z9bDO^5OgZ-UrA=ctn^d4?Go>&Li6h)AfOSQ{I|7LK=O%k5D_*2(B9kt^!fe4H#(vI ze4Mh;@{_}r7%1};Ye(1hsKHW zY<%GDJ~`-hM-FE(LnVLe#%W5YlsvDgaRY zP%Uup_-U_5vK{Q;)&j_DUR(=GnlvXls^%Z# zv%CiBy;ry3Qzyf7;`tCZJy=|j@Hat|*mZ#GH8nH`8tdpQpKcC{Lx2OLrIyFy1#58* ze7AS1GZPj*GYT&}0{q@_N4|8;424p4Zf1k3Z@TJ_kSjXXt<~LiZ=gJU^B5498w5zQ zLis2wfY4Jzjhq5dYB4U}8_5>7?3ZSl7norSywdSw*k{+W=U2SZi-h~54ZV6ZDC_AS>9#g0}%>oAahxmPn~UVDmK3)UVZ}-c>_QZY1JH$ z^O5l^7-3qwqO~R^&~wq166C%-?R@iMVLzcIoeQrW0_x*6VlZNbQZa0C^ieu|o7Lql3We7A@1MoN_<(oyf11!)b~s%eO>2 zceC?&kd?Eyf&eshb@2BbjX2WcD`x4Rb{$!>Ne&*LsYSGY9__^>N3_Ivq-75bocEuQ zlY;a*vBt_i&W2P+KkKil|1LCH~E_heq+7rlFJb~5d;%f&8Gc0t?4W z-PBHtN$H5QoFoo2%fm%S2PAM)-r}1FV%9RVrx~#^0xJhu0%EQEbal1%$3@L9%*zXo zk@sRLA&HwUa=D=d%I)NE+bv1pG$Wex#+cmrKB+>E(vrdRXx(EwGKK5$GZwh*v(6}1 z@G-oiqL&jt79bz`xXOUis3m**y(0iiuB{G{I(>119mR`MWPibN~H8||$LzR|ZYXz2e^4~_1f0MaJBJ%_;sckI} zznE{4r;+M4_ceMO5CVFZBDgpJZxP%6Dx;WXH0%*nlicy*zE9)xfGwM|#00(8eZUpY&G zs*EwJY>5CFDppim0hQA?Bh6)yi7{zp*1o^_fz72U7639@?08~R8_Dwg^h(z%mSs2g zp?{!*T2`GXD#xxUYT8VLU;K9b6#?)bW zQ3YAeP`N}sTyFNAj~}Y3X|%CMqYMPKBSB8`55cv$I+Aq$t_5+F0MVeAHRLW8C!?RS zFNfVXq0A&mtGa(Y)nl>*WyFHH@G`2_0P&IcC!uW#5~;utN0(Cgs0c3vgOD0j}i=YB^nEXL7F|G{g@q-`Sk>C8XXA0eXT@ z*ViSWYKP`;!NQC+szW)cS&fd(<1nm zT93IlocaY=b+vZ{Vu|hs`MGp*&`1HD;iq%x!f%x_RMO+7REM$#R=X)s8XPZ-&kVf| zuJ6zDd!G`>m%RaQ>+}w|c0>P3!s+pXmb|9t#(o1plz(z$tQKg6dtsa#nMsL&+v2bSV0HowGvR zO7ohXmH-8)$;miYU*Ak%wq6nw5p@8}vPFE2=k9N2aRp9@3Gx=FyNW+r)PyO1M$<}$ z2zrC+=#PDOe87EZFD-7G++j7v+zZNEM$&D*uwII>TbvQkK(iO#%vsH}4`}Hb<_U>@ zra1fJXd_GX{VlA@3YbZba(f?3mwUznh5a}iwCOj^ZjeMDWsRkQBziI=(Fb7;eTV|$*t1)O1-1k8W_-%MK>`lDGVEy z-1))TjE_14Jfj{L8q036MjV!;?xQTj^yA)7;BivHN~^w4UCu z8`YILtQP!X*}IwPm#!qso5m1Oir<_b$6k+fp-fUVbo+KgwTGhRBJ&&zsKgNc@|r=e zvKOeG6McPmehzd4CBP!`ObJNGU4UXFpBNz(SQZhud7h92R+!Z7QM(?VFM#m|$VSyNVZfax=Q1$C&Lihn7UF)ez9yK%v1Z?9R1_wa57!A&G zA~t^3R=6kAfHBS|0ZSICFL=NIUE4unC?9(dm<%Z#%2RR63h)rtYp&EKKnJM*A2^1; zOs^PI>F9yP(|v9o;m1t0)H<>t4SI161x(f}tz*MeR#W-%LDLoyg>j*z(?|(*{y9m( z5!dS+yLK*s6nyv0G~l@7v+b25aj;?Cy37;_mv?PQk{S5T z76Z$q(>*adAc7@Yxp-?1*g~f5@xupx*|O+JE5#&-$50dK6dNwkk6?d((N-8^eB(jo zkY6T^bj_Zu@W7e`1-OCsQb3zTU|1^4Bz~M4i-SE-f0u=dynW|VYC0-2FGi}Yap>ZOpIDJkM!O`u*?smMNc}X%P-7t$JE@cWGX}6WM-JJHD?_+l- zUkUy`{o=sH*4n0Ee0lN-c1>w|OsDjCJ44v~vr-@XC1v~NCaQbukRd7M3{24edpq;H z3!1Nd?@|;roH&)Ttc1vtd#exjy8&tFxRp~xjd!5|I)ARI(EQVZu0e?F$4e!|Zr0X7}p%>QV=HN?8$ggG#S1@MHqx1D!PN z4s>e_;DcMEMv4EP^nK1tct>1f8Lj)V_dqF4H0EGN%+=}V9;0h1+LIHky6Rt?tbE`o zM?S%p&TIh`!qVOJ2B`TD0F4TAlFE$-zJ*9tJVi&A(oieMyH$po3x(cti&r5B{uIdn z9jXA-!|SHzZH}xaddxT6M9EnIzPHJpw4_I0CGCGqtBUez+Tbq$eV>w|nXP`&PRf0w z=x{klA|}j&$KaRH(dz8Im;QD*@wK3)mH>-?RvPfZ{s}_c9?xo_+tNXo9xCceVc@U+ zI8f;`$;i-H2Wh+ThV>6@98@&*I6edlPRspNtQ4ItAkWc%U_i(bV|?m(o;#&IsDfxTbNC zKCabXvp5Zmz57g6xY@AZ_9wi29g2{OyoxaJOy+9gxM(w4sVZ^=l9-KG0p_dBBcld6 zkib3p4VK<+NyxWI%YE$a#LTpqXM0i!+yVu-zba<~{mP9KK;}+KQYk5E!^|fp3Z?XC z#%6Up!^p>C&CcH<0F!K>;B`pSX@CN}nUf@N^@UMVW^Kn2&~R2;riAvV-%~Cx{fgU; zB%Kyo&P+*mB~6@s<;Oy?{Gbj1BXa=#YFJr7{cxM3eIZpcgo01IBGKd{X+yX0Jg!v( zFwzOI%3WVXw@J-ro8HO*#U1%?!2Q8V$TRKHA|bSSxuO>;B_MW1 z7V;sGK0iNN7~{HhAG8-Z7^FS;E!8=z40y)oE8omgNYJ?2%mU-s$Oz|Zx6c(q_(%bo zeZdy|br!kn^k(%p;ZprbqO^|*3zfJ3#PI3;fUl|ehqC9U`mdJJ456JYdXc3IVJFa< z%)MrhzF!#sxlVNwxjze9hfNM(B4s0O@Pn?)sa-({sLz?- z1GqM38pCPfFBoC*q|=c77Vku9&f}TUkwwG8@mb&Knvf2tIi`#;E0!!3l=IM77~KB} zH9!y5cF5n~T60<8=ljM|2d>}^U{q}aql)#nQFV3&1E%?kW)B;ubX>qu(9*|E^|nuD zn3C*F^r3Ip7%pVV`6>D3UL9d;GWdM5IV59Vo}1bW743>_e5%^yX`f3;9nTI@$R%~3 z-%c=yp(2vP@xohgakp?(1-39;axU)zrly6%l^wzpP}*@hV5}BE4sf=c9Isv=5-7~7 zC?_(h2W~cFqbu#8OHTGuW;|A|Y z)0W|&BWv|10KR$3Zo4F1Tg~}$DOefhv@L{ zL=AZx;NcQ4NWz#exD-$XB@HKO@Ng<5VdVczE-D1Qf0w+y;ZHR{M^I{V|9arZ8dvL@ zct)M&YWpDRrj(6a+7MC`uY<`d-(^Qnfyg>7!={9JVtyopVHWuIdc!t=gUXaK#rr6@ zXjf9>b@D9P_6VX>QZK=$)ud|$f~bFUgs9*Su+F!E9T<;q$nH?Z)47}ge&on6X%3+e zTCPehhV#mGLCI6arWe{; zCLaxl+VTsP@&cdIIG=MxbWwA}b^fHpB75_w0#}6TJk%cmZg;%VO3D9B+$J~9r+JjW z8FGMU67sf8&KayuI3N@3Krgk3WY?WULk`ug?ome( z&1k!QLA{0O2X@pXTcBJp8fwxr%0Cs|V|ZcQ0QkIQR*Hw)fxpyzmFi=EQMPH=8Zd?L z@3Qjhy&vf368wC6u#hlS$|_yAe_^w7^yapIU31SU|D|rwwIzV6yf5kLg>aC(Ruu>{0}v*~OX`Qqk;(`4pll%?dq>?E5~nOZ1YxtY|)p0@bC%d7YHg zJ+sL)BzB-MgWgh7ZdH@7WzyE@s-{&_YWMB>d?0WyOH6zHzPq=j6a6Q8+PAE*D&s6F zPhzP&-9ShHnQ_k;a{auA40yZnDzWM)=adoN$HiSWy3W)!ymG6y%A!Dl;`kPmFzvMQ zom>u{(8SDYc(gsC8+e-aR)2*&5wzm$l7uC@WRkCgJ2SjG!AS&O_1NEumuVvPdmtym zDc1C(9|&Zy8AtrEWz2AT%7g$Y2Ryb7R-$bBLmP|*alPM+)W`^`UeWyw{1O(knLmj% zmgcs7wIJ|-;@y*lSWuuqdD5K0q!g7_Wtc^t@O*zXrxUodOsQSB6;!czCGhmfq8t9(OIAB6qEyXJ!{#g&=G(!b?Qff3H z+_95oD)kAvqV78YsrsHffBd?Uze>IogM0uKC#&3K-~(!)EQpg0s9eAP1Y{=lS~C(* z{uVyxS%igal7rp;M}|@DM{@Ub0w}R?z%$CzF>#KZ zmeRm7w9T``NN6vgK^~jn+u<6{`vf_T?X!<4gwuguo=JDPxL57)H#4ROROhzc;d6rfQ1?|0o#p8^#sKRr%t};j7 zFEptKV|y~+8lhwd6wDkn^rGV78wXDkA-d$?n*O1S3Q(I1-rg{DNimx-(CgCH2zr7E zjHg@E@8KP+b!gIx#;-|$f?9}C{~q1GZ^KR`aU2S*A`frxogLoqfD~G_d<-&yUz4P8 z-bSRu+Kd9QKq14QiWU$ObFOKzy!Z)(*wRSX(UD={C0mI%z6sE$<%+Qfc#!eT%|y+| z=1}WWlTQB0+1rL(y$Q4{(Oo~J4Wl2jy5C`}$Cw~#VSW9avelvfGM*gqwPF#0!o1b{ z$lYM34}%bS1#!IM@E5;NT%h%#l_y4br(;GGy;H^+J4`zr2BamH*9l;3{QOC}c;Xdz;e(_6X69RF3>4W%?D8YZ<|S(mU@xZ_(2;NMi}`^>d+3=2Uut6l-H60xo*x_gNW>(SD<5V;(=5K zC_VD!eZT;DpQ0sX Xix-<}SaP9(|0Km_#quA&eD{9<(|qx3 literal 0 HcmV?d00001 diff --git a/docs/source/models/dssm_derivatives.md b/docs/source/models/dssm_derivatives.md new file mode 100644 index 000000000..2e2a525e0 --- /dev/null +++ b/docs/source/models/dssm_derivatives.md @@ -0,0 +1,77 @@ +# DSSM衍生扩展模型 + +## DSSM + SENet +### 简介 + +在推荐场景中，往往存在多种用户特征和物品特征，特征类型各不相同，各种特征经过embedding层后进入双塔模型的DNN层进行训练，在部分场景中甚至还会引入多模态embedding特征, 如图像和文本的embedding。 +然而各个特征对目标的影响不尽相同，有的特征重要性高，对模型整体表现影响大，有的特征则影响较小。因此当特征不断增多时，可以结合SENet自动学习每个特征的权重，增强重要信息到塔顶的能力。 + + +![dssm+senet](../../images/models/dssm+senet.png) + +### 配置说明 + +```protobuf +model_config:{ + model_class: "DSSM_SENet" + feature_groups: { + group_name: 'user' + feature_names: 'user_id' + feature_names: 'cms_segid' + feature_names: 'cms_group_id' + feature_names: 'age_level' + feature_names: 'pvalue_level' + feature_names: 'shopping_level' + feature_names: 'occupation' + feature_names: 'new_user_class_level' + feature_names: 'tag_category_list' + feature_names: 'tag_brand_list' + wide_deep:DEEP + } + feature_groups: { + group_name: "item" + feature_names: 'adgroup_id' + feature_names: 'cate_id' + feature_names: 'campaign_id' + feature_names: 'customer' + feature_names: 'brand' + #feature_names: 'price' + #feature_names: 'pid' + wide_deep:DEEP + } + dssm_senet { + user_tower { + id: "user_id" + senet { + num_squeeze_group : 2 + reduction_ratio: 4 + } + dnn { + hidden_units: [128, 32] + } + } + item_tower { + id: "adgroup_id" + senet { + num_squeeze_group : 2 + reduction_ratio: 4 + } + dnn { + hidden_units: [128, 32] + } + } + simi_func: COSINE + scale_simi: false + temperature: 0.01 + l2_regularization: 1e-6 + } + loss_type: SOFTMAX_CROSS_ENTROPY + embedding_regularization: 5e-5 +} +``` + +### 示例Config +[dssm_senet_on_taobao.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/dssm_senet_on_taobao.config) + +### 参考论文 +[Squeeze-and-Excitation Networks](https://arxiv.org/abs/1709.01507) \ No newline at end of file diff --git a/easy_rec/python/model/dssm_senet.py b/easy_rec/python/model/dssm_senet.py index 1866b08a3..f1c5446bb 100644 --- a/easy_rec/python/model/dssm_senet.py +++ b/easy_rec/python/model/dssm_senet.py @@ -9,13 +9,14 @@ from easy_rec.python.protos.simi_pb2 import Similarity from easy_rec.python.utils.proto_util import copy_obj from easy_rec.python.layers import senet +from easy_rec.python.model.dssm import DSSM if tf.__version__ >= '2.0': tf = tf.compat.v1 losses = tf.losses -class DSSM_SENet(MatchModel): +class DSSM_SENet(DSSM): def __init__(self, model_config, @@ -23,9 +24,10 @@ def __init__(self, features, labels=None, is_training=False): - super(DSSM_SENet, self).__init__(model_config, feature_configs, features, labels, - is_training) - assert self._model_config.WhichOneof('model') == 'DSSM_SENet', \ + + MatchModel.__init__(self, model_config, feature_configs, features, labels, is_training) + + assert self._model_config.WhichOneof('model') == 'dssm_senet', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') self._model_config = self._model_config.dssm_senet assert isinstance(self._model_config, DSSM_SENet_Config) @@ -133,50 +135,7 @@ def build_predict_graph(self): tf.as_string(item_tower_emb), axis=-1, separator=',') return self._prediction_dict - def get_outputs(self): - if self._loss_type == LossType.CLASSIFICATION: - return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb' - ] - elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: - self._prediction_dict['logits'] = tf.squeeze( - self._prediction_dict['logits'], axis=-1) - self._prediction_dict['probs'] = tf.nn.sigmoid( - self._prediction_dict['logits']) - return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb' - ] - elif self._loss_type == LossType.L2_LOSS: - return ['y', 'user_emb', 'item_emb', 'user_tower_emb', 'item_tower_emb'] - else: - raise ValueError('invalid loss type: %s' % str(self._loss_type)) - def build_output_dict(self): - output_dict = super(DSSM_SENet, self).build_output_dict() - # output_dict['user_tower_feature'] = tf.reduce_join( - # tf.as_string(self.user_tower_feature), axis=-1, separator=',') - # output_dict['item_tower_feature'] = tf.reduce_join( - # tf.as_string(self.item_tower_feature), axis=-1, separator=',') - return output_dict - - def build_rtp_output_dict(self): - output_dict = super(DSSM_SENet, self).build_rtp_output_dict() - if 'user_tower_emb' not in self._prediction_dict: - raise ValueError( - 'User tower embedding does not exist. Please checking predict graph.') - output_dict['user_embedding_output'] = tf.identity( - self._prediction_dict['user_tower_emb'], name='user_embedding_output') - if 'item_tower_emb' not in self._prediction_dict: - raise ValueError( - 'Item tower embedding does not exist. Please checking predict graph.') - output_dict['item_embedding_output'] = tf.identity( - self._prediction_dict['item_tower_emb'], name='item_embedding_output') - if self._loss_type == LossType.CLASSIFICATION: - if 'probs' not in self._prediction_dict: - raise ValueError( - 'Probs output does not exist. Please checking predict graph.') - output_dict['rank_predict'] = tf.identity( - self._prediction_dict['probs'], name='rank_predict') - return output_dict + output_dict = MatchModel.build_output_dict(self) + + return output_dict \ No newline at end of file diff --git a/easy_rec/python/protos/dssm_senet.proto b/easy_rec/python/protos/dssm_senet.proto index 90471d268..fd49b9f76 100644 --- a/easy_rec/python/protos/dssm_senet.proto +++ b/easy_rec/python/protos/dssm_senet.proto @@ -3,9 +3,9 @@ package protos; import "easy_rec/python/protos/dnn.proto"; import "easy_rec/python/protos/simi.proto"; -import "easy_rec/python/protos/senet.proto" +import "easy_rec/python/protos/layer.proto"; -message DSSMTower { +message DSSM_SENet_Tower { required string id = 1; required SENet senet = 2; required DNN dnn = 3; @@ -14,8 +14,8 @@ message DSSMTower { message DSSM_SENet { - required DSSMTower user_tower = 1; - required DSSMTower item_tower = 2; + required DSSM_SENet_Tower user_tower = 1; + required DSSM_SENet_Tower item_tower = 2; required float l2_regularization = 3 [default = 1e-4]; optional Similarity simi_func = 4 [default=COSINE]; // add a layer for scaling the similarity diff --git a/easy_rec/python/protos/easy_rec_model.proto b/easy_rec/python/protos/easy_rec_model.proto index 56f5b713e..6fee5ebea 100644 --- a/easy_rec/python/protos/easy_rec_model.proto +++ b/easy_rec/python/protos/easy_rec_model.proto @@ -27,6 +27,7 @@ import "easy_rec/python/protos/variational_dropout.proto"; import "easy_rec/python/protos/multi_tower_recall.proto"; import "easy_rec/python/protos/tower.proto"; import "easy_rec/python/protos/pdn.proto"; +import "easy_rec/python/protos/dssm_senet.proto"; // for input performance test message DummyModel { @@ -106,6 +107,7 @@ message EasyRecModel { DropoutNet dropoutnet = 203; CoMetricLearningI2I metric_learning = 204; PDN pdn = 205; + DSSM_SENet dssm_senet = 206; MMoE mmoe = 301; ESMM esmm = 302; diff --git a/easy_rec/python/protos/senet.proto b/easy_rec/python/protos/senet.proto deleted file mode 100644 index 9830d9211..000000000 --- a/easy_rec/python/protos/senet.proto +++ /dev/null @@ -1,8 +0,0 @@ -syntax = "proto2"; -package protos; - - -message SENet { - uint32 num_squeeze_group = 1 [default = 2]; - uint32 reduction_ratio = 2 [default = 4]; -} diff --git a/samples/model_config/dssm_senet_on_taobao.config b/examples/configs/dssm_senet_on_taobao.config similarity index 99% rename from samples/model_config/dssm_senet_on_taobao.config rename to examples/configs/dssm_senet_on_taobao.config index 9bee96337..f8f415d1a 100644 --- a/samples/model_config/dssm_senet_on_taobao.config +++ b/examples/configs/dssm_senet_on_taobao.config @@ -1,6 +1,6 @@ train_input_path: "data/test/tb_data/taobao_train_data" eval_input_path: "data/test/tb_data/taobao_test_data" -model_dir: "experiments/dssm_taobao_ckpt" +model_dir: "experiments/dssm_senet_taobao_ckpt" train_config { log_step_count_steps: 100 From eb31675be65a9de29153fac75cc27abb865c3a3c Mon Sep 17 00:00:00 2001 From: "eric.gc" Date: Fri, 11 Oct 2024 13:58:43 +0800 Subject: [PATCH 3/7] update recall.rst --- docs/source/models/recall.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/models/recall.rst b/docs/source/models/recall.rst index 527c0db6a..86187ccdd 100644 --- a/docs/source/models/recall.rst +++ b/docs/source/models/recall.rst @@ -6,6 +6,7 @@ dssm dssm_neg_sampler + dssm_derivatives mind co_metric_learning_i2i pdn From 66dcbac4d88f24141f77cd35bdabf75c853c027e Mon Sep 17 00:00:00 2001 From: "eric.gc" Date: Fri, 11 Oct 2024 14:07:53 +0800 Subject: [PATCH 4/7] update senet docs --- docs/source/models/dssm_derivatives.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/models/dssm_derivatives.md b/docs/source/models/dssm_derivatives.md index 2e2a525e0..747bf696f 100644 --- a/docs/source/models/dssm_derivatives.md +++ b/docs/source/models/dssm_derivatives.md @@ -70,6 +70,10 @@ model_config:{ } ``` +- senet参数配置: + - num_squeeze_group: 每个特征embedding的分组个数， 默认为2 + - reduction_ratio: 维度压缩比例， 默认为4 + ### 示例Config [dssm_senet_on_taobao.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/dssm_senet_on_taobao.config) From 6d0bae6a15b2fa4e707faea8732effb4a84f0132 Mon Sep 17 00:00:00 2001 From: "eric.gc" Date: Fri, 11 Oct 2024 16:16:53 +0800 Subject: [PATCH 5/7] code style fix --- docs/source/models/dssm_derivatives.md | 10 +++--- easy_rec/python/layers/senet.py | 36 +++++++++---------- easy_rec/python/model/dssm_senet.py | 46 +++++++++++++------------ easy_rec/python/protos/dssm_senet.proto | 2 +- 4 files changed, 49 insertions(+), 45 deletions(-) diff --git a/docs/source/models/dssm_derivatives.md b/docs/source/models/dssm_derivatives.md index 747bf696f..d74aa9057 100644 --- a/docs/source/models/dssm_derivatives.md +++ b/docs/source/models/dssm_derivatives.md @@ -1,12 +1,12 @@ -# DSSM衍生扩展模型 +# DSSM衍生扩展模型 ## DSSM + SENet + ### 简介 在推荐场景中，往往存在多种用户特征和物品特征，特征类型各不相同，各种特征经过embedding层后进入双塔模型的DNN层进行训练，在部分场景中甚至还会引入多模态embedding特征, 如图像和文本的embedding。 然而各个特征对目标的影响不尽相同，有的特征重要性高，对模型整体表现影响大，有的特征则影响较小。因此当特征不断增多时，可以结合SENet自动学习每个特征的权重，增强重要信息到塔顶的能力。 - ![dssm+senet](../../images/models/dssm+senet.png) ### 配置说明 @@ -70,12 +70,14 @@ model_config:{ } ``` -- senet参数配置: +- senet参数配置: - num_squeeze_group: 每个特征embedding的分组个数， 默认为2 - reduction_ratio: 维度压缩比例， 默认为4 ### 示例Config + [dssm_senet_on_taobao.config](https://github.com/alibaba/EasyRec/tree/master/examples/configs/dssm_senet_on_taobao.config) ### 参考论文 -[Squeeze-and-Excitation Networks](https://arxiv.org/abs/1709.01507) \ No newline at end of file + +[Squeeze-and-Excitation Networks](https://arxiv.org/abs/1709.01507) diff --git a/easy_rec/python/layers/senet.py b/easy_rec/python/layers/senet.py index 5715d189f..777079341 100644 --- a/easy_rec/python/layers/senet.py +++ b/easy_rec/python/layers/senet.py @@ -7,8 +7,7 @@ class SENet: - ''' - Squeeze and Excite Network + """Squeeze and Excite Network. Input shape - A list of 2D tensor with shape: ``(batch_size,embedding_size)``. @@ -20,15 +19,20 @@ class SENet: reduction_ratio: int, reduction ratio for squeeze. l2_reg: float, l2 regularizer for embedding. name: str, name of the layer. + """ - ''' - def __init__(self, num_fields, num_squeeze_group, reduction_ratio, l2_reg, name='SENet'): + def __init__(self, + num_fields, + num_squeeze_group, + reduction_ratio, + l2_reg, + name='SENet'): self.num_fields = num_fields self.num_squeeze_group = num_squeeze_group self.reduction_ratio = reduction_ratio self._l2_reg = l2_reg self._name = name - + def __call__(self, inputs): g = self.num_squeeze_group f = self.num_fields @@ -39,7 +43,6 @@ def __call__(self, inputs): for input in inputs: emb_size += int(input.shape[-1]) - group_embs = [ tf.reshape(emb, [-1, g, int(emb.shape[-1]) // g]) for emb in inputs ] @@ -50,24 +53,21 @@ def __call__(self, inputs): squeezed.append(tf.reduce_mean(emb, axis=-1)) # [B, g] z = tf.concat(squeezed, axis=1) # [bs, field_size * num_groups * 2] - - reduced = tf.layers.dense( - inputs=z, - units=reduction_size, - kernel_regularizer=self._l2_reg, - activation='relu', - name='%s/reduce' % self._name) - + inputs=z, + units=reduction_size, + kernel_regularizer=self._l2_reg, + activation='relu', + name='%s/reduce' % self._name) + excited_weights = tf.layers.dense( inputs=reduced, - units=emb_size, - kernel_initializer='glorot_normal', + units=emb_size, + kernel_initializer='glorot_normal', name='%s/excite' % self._name) - # Re-weight inputs = tf.concat(inputs, axis=-1) output = inputs * excited_weights - return output \ No newline at end of file + return output diff --git a/easy_rec/python/model/dssm_senet.py b/easy_rec/python/model/dssm_senet.py index f1c5446bb..406d3cbdf 100644 --- a/easy_rec/python/model/dssm_senet.py +++ b/easy_rec/python/model/dssm_senet.py @@ -3,13 +3,14 @@ import tensorflow as tf from easy_rec.python.layers import dnn +from easy_rec.python.layers import senet +from easy_rec.python.model.dssm import DSSM from easy_rec.python.model.match_model import MatchModel -from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.protos.simi_pb2 import Similarity from easy_rec.python.utils.proto_util import copy_obj -from easy_rec.python.layers import senet -from easy_rec.python.model.dssm import DSSM + +from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -25,7 +26,8 @@ def __init__(self, labels=None, is_training=False): - MatchModel.__init__(self, model_config, feature_configs, features, labels, is_training) + MatchModel.__init__(self, model_config, feature_configs, features, labels, + is_training) assert self._model_config.WhichOneof('model') == 'dssm_senet', \ 'invalid model config: %s' % self._model_config.WhichOneof('model') @@ -35,13 +37,15 @@ def __init__(self, # copy_obj so that any modification will not affect original config self.user_tower = copy_obj(self._model_config.user_tower) - self.user_seq_features, self.user_plain_features, self.user_feature_list = self._input_layer(self._feature_dict, 'user', is_combine=False) + self.user_seq_features, self.user_plain_features, self.user_feature_list = self._input_layer( + self._feature_dict, 'user', is_combine=False) self.user_num_fields = len(self.user_feature_list) # copy_obj so that any modification will not affect original config self.item_tower = copy_obj(self._model_config.item_tower) - self.item_seq_features, self.item_plain_features, self.item_feature_list = self._input_layer(self._feature_dict, 'item', is_combine=False) + self.item_seq_features, self.item_plain_features, self.item_feature_list = self._input_layer( + self._feature_dict, 'item', is_combine=False) self.item_num_fields = len(self.item_feature_list) self._user_tower_emb = None @@ -49,12 +53,11 @@ def __init__(self, def build_predict_graph(self): user_senet = senet.SENet( - num_fields=self.user_num_fields, - num_squeeze_group=self.user_tower.senet.num_squeeze_group, - reduction_ratio=self.user_tower.senet.reduction_ratio, - l2_reg=self._l2_reg, - name='user_senet' - ) + num_fields=self.user_num_fields, + num_squeeze_group=self.user_tower.senet.num_squeeze_group, + reduction_ratio=self.user_tower.senet.reduction_ratio, + l2_reg=self._l2_reg, + name='user_senet') user_senet_output_list = user_senet(self.user_feature_list) user_senet_output = tf.concat(user_senet_output_list, axis=-1) @@ -70,15 +73,14 @@ def build_predict_graph(self): name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1)) item_senet = senet.SENet( - num_fields=self.item_num_fields, - num_squeeze_group=self.item_tower.senet.num_squeeze_group, - reduction_ratio=self.item_tower.senet.reduction_ratio, - l2_reg=self._l2_reg, - name='item_senet' - ) - + num_fields=self.item_num_fields, + num_squeeze_group=self.item_tower.senet.num_squeeze_group, + reduction_ratio=self.item_tower.senet.reduction_ratio, + l2_reg=self._l2_reg, + name='item_senet') + item_senet_output_list = item_senet(self.item_feature_list) - item_senet_output = tf.concat(item_senet_output_list, axis=-1) + item_senet_output = tf.concat(item_senet_output_list, axis=-1) num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) last_item_hidden = self.item_tower.dnn.hidden_units.pop() @@ -137,5 +139,5 @@ def build_predict_graph(self): def build_output_dict(self): output_dict = MatchModel.build_output_dict(self) - - return output_dict \ No newline at end of file + + return output_dict diff --git a/easy_rec/python/protos/dssm_senet.proto b/easy_rec/python/protos/dssm_senet.proto index fd49b9f76..ee941104f 100644 --- a/easy_rec/python/protos/dssm_senet.proto +++ b/easy_rec/python/protos/dssm_senet.proto @@ -9,7 +9,7 @@ message DSSM_SENet_Tower { required string id = 1; required SENet senet = 2; required DNN dnn = 3; - + }; From 5a9b8e0a1d77a65c64c128f1d0347806bc961e86 Mon Sep 17 00:00:00 2001 From: "eric.gc" Date: Sat, 12 Oct 2024 10:22:40 +0800 Subject: [PATCH 6/7] code style fix --- easy_rec/python/compat/early_stopping.py | 2 +- easy_rec/python/model/dssm_senet.py | 3 +-- easy_rec/python/test/train_eval_test.py | 2 +- setup.cfg | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/easy_rec/python/compat/early_stopping.py b/easy_rec/python/compat/early_stopping.py index fe4c12132..fc850fb62 100644 --- a/easy_rec/python/compat/early_stopping.py +++ b/easy_rec/python/compat/early_stopping.py @@ -21,9 +21,9 @@ import os import threading import time -from distutils.version import LooseVersion import tensorflow as tf +from distutils.version import LooseVersion from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import init_ops diff --git a/easy_rec/python/model/dssm_senet.py b/easy_rec/python/model/dssm_senet.py index 406d3cbdf..40b7faec6 100644 --- a/easy_rec/python/model/dssm_senet.py +++ b/easy_rec/python/model/dssm_senet.py @@ -6,12 +6,11 @@ from easy_rec.python.layers import senet from easy_rec.python.model.dssm import DSSM from easy_rec.python.model.match_model import MatchModel +from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config # NOQA from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.protos.simi_pb2 import Similarity from easy_rec.python.utils.proto_util import copy_obj -from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config - if tf.__version__ >= '2.0': tf = tf.compat.v1 losses = tf.losses diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 0f7b82a28..ca29fc89c 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -7,11 +7,11 @@ import threading import time import unittest -from distutils.version import LooseVersion import numpy as np import six import tensorflow as tf +from distutils.version import LooseVersion from tensorflow.python.platform import gfile from easy_rec.python.main import predict diff --git a/setup.cfg b/setup.cfg index 337833a0f..b43211827 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,7 @@ multi_line_output = 7 force_single_line = true known_standard_library = setuptools known_first_party = easy_rec -known_third_party = absl,common_io,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,yaml +known_third_party = absl,common_io,distutils,docutils,eas_prediction,faiss,future,google,graphlearn,kafka,matplotlib,numpy,oss2,pai,pandas,psutil,six,sklearn,sparse_operation_kit,sphinx_markdown_tables,sphinx_rtd_theme,tensorflow,yaml no_lines_before = LOCALFOLDER default_section = THIRDPARTY skip = easy_rec/python/protos From d66ee9e43325f1f70ce23c104a48e4b0ad227e6c Mon Sep 17 00:00:00 2001 From: "eric.gc" Date: Mon, 14 Oct 2024 16:40:53 +0800 Subject: [PATCH 7/7] add dssm_senet unit test --- easy_rec/python/model/dssm_senet.py | 3 +- easy_rec/python/test/train_eval_test.py | 6 + .../model_config/dssm_senet_on_taobao.config | 321 ++++++++++++++++++ 3 files changed, 329 insertions(+), 1 deletion(-) create mode 100644 samples/model_config/dssm_senet_on_taobao.config diff --git a/easy_rec/python/model/dssm_senet.py b/easy_rec/python/model/dssm_senet.py index 40b7faec6..c84d52161 100644 --- a/easy_rec/python/model/dssm_senet.py +++ b/easy_rec/python/model/dssm_senet.py @@ -6,11 +6,12 @@ from easy_rec.python.layers import senet from easy_rec.python.model.dssm import DSSM from easy_rec.python.model.match_model import MatchModel -from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config # NOQA from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.protos.simi_pb2 import Similarity from easy_rec.python.utils.proto_util import copy_obj +from easy_rec.python.protos.dssm_senet_pb2 import DSSM_SENet as DSSM_SENet_Config # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 losses = tf.losses diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index ca29fc89c..ca98a4192 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -1248,6 +1248,12 @@ def test_pdn(self): 'samples/model_config/pdn_on_taobao.config', self._test_dir) self.assertTrue(self._success) + @unittest.skipIf(gl is None, 'graphlearn is not installed') + def test_dssm_senet(self): + self._success = test_utils.test_single_train_eval( + 'samples/model_config/dssm_senet_on_taobao.config', self._test_dir) + self.assertTrue(self._success) + if __name__ == '__main__': tf.test.main() diff --git a/samples/model_config/dssm_senet_on_taobao.config b/samples/model_config/dssm_senet_on_taobao.config new file mode 100644 index 000000000..3c059f6e2 --- /dev/null +++ b/samples/model_config/dssm_senet_on_taobao.config @@ -0,0 +1,321 @@ +train_input_path: "data/test/tb_data/taobao_train_data" +eval_input_path: "data/test/tb_data/taobao_test_data" +model_dir: "experiments/dssm_senet_taobao_ckpt" + +train_config { + log_step_count_steps: 200 + optimizer_config: { + adam_optimizer: { + learning_rate: { + exponential_decay_learning_rate { + # initial_learning_rate: 0.001 + initial_learning_rate: 0.0001 + decay_steps: 4000 + decay_factor: 0.5 + min_learning_rate: 0.00001 + } + } + } + use_moving_average: false + } + save_checkpoints_steps: 4000 + sync_replicas: false + num_steps: 100 +} + +eval_config { + + metrics_set: { + recall_at_topk { + topk: 50 + } + } + metrics_set: { + recall_at_topk { + topk: 10 + } + } + metrics_set: { + recall_at_topk { + topk: 5 + } + } + metrics_set: { + recall_at_topk { + topk: 1 + } + } +} + +data_config { + input_fields { + input_name:'clk' + input_type: INT32 + } + input_fields { + input_name:'buy' + input_type: INT32 + } + input_fields { + input_name: 'pid' + input_type: STRING + } + input_fields { + input_name: 'adgroup_id' + input_type: STRING + } + input_fields { + input_name: 'cate_id' + input_type: STRING + } + input_fields { + input_name: 'campaign_id' + input_type: STRING + } + input_fields { + input_name: 'customer' + input_type: STRING + } + input_fields { + input_name: 'brand' + input_type: STRING + } + input_fields { + input_name: 'user_id' + input_type: STRING + } + input_fields { + input_name: 'cms_segid' + input_type: STRING + } + input_fields { + input_name: 'cms_group_id' + input_type: STRING + } + input_fields { + input_name: 'final_gender_code' + input_type: STRING + } + input_fields { + input_name: 'age_level' + input_type: STRING + } + input_fields { + input_name: 'pvalue_level' + input_type: STRING + } + input_fields { + input_name: 'shopping_level' + input_type: STRING + } + input_fields { + input_name: 'occupation' + input_type: STRING + } + input_fields { + input_name: 'new_user_class_level' + input_type: STRING + } + input_fields { + input_name: 'tag_category_list' + input_type: STRING + } + input_fields { + input_name: 'tag_brand_list' + input_type: STRING + } + input_fields { + input_name: 'price' + input_type: INT32 + } + + label_fields: 'clk' + batch_size: 4096 + num_epochs: 10000 + prefetch_size: 32 + input_type: CSVInput + + negative_sampler { + input_path: 'data/test/tb_data/taobao_ad_feature_gl' + num_sample: 1024 + num_eval_sample: 2048 + attr_fields: 'adgroup_id' + attr_fields: 'cate_id' + attr_fields: 'campaign_id' + attr_fields: 'customer' + attr_fields: 'brand' + item_id_field: 'adgroup_id' + } +} + +feature_config: { + features: { + input_names: 'pid' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'adgroup_id' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features: { + input_names: 'cate_id' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10000 + } + features: { + input_names: 'campaign_id' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features: { + input_names: 'customer' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features: { + input_names: 'brand' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features: { + input_names: 'user_id' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100000 + } + features: { + input_names: 'cms_segid' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100 + } + features: { + input_names: 'cms_group_id' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 100 + } + features: { + input_names: 'final_gender_code' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'age_level' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'pvalue_level' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'shopping_level' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'occupation' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'new_user_class_level' + feature_type: IdFeature + embedding_dim: 16 + hash_bucket_size: 10 + } + features: { + input_names: 'tag_category_list' + feature_type: TagFeature + separator: '|' + hash_bucket_size: 100000 + embedding_dim: 16 + } + features: { + input_names: 'tag_brand_list' + feature_type: TagFeature + separator: '|' + hash_bucket_size: 100000 + embedding_dim: 16 + } + features: { + input_names: 'price' + feature_type: IdFeature + embedding_dim: 16 + num_buckets: 50 + } +} +model_config:{ + model_class: "DSSM_SENet" + feature_groups: { + group_name: 'user' + feature_names: 'user_id' + feature_names: 'cms_segid' + feature_names: 'cms_group_id' + feature_names: 'age_level' + feature_names: 'pvalue_level' + feature_names: 'shopping_level' + feature_names: 'occupation' + feature_names: 'new_user_class_level' + feature_names: 'tag_category_list' + feature_names: 'tag_brand_list' + wide_deep:DEEP + } + feature_groups: { + group_name: "item" + feature_names: 'adgroup_id' + feature_names: 'cate_id' + feature_names: 'campaign_id' + feature_names: 'customer' + feature_names: 'brand' + #feature_names: 'price' + #feature_names: 'pid' + wide_deep:DEEP + } + dssm_senet { + user_tower { + id: "user_id" + senet { + num_squeeze_group : 2 + reduction_ratio: 4 + } + dnn { + hidden_units: [ 128, 32] + } + } + item_tower { + id: "adgroup_id" + senet { + num_squeeze_group : 2 + reduction_ratio: 4 + } + dnn { + hidden_units: [128, 32] + } + } + simi_func: COSINE + scale_simi: false + temperature: 0.01 + l2_regularization: 1e-6 + } + loss_type: SOFTMAX_CROSS_ENTROPY + embedding_regularization: 5e-5 +} + +export_config { +}