From 0a9ee0a6b70681e97a00bd96899bd6b017a32c7a Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:03:30 +0800 Subject: [PATCH 01/95] Add finished status when find exp is not alive but not labeled --- src/lumo/exp/watch.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index e2dc867..e1203fc 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -228,6 +228,12 @@ def progress(self, is_alive=True): exp = Experiment.from_disk(test_root) if exp.is_alive == is_alive: res.append(exp.dict()) + if not exp.is_alive and exp.properties['progress'].get('finished', None) is None: + exp.dump_info('progress', + { + 'end': strftime(), 'finished': True, 'end_code': -1, + 'msg': 'ended by watcher'} + ) except: continue return pd.DataFrame(res) From 77166853e7a42f635ccad1b2574cbb8d690fe806 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:03:42 +0800 Subject: [PATCH 02/95] Metric --- src/lumo/exp/metric.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lumo/exp/metric.py b/src/lumo/exp/metric.py index abf5f61..641d7fa 100644 --- a/src/lumo/exp/metric.py +++ b/src/lumo/exp/metric.py @@ -45,14 +45,16 @@ def dump_metric(self, key, value, cmp: str, flush=True, **kwargs): dic[key] = value for kk, vv in kwargs.items(): dic[kk] = vv + else: + value = older if flush: self.flush() return value def dump_metrics(self, dic: dict, cmp: str): - for k, v in dic.items(): - self.dump_metric(k, v, cmp) + return {k: self.dump_metric(k, v, cmp) + for k, v in dic.items()} def flush(self): """Writes the value of the row to a file.""" From 4213793526b917ffd6cd5c13df450df32343c6b5 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:03:59 +0800 Subject: [PATCH 03/95] End status content --- src/lumo/exp/experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo/exp/experiment.py b/src/lumo/exp/experiment.py index 358e9c3..d9f5392 100644 --- a/src/lumo/exp/experiment.py +++ b/src/lumo/exp/experiment.py @@ -679,7 +679,7 @@ def end(self, end_code=0, *args, **extra): if end_code == 0: self.dump_progress(1) - self.dump_info('progress', {'end': strftime(), 'finished': end_code == 0}, append=True) + self.dump_info('progress', {'end': strftime(), 'finished': True, 'end_code': end_code}, append=True) for hook in self._hooks.values(): # type: BaseExpHook hook.on_end(self, end_code=end_code, *args, **extra) return self From c516ee7dfda0be278e57fe961e55f6d52d383041 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:04:05 +0800 Subject: [PATCH 04/95] Add test for metric --- tests/exp/test_metric.py | 49 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tests/exp/test_metric.py diff --git a/tests/exp/test_metric.py b/tests/exp/test_metric.py new file mode 100644 index 0000000..ca42c54 --- /dev/null +++ b/tests/exp/test_metric.py @@ -0,0 +1,49 @@ +import os +import pytest +import tempfile + +from lumo.exp.metric import Metric + + +@pytest.fixture +def metric(): + with tempfile.TemporaryDirectory() as tmpdir: + metric_fn = os.path.join(tmpdir, "test_metric.pkl") + yield Metric(metric_fn) + + +def test_value(metric): + assert isinstance(metric.value, dict) + + +def test_dump_metric_max(metric): + key = "test_key_max" + values = [2, 4, 1, 7, 5] + expected_value = max(values) + cur = values[0] + for value in values: + max_val = metric.dump_metric(key, value, "max") + cur = max(value, cur) + assert max_val == cur + + assert metric.value[key] == expected_value + + +def test_dump_metric(metric): + key = "test_key" + value = 10 + metric.dump_metric(key, value, "max") + assert metric.value[key] == value + + +def test_dump_metrics(metric): + dic = {"key1": 1, "key2": 2, "key3": 3} + cmp = "min" + result = metric.dump_metrics(dic, cmp) + assert result == {"key1": 1, "key2": 2, "key3": 3} + + +def test_flush(metric): + metric.flush() + assert os.path.exists(metric.fn) + os.remove(metric.fn) From cc14d85e3cf748d58e8bcb55430e81684211d0a8 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:28:22 +0800 Subject: [PATCH 05/95] Add deprecated warning --- src/lumo/analyse/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lumo/analyse/__init__.py b/src/lumo/analyse/__init__.py index 7eeb38a..0af59d3 100644 --- a/src/lumo/analyse/__init__.py +++ b/src/lumo/analyse/__init__.py @@ -1,2 +1,5 @@ +import warnings + from .collect import collect_table_rows, flatten_dict, flatten_params, flatten_metric from .condition import C, filter_by_condition +warnings.warn("lumo.analyse has been deprecated and will be removed soon, please use lumo.exp.Watcher instead.") \ No newline at end of file From 3ddc382a5eda4139c4e22d760574e18029a06180 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:28:34 +0800 Subject: [PATCH 06/95] Add notes, fix no return problem --- src/lumo/exp/experiment.py | 46 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/src/lumo/exp/experiment.py b/src/lumo/exp/experiment.py index d9f5392..f54b1ce 100644 --- a/src/lumo/exp/experiment.py +++ b/src/lumo/exp/experiment.py @@ -367,12 +367,24 @@ def exec_argv(self): return [] def _trigger_change(self, func): - # test_root update some files + """ + Decorator function that updates the heartbeat file before executing the decorated function. + + The heartbeat file indicates that a change has occurred in the experiment directory. + + Args: + func: the function to be decorated. + + Returns: + A decorated function. + """ + + # test_root update some files @wraps(func) def inner(*args, **kwargs): fn = self.heartbeat_fn io.dump_text(self.info_dir, fn) - func(*args, **kwargs) + return func(*args, **kwargs) return inner @@ -479,15 +491,39 @@ def load_info(self, key: str): return {} def load_note(self): + """ + Loads the contents of the note file, if it exists. + + Returns: + A string representing the contents of the note file, or an empty string if the file does not exist. + """ fn = self.mk_ipath('note.md') if os.path.exists(fn): return io.load_text(fn) return '' def dump_tags(self, *tags): + """ + Dumps the experiment's tags to the info file. + + Args: + *tags: a variable-length argument list of tags to be added to the experiment. + + Returns: + None. + """ self.dump_info('tags', tags) def dump_note(self, note: str): + """ + Dumps the contents of the note to the note file. + + Args: + note: a string representing the contents of the note. + + Returns: + None. + """ fn = self.mk_ipath('note.md') self.set_prop('note', note) io.dump_text(note, fn) @@ -521,9 +557,15 @@ def load_string(self, key: str): return io.load_text(fn) def dump_metric(self, key, value, cmp: str, flush=True, **kwargs): + """ + See Metric for details. + """ return self.metric.dump_metric(key, value, cmp, flush, **kwargs) def dump_metrics(self, dic: dict, cmp: str): + """ + See Metric for details. + """ return self.metric.dump_metrics(dic, cmp) @property From 91f201162f135304868051d53d08aec57f98abd4 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:28:40 +0800 Subject: [PATCH 07/95] Add docstring --- src/lumo/exp/metric.py | 65 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/src/lumo/exp/metric.py b/src/lumo/exp/metric.py index 641d7fa..e7e9fdd 100644 --- a/src/lumo/exp/metric.py +++ b/src/lumo/exp/metric.py @@ -4,9 +4,26 @@ class Metric: """ + A class that handles metric values and saving/loading them to/from disk. + + Attributes: + fn (str): The file path of the metric file. + _metric (dict): A dictionary containing the metric values. + persistent (bool): A boolean value indicating whether to save the metric values to disk. """ def __init__(self, metric_fn, persistent=True): + """ + Initializes a new instance of the Metric class. + + Args: + metric_fn (str): The file path of the metric file. + persistent (bool): A boolean value indicating whether to save the metric values to disk. + Default is True. + + Returns: + None. + """ os.makedirs(os.path.dirname(os.path.abspath(metric_fn)), exist_ok=True) self.fn = metric_fn self._metric = {} @@ -17,14 +34,35 @@ def __init__(self, metric_fn, persistent=True): @property def value(self): """ - A property that returns the metric values of the row. + A property that returns the metric values. Returns: - dict: A dictionary containing the metric values of the row. + dict: A dictionary containing the metric values. """ return self._metric def dump_metric(self, key, value, cmp: str, flush=True, **kwargs): + """ + Updates the metric value for a given key. + + If the metric value for the given key is not set or the new value is better than the + existing value based on the comparison type specified by cmp, the metric value is updated + with the new value. The function returns the updated value. + + Args: + key (str): The key for the metric value. + value (float): The new metric value. + cmp (str): The type of comparison to use when updating the metric value. Must be 'max' or 'min'. + flush (bool): A boolean value indicating whether to save the updated metric values to disk. + Default is True. + **kwargs: Additional key-value pairs to store with the metric value. + + Returns: + float: The updated metric value. + + Raises: + NotImplementedError: If cmp is not 'max' or 'min'. + """ dic = self.value older = dic.setdefault(key, None) @@ -53,10 +91,27 @@ def dump_metric(self, key, value, cmp: str, flush=True, **kwargs): return value def dump_metrics(self, dic: dict, cmp: str): - return {k: self.dump_metric(k, v, cmp) - for k, v in dic.items()} + """ + Updates multiple metric values with a dictionary. + + The function calls dump_metric for each key-value pair in the input dictionary and returns + a dictionary containing the updated metric values. + + Args: + dic (dict): A dictionary containing the key-value pairs to update. + cmp (str): The type of comparison to use when updating the metric values. Must be 'max' or 'min'. + + Returns: + dict: A dictionary containing the updated metric values. + """ + res = {k: self.dump_metric(k, v, cmp, flush=False) + for k, v in dic.items()} + self.flush() + return res def flush(self): - """Writes the value of the row to a file.""" + """ + Writes the metric values to a file. + """ if self.persistent: IO.dump_pkl(self.value, self.fn) From e0357bb1797a06585946250bd582d3108675d664 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:28:44 +0800 Subject: [PATCH 08/95] Add docstring --- src/lumo/proc/path.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lumo/proc/path.py b/src/lumo/proc/path.py index 8cff01b..55547ba 100644 --- a/src/lumo/proc/path.py +++ b/src/lumo/proc/path.py @@ -112,6 +112,7 @@ def metricroot(): def dbroot(): + """Root path to store experiment information. Default is `~/.lumo/database`""" DB_ROOT = glob.get('db_root', None) if DB_ROOT: res = DB_ROOT From 20d3653efc4d018499fa4bb986205c9f747d7ee8 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:28:50 +0800 Subject: [PATCH 09/95] Add docstring --- src/lumo/trainer/trainer.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/lumo/trainer/trainer.py b/src/lumo/trainer/trainer.py index 27338f9..93b700d 100644 --- a/src/lumo/trainer/trainer.py +++ b/src/lumo/trainer/trainer.py @@ -1012,28 +1012,63 @@ def wait_for_everyone(self): self.accelerate.wait_for_everyone() def save_best_model(self): + """ + Saves the best model checkpoint and metadata. + + If the current process is the main process, saves the best model checkpoint as 'best_model.ckpt' + and its metadata as 'best_model.json'. If not, saves the checkpoint and metadata with the process rank + appended to the filename, e.g., 'best_model-.ckpt' and 'best_model-.json'. + + The saved metadata includes the global training steps and the value of the experiment's best metric. + + Args: + self: the Experiment object. + + Returns: + None. + """ if self.is_main: file = self.exp.mk_bpath('models', 'best_model.ckpt') file_info = self.exp.mk_bpath('models', 'best_model.json') else: file = self.exp.mk_bpath('models', f'best_model-{self.local_rank}.ckpt') file_info = self.exp.mk_bpath('models', f'best_model-{self.local_rank}.json') + torch.save(self.state_dict(), file) with open(file_info, 'w') as w: w.write(json.dumps({'global_steps': self.global_steps, 'metric': self.exp.metric.value})) + self.logger.info(f'saved best model at {file}') self.wait_for_everyone() def save_last_model(self): + """ + Saves the last model checkpoint and metadata. + + If the current process is the main process, saves the last model checkpoint as 'last_model.ckpt' + and its metadata as 'last_model.json'. If not, saves the checkpoint and metadata with the process rank + appended to the filename, e.g., 'last_model-.ckpt' and 'last_model-.json'. + + The saved metadata includes the global training steps and the value of the experiment's best metric. + + Args: + self: the Experiment object. + + Returns: + None. + """ if self.is_main: file = self.exp.mk_bpath('models', 'last_model.ckpt') file_info = self.exp.mk_bpath('models', 'last_model.json') else: file = self.exp.mk_bpath('models', f'last_model-{self.local_rank}.ckpt') file_info = self.exp.mk_bpath('models', f'last_model-{self.local_rank}.json') + torch.save(self.state_dict(), file) + with open(file_info, 'w') as w: w.write(json.dumps({'global_steps': self.global_steps, 'metric': self.exp.metric.value})) + self.logger.info(f'saved last model at {file}') self.wait_for_everyone() From 5224a5b421c044ded72409d5c46b81515ded4a48 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:28:55 +0800 Subject: [PATCH 10/95] Add docstring --- src/lumo/utils/subprocess.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/lumo/utils/subprocess.py b/src/lumo/utils/subprocess.py index 5ed5e9f..f1e073d 100644 --- a/src/lumo/utils/subprocess.py +++ b/src/lumo/utils/subprocess.py @@ -5,6 +5,17 @@ def run_command(command, cwd=None, env=None): + """ + Executes a command in the shell and captures its standard output and standard error. + + Args: + command: a string representing the command to execute in the shell. + cwd: a string representing the working directory to execute the command in. Default is None. + env: a dictionary representing the environment variables to set for the command. Default is None. + + Returns: + The return code of the executed command. + """ proc = subprocess.Popen(command, cwd=cwd, env=env, From 05123edd7d0dd4f92ceeda0f9eefd90fa2634087 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:29:48 +0800 Subject: [PATCH 11/95] Add docstring --- src/lumo/utils/fmt.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/lumo/utils/fmt.py b/src/lumo/utils/fmt.py index 7ba7897..ef6c537 100644 --- a/src/lumo/utils/fmt.py +++ b/src/lumo/utils/fmt.py @@ -72,7 +72,15 @@ def indent_print(text, indent=' '): def format_second(sec: int) -> str: - """Formats a duration given in seconds into a human-readable string.""" + """ + Formats a duration given in seconds into a human-readable string. + + Args: + sec: the duration in seconds. + + Returns: + A human-readable string representing the duration. + """ sec, ms = divmod(sec, 1) if sec > 60: min, sec = divmod(sec, 60) @@ -90,5 +98,14 @@ def format_second(sec: int) -> str: return fmt -def format_timedelta(td: timedelta): +def format_timedelta(td: timedelta) -> str: + """ + Formats a timedelta object into a human-readable string. + + Args: + td: a timedelta object. + + Returns: + A human-readable string representing the timedelta. + """ return format_second(td.total_seconds()) From 92058189089d3da321ab2203f6ad49a0bfd3fff2 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:37:28 +0800 Subject: [PATCH 12/95] Add docstring --- src/lumo/exp/experiment.py | 75 +++++++++++++- src/lumo/exp/watch.py | 200 ++++++++++++------------------------- 2 files changed, 136 insertions(+), 139 deletions(-) diff --git a/src/lumo/exp/experiment.py b/src/lumo/exp/experiment.py index f54b1ce..8d51146 100644 --- a/src/lumo/exp/experiment.py +++ b/src/lumo/exp/experiment.py @@ -382,6 +382,7 @@ def _trigger_change(self, func): # test_root update some files @wraps(func) def inner(*args, **kwargs): + """wrap function""" fn = self.heartbeat_fn io.dump_text(self.info_dir, fn) return func(*args, **kwargs) @@ -616,7 +617,17 @@ def blob_dir(self): os.makedirs(d, exist_ok=True) return d - def _mk_path(self, *path: str, is_dir) -> str: + def _mk_path(self, *path: str, is_dir: bool) -> str: + """ + Helper method to create a directory path if it does not exist and return the path. + + Args: + *path: tuple of path strings to be joined. + is_dir: boolean flag indicating whether the path is a directory. + + Returns: + str: the full path created. + """ path = os.path.join(*path) if is_dir: os.makedirs(path, exist_ok=True) @@ -624,16 +635,56 @@ def _mk_path(self, *path: str, is_dir) -> str: os.makedirs(os.path.dirname(path), exist_ok=True) return path - def mk_ipath(self, *path, is_dir=False): + def mk_ipath(self, *path: str, is_dir: bool = False) -> str: + """ + Creates a directory path within the experiment's info directory. + + Args: + *path: tuple of path strings to be joined. + is_dir: boolean flag indicating whether the path is a directory. Default is False. + + Returns: + str: the full path created. + """ return self._mk_path(self.info_dir, *path, is_dir=is_dir) - def mk_cpath(self, *path, is_dir=False): + def mk_cpath(self, *path: str, is_dir: bool = False) -> str: + """ + Creates a directory path within the experiment's cache directory. + + Args: + *path: tuple of path strings to be joined. + is_dir: boolean flag indicating whether the path is a directory. Default is False. + + Returns: + str: the full path created. + """ return self._mk_path(self.cache_dir, *path, is_dir=is_dir) - def mk_bpath(self, *path, is_dir=False): + def mk_bpath(self, *path: str, is_dir: bool = False) -> str: + """ + Creates a directory path within the experiment's blob directory. + + Args: + *path: tuple of path strings to be joined. + is_dir: boolean flag indicating whether the path is a directory. Default is False. + + Returns: + str: the full path created. + """ return self._mk_path(self.blob_dir, *path, is_dir=is_dir) - def mk_rpath(self, *path, is_dir=False): + def mk_rpath(self, *path: str, is_dir: bool = False) -> str: + """ + Creates a directory path within the user's home directory. + + Args: + *path: tuple of path strings to be joined. + is_dir: boolean flag indicating whether the path is a directory. Default is False. + + Returns: + str: the full path created. + """ return self._mk_path(libhome(), *path, is_dir=is_dir) @classmethod @@ -765,6 +816,18 @@ def exp_func(): @classmethod def from_cache(cls, dic: dict): + """ + Creates an Experiment object from a cached dictionary. + + The cached dictionary should have the same format as the one returned by the Experiment.to_cache() method. + + Args: + cls: the Experiment class. + dic: a dictionary with the cached Experiment data. + + Returns: + An Experiment object. + """ paths = dic.pop('paths', {}) _ = dic.pop('metrics') self = cls(exp_name=dic['exp_name'], test_name=dic['test_name'], paths=paths) @@ -816,12 +879,14 @@ def from_disk(cls, path): return self def cache(self): + """Cache information of current test.""" return { **self.properties, 'metrics': self.metric.value, } def dict(self): + """Get full information of current test, including dynamic status.""" return { **self.properties, 'is_alive': self.is_alive, diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index e1203fc..f7628cc 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -74,6 +74,8 @@ def not_in_(ser, value): class Condition: + """Represents a condition to filter data based on a certain criteria.""" + def __init__(self, name: str = None, value=None, op=None): self.name = name self.value = value @@ -130,18 +132,43 @@ def __repr__(self): return f'C({self.name} {self.op} {self.value})' def in_(self, lis): - """condition of `in` operation""" + """ + Sets the condition to evaluate if the value is in a given list. + + Args: + lis (list): the list of values to compare against. + + Returns: + The current instance of the Condition class with the comparison operator and value set. + """ self.op = 'in' self.value = set(lis) return self def not_in_(self, lis): - """condition of `.duplicated(value) == False` operation""" + """ + Sets the condition to evaluate if the value is not in a given list. + + Args: + lis (list): the list of values to compare against. + + Returns: + The current instance of the Condition class with the comparison operator and value set. + """ self.op = 'notin' self.value = set(lis) return self def mask(self, df): + """ + Returns a boolean mask of the given DataFrame based on the condition. + + Args: + df (pd.DataFrame): the DataFrame to evaluate. + + Returns: + A boolean mask of the given DataFrame based on the condition. + """ names = self.name.split('.') value = df for i in names: @@ -152,6 +179,7 @@ def mask(self, df): return mapping[self.op](value, self.value) def apply(self, df): + """Returns a new DataFrame with only the rows that meet the condition.""" return df[self.mask(df)] @@ -159,10 +187,14 @@ def apply(self, df): class Watcher: - """List and watch experiments with time order + """ + A class for listing and watching experiments with time order and caching test information in 'metrics/.sqlite'. - Cache test_information in - metrics/.sqlite + Attributes: + exp_root (str): The root directory to search for experiments. + hb_root (str): The root directory to search for heartbeat files. + pid_root (str): The root directory to search for PID files. + db_root (str): The root directory to store the experiment databases. """ def __init__(self, exp_root=None, hb_root=None, pid_root=None, db_root=None): @@ -183,6 +215,12 @@ def __init__(self, exp_root=None, hb_root=None, pid_root=None, db_root=None): self.pid_root = pid_root def load(self): + """ + Loads the experiment information from heartbeat files and the experiment databases. + + Returns: + A pandas DataFrame containing the experiment information sorted by experiment name and test name. + """ res = {} updates = {} if not os.path.exists(self.hb_root): @@ -217,7 +255,15 @@ def load(self): return df.reset_index(drop=True) def progress(self, is_alive=True): - """return the alive process""" + """ + Returns a DataFrame of alive experiments. + + Args: + is_alive (bool): A boolean flag indicating whether to return only alive experiments. + + Returns: + A pandas DataFrame containing the experiment information of alive experiments. + """ res = [] for root, dirs, fs in os.walk(self.pid_root): for f in fs: @@ -264,6 +310,7 @@ def widget(self, params_filter: list = None, metric_filter: list = None ): + """Create a user interface in jupyter with ipywidget""" assert params_filter is None or isinstance(params_filter, list) assert metric_filter is None or isinstance(metric_filter, list) @@ -271,12 +318,23 @@ def widget(self, from IPython.display import display def make_row(dic: dict): + """ + Helper function for creating a row in the widgets grid. + + Args: + dic (dict): The dictionary containing the experiment information. + + Returns: + A list of ipywidgets objects for the row. + """ exp = Experiment.from_cache(dic.copy()) def on_note_update(sender): + """when note textarea update""" exp.dump_note(sender['new']) def on_tag_update(sender): + """when tag component update""" exp.dump_tags(*sender['new']) note_ui = widgets.Textarea(dic['note']) @@ -319,6 +377,7 @@ def on_tag_update(sender): end_filter = widgets.DatetimePicker() def status_filter(sender): + """when filter condition changed""" print(sender) make() @@ -332,6 +391,7 @@ def make( start=widgets.DatetimePicker(), end=widgets.DatetimePicker(), ): + """make widgets with filter condition.""" if status == 'running': df = self.progress() elif status == 'finished': @@ -412,131 +472,3 @@ def make( grid, ) - - # return display( - # widgets.HTML(styles['row-radio']), - # widgets.HTML(""" - # - # """), - # grid, clear=True) - - -class ExperimentWidget: - @overload - def __init__(self, exp_name, test_name, - progress: dict, - params: dict, metrics: dict, note: str, tags: set, exp: Experiment): - pass - - def __init__(self, **kwargs): - from ipywidgets import widgets - self.wid = widgets - self.exp = kwargs.pop('exp') # type: Experiment - self._prop = kwargs - - self._widgets = { - 'exp_name': widgets.HTML(self._prop['exp_name']), - 'test_name': widgets.HTML(self._prop['test_name']), - 'metrics': widgets.VBox( - [widgets.HTML(f'{k}: {v}') for k, v in self._prop['metrics'].items() if - isinstance(v, numbers.Number)]), - } - - self._params_widgets = {} - - note_ui = widgets.Textarea(self._prop['note']) - - note_ui.continuous_update = False - note_ui.observe(self.on_note_update, names='value', type='change') - self._widgets['note'] = note_ui - - tag_ui = widgets.TagsInput(value=list(self._prop['tags'])) - self._widgets['tags'] = tag_ui - tag_ui.observe(self.on_tag_update, names='value', type='change') - - def on_note_update(self, sender): - self.exp.dump_note(sender['new']) - - def on_tag_update(self, sender): - self.exp.dump_tags(*sender['new']) - - def set_key_params(self, keys: list): - self._params_widgets.clear() - for key in keys: - self._params_widgets[key] = self.wid.HTML( - f"""{key}: {pformat(self._prop['params'][key], width=10, indent=2, compact=True)}""") - - def sep(self): - return self.wid.Output(layout={'border': '1px solid black'}) - - def id_flag(self): - return self.wid.VBox([ - self._widgets['exp_name'], - self._widgets['test_name'], - ]) - - def key_params(self): - return self.wid.VBox([ - *self._params_widgets.values() - ]) - - def editable(self): - return self.wid.VBox([ - self._widgets['note'], - self.sep(), - self._widgets['tags'], - ]) - - def time(self): - now = datetime.now() - start = strptime(datestr=self._prop['progress']['start']) - end = strptime(datestr=self._prop['progress']['start']) - return self.wid.VBox([ - self.wid.HTML(f"""Start at: {format_timedelta(now - start)}"""), - self.wid.HTML(f"""End at: {format_timedelta(now - end)}"""), - ]) - - def widget_dict(self): - return { - 'id_flag': self.id_flag(), - 'time': self.time(), - 'editable': self.editable(), - 'params': self.key_params(), - } - - def widget(self): - params = self.key_params() - params = [ - self.sep(), - params, - ] - - hbox = self.wid.HBox([ - self.id_flag(), - self.time(), - self._widgets['metrics'], - self.editable(), - self.key_params(), - ]) - - return hbox - - @classmethod - def from_experiment(cls, exp: Experiment): - tags = exp.properties.get('tags', []) - try: - tags = set(tags) - except: - tags = set() - return cls( - exp_name=exp.exp_name, - test_name=exp.test_name, - progress=exp.properties.get('progress', {}), - params=exp['params'], - metrics=exp.metric.value, - note=exp.properties.get('note', ''), - tags=tags, - exp=exp, - ) From 33bb865c92e678d066c3e37adf46a6c649ef180e Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:48:37 +0800 Subject: [PATCH 13/95] Deprecated some code --- src/lumo/exp/experiment.py | 4 +-- src/lumo/exp/finder.py | 29 +-------------- src/lumo/exp/watch.py | 68 +++++++++++++++++++++++++++++------ src/lumo/sketch/vis/parser.py | 38 ++++++++++---------- 4 files changed, 80 insertions(+), 59 deletions(-) diff --git a/src/lumo/exp/experiment.py b/src/lumo/exp/experiment.py index 8d51146..f2b7999 100644 --- a/src/lumo/exp/experiment.py +++ b/src/lumo/exp/experiment.py @@ -123,7 +123,6 @@ def __init__(self, exp_name: str, test_name=None, paths=None): self.dump_note = self._trigger_change(self.dump_note) self.dump_info = self._trigger_change(self.dump_info) - self.add_exit_hook(self.end) self.logger = Logger() def __getitem__(self, item): @@ -737,6 +736,7 @@ def initial(self): self.dump_progress(0) # register progress io.dump_text(self.info_dir, self.pid_fn) + self.add_exit_hook(self.end) @call_on_main_process_wrap def start(self): @@ -848,7 +848,7 @@ def from_disk(cls, path): Raises: ValueError: If the path is not a valid test root directory. """ - from .finder import is_test_root + from lumo.exp.watch import is_test_root if not is_test_root(path): raise ValueError(f'{path} is not a valid test_root') path = os.path.abspath(path) diff --git a/src/lumo/exp/finder.py b/src/lumo/exp/finder.py index 5398917..dec377f 100644 --- a/src/lumo/exp/finder.py +++ b/src/lumo/exp/finder.py @@ -11,9 +11,9 @@ import os from typing import List, Dict, Any +from lumo.exp.watch import is_test_root, is_test_name from lumo.proc.path import libhome, exproot, metricroot from lumo.utils.fmt import indent_print -from lumo.utils import re from . import Experiment @@ -112,33 +112,6 @@ def find_path_from_test_name(test_name: str) -> str: return None -def is_test_name(test_name: str) -> bool: - """ - Determines if the specified string is a valid test name. - - Args: - test_name: The string to check. - - Returns: - True if the string is a valid test name, False otherwise. - """ - return re.search(r'^\d{6}\.\d{3}\.[a-z\d]{2}t$', test_name) is not None - - -def is_test_root(path: str) -> bool: - """ - Determines if the specified path is a valid test root. - - Args: - path: The path to check. - - Returns: - True if the path is a valid test root, False otherwise. - """ - test_name = os.path.basename(path.rstrip('/')) - return is_test_name(test_name) - - def retrieval_test_root(test_flag: str) -> str: """ Returns the test root directory for the specified test name or test root. diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index f7628cc..06fa476 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -19,7 +19,9 @@ """ import numbers +import os import os.path +import re from typing import List, Dict, overload from pprint import pformat import pandas as pd @@ -214,14 +216,11 @@ def __init__(self, exp_root=None, hb_root=None, pid_root=None, db_root=None): self.hb_root = hb_root self.pid_root = pid_root - def load(self): - """ - Loads the experiment information from heartbeat files and the experiment databases. + def retrieve(self, test_name=None): + raise NotImplementedError() - Returns: - A pandas DataFrame containing the experiment information sorted by experiment name and test name. - """ - res = {} + def update(self): + """Diff & Update""" updates = {} if not os.path.exists(self.hb_root): return pd.DataFrame() @@ -242,13 +241,35 @@ def load(self): for exp_name, tests in updates.items(): dic = PDict(os.path.join(self.db_root, f'{exp_name}.sqlite')) - for test_name, test_prop in dic.items(): - res[test_name] = test_prop for test in tests: dic[test['test_name']] = test - res[test['test_name']] = test dic.flush() + return updates + + def load(self): + """ + Loads the experiment information from heartbeat files and the experiment databases. + + Returns: + A pandas DataFrame containing the experiment information sorted by experiment name and test name. + """ + res = {} + updates = self.update() + + for dic_fn in os.listdir(self.db_root): + if not dic_fn.endswith('sqlite'): + continue + dic = PDict(os.path.join(self.db_root, dic_fn)) + exp_name = os.path.splitext(dic_fn)[0] + + for test_name, test_prop in dic.items(): + res[test_name] = test_prop + + if exp_name in updates: + for test in updates[exp_name]: + res[test['test_name']] = test + dic.flush() df = pd.DataFrame(res.values()) df = df.sort_values(['exp_name', 'test_name']) @@ -472,3 +493,30 @@ def make( grid, ) + + +def is_test_root(path: str) -> bool: + """ + Determines if the specified path is a valid test root. + + Args: + path: The path to check. + + Returns: + True if the path is a valid test root, False otherwise. + """ + test_name = os.path.basename(path.rstrip('/')) + return is_test_name(test_name) + + +def is_test_name(test_name: str) -> bool: + """ + Determines if the specified string is a valid test name. + + Args: + test_name: The string to check. + + Returns: + True if the string is a valid test name, False otherwise. + """ + return re.search(r'^\d{6}\.\d{3}\.[a-z\d]{2}t$', test_name) is not None diff --git a/src/lumo/sketch/vis/parser.py b/src/lumo/sketch/vis/parser.py index b1f9406..7d47aea 100644 --- a/src/lumo/sketch/vis/parser.py +++ b/src/lumo/sketch/vis/parser.py @@ -23,25 +23,25 @@ class Step: step: int -def find_metric_fron_test_root(test_root): - test_root = finder.retrieval_test_root(test_root) - if test_root is None: - return False, {} - - exp = Experiment.from_disk(test_root) - if exp.has_prop('tensorboard_args'): - tb = exp.properties.get('tensorboard_args') - metrics = parse_fron_tensorboard(tb['log_dir']) - elif exp.has_prop('logger_args'): - tb = exp.properties.get('logger_args') - metrics = parse_from_log(tb['log_dir']) - else: - fs = [i for i in os.listdir(exp.test_root)] - if len([f for f in fs if f.endswith('.log')]) > 0: - metrics = parse_from_log(os.path.join(exp.test_root, fs[0])) - else: - metrics = {} - return True, metrics +# def find_metric_fron_test_root(test_root): +# test_root = finder.retrieval_test_root(test_root) +# if test_root is None: +# return False, {} +# +# exp = Experiment.from_disk(test_root) +# if exp.has_prop('tensorboard_args'): +# tb = exp.properties.get('tensorboard_args') +# metrics = parse_fron_tensorboard(tb['log_dir']) +# elif exp.has_prop('logger_args'): +# tb = exp.properties.get('logger_args') +# metrics = parse_from_log(tb['log_dir']) +# else: +# fs = [i for i in os.listdir(exp.test_root)] +# if len([f for f in fs if f.endswith('.log')]) > 0: +# metrics = parse_from_log(os.path.join(exp.test_root, fs[0])) +# else: +# metrics = {} +# return True, metrics def parse_from_log(log) -> Dict[str, List[Step]]: From e6302268b5b315a69b125510ec7e111d64a8a631 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 14:48:46 +0800 Subject: [PATCH 14/95] Reference update --- src/lumo/trainer/callbacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo/trainer/callbacks.py b/src/lumo/trainer/callbacks.py index 2f71b22..1018254 100644 --- a/src/lumo/trainer/callbacks.py +++ b/src/lumo/trainer/callbacks.py @@ -805,7 +805,7 @@ class SkipWhenParamsEq(TrainCallback, InitialCallback): def on_hooked(self, source: Trainer, params: ParamsType): super().on_hooked(source, params) from dbrecord import PDict - from lumo.exp.finder import is_test_root + from lumo.exp.watch import is_test_root self.fn = source.exp.mk_rpath('contrib', 'params_key.sqlite') olds = PDict(self.fn) From c6f5fcefad5b44ef4e17261db7c1eb9d04b5ffc6 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 18:15:27 +0800 Subject: [PATCH 15/95] optional for load DataFrame, lazy load pandas module --- src/lumo/exp/watch.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index 06fa476..7ca031b 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -24,7 +24,7 @@ import re from typing import List, Dict, overload from pprint import pformat -import pandas as pd + from dbrecord import PDict from datetime import datetime from operator import gt, ge, le, lt, eq, ne @@ -247,7 +247,7 @@ def update(self): dic.flush() return updates - def load(self): + def load(self, with_pandas=True): """ Loads the experiment information from heartbeat files and the experiment databases. @@ -270,10 +270,13 @@ def load(self): for test in updates[exp_name]: res[test['test_name']] = test dic.flush() - - df = pd.DataFrame(res.values()) - df = df.sort_values(['exp_name', 'test_name']) - return df.reset_index(drop=True) + if with_pandas: + import pandas as pd + df = pd.DataFrame(res.values()) + df = df.sort_values(['exp_name', 'test_name']) + return df.reset_index(drop=True) + else: + return res def progress(self, is_alive=True): """ From b4d999f73b29c9ddc395e5db758115c9f0c8cff6 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 18:17:00 +0800 Subject: [PATCH 16/95] optional for load DataFrame, lazy load pandas module --- src/lumo/exp/watch.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index 7ca031b..0d161da 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -251,8 +251,14 @@ def load(self, with_pandas=True): """ Loads the experiment information from heartbeat files and the experiment databases. + Args: + with_pandas (bool, optional): whether to return the experiment information as a pandas DataFrame. + Defaults to True. + Returns: - A pandas DataFrame containing the experiment information sorted by experiment name and test name. + If with_pandas is True, returns a pandas DataFrame containing the experiment information + sorted by experiment name and test name. Otherwise, returns a dictionary containing the + experiment information. """ res = {} updates = self.update() @@ -271,7 +277,12 @@ def load(self, with_pandas=True): res[test['test_name']] = test dic.flush() if with_pandas: - import pandas as pd + try: + import pandas as pd + except ImportError as e: + print( + 'with_padnas=True requires pandas to be installed, use pip install pandas or call `.load(with_padnas=False)`') + df = pd.DataFrame(res.values()) df = df.sort_values(['exp_name', 'test_name']) return df.reset_index(drop=True) From 1102e8895910204aa329354dbdf0bacf7757f332 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 18:58:32 +0800 Subject: [PATCH 17/95] optional for load DataFrame, lazy load pandas module --- src/lumo/exp/watch.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index 0d161da..a7f2e82 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -263,6 +263,9 @@ def load(self, with_pandas=True): res = {} updates = self.update() + def valid_row(dic): + return isinstance(dic, dict) and 'test_name' in dic + for dic_fn in os.listdir(self.db_root): if not dic_fn.endswith('sqlite'): continue @@ -270,12 +273,15 @@ def load(self, with_pandas=True): exp_name = os.path.splitext(dic_fn)[0] for test_name, test_prop in dic.items(): - res[test_name] = test_prop + if valid_row(test_prop): + res[test_name] = test_prop if exp_name in updates: for test in updates[exp_name]: - res[test['test_name']] = test + if valid_row(test): + res[test['test_name']] = test dic.flush() + if with_pandas: try: import pandas as pd From 512378d0c71b0831debd45fb555e77b11c316401 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 20:05:58 +0800 Subject: [PATCH 18/95] Add scripts for generate shell scripts for multiple tests --- src/lumo/contrib/scan.py | 72 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/lumo/contrib/scan.py diff --git a/src/lumo/contrib/scan.py b/src/lumo/contrib/scan.py new file mode 100644 index 0000000..b277b0f --- /dev/null +++ b/src/lumo/contrib/scan.py @@ -0,0 +1,72 @@ +""" +基线对比 +""" +import time +from itertools import cycle +from typing import List +import torch + +from lumo import Params, Logger +import sys + +DATE_FLAG = '2023.03.04' + + +class ScanBaseParams(Params): + + def __init__(self): + super().__init__() + self.gpus = None + self.group_code = None + self.interval = 5 # sleep interval between two tests + + +def format_args(**kwargs): + return ' '.join([f'--{k}={v}' for k, v in kwargs.items()]) + + +def base_main(pm: ScanBaseParams, files: List[str], dics: List[dict]): + assert isinstance(pm, ScanBaseParams) + log = Logger() + log.use_stdout = False + log.add_log_dir(f'./log_{pm.group_code}') + + base = ("sleep {sleep} ; " + + sys.executable + + " {file}.py {kwargs} --device={device}{group} & \n" + ) + + if not torch.cuda.is_available(): + gpus = ['cpu'] + elif pm.gpus is None: + gpus = list(range(torch.cuda.device_count())) + elif isinstance(pm.gpus, (int, str)): + gpus = [torch.device(pm.gpus).index] + else: + gpus = pm.gpus + + # append None to identity loop end. + gpus.append(None) + gpus = cycle(gpus) + c = 0 + for i, (file, kwargs) in enumerate(zip(files, dics)): + device = next(gpus) + if device is None: + # wait until all devices are free. + c = 0 + print('wait', flush=True) + device = next(gpus) + + if pm.group_code is not None: + group = f" --group={pm.group_code} " + else: + group = '' + + cur = base.format( + sleep=c * pm.interval, + file=file, kwargs=format_args(**kwargs), + device=device, group=group) + c += 1 + log.info(cur.strip()) + print(cur, flush=True, end='') + print('wait', flush=True) From 4df80806547ed6e4b48b47e4d7b44ad1afd860f9 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 20:07:08 +0800 Subject: [PATCH 19/95] Add scripts for generate shell scripts for multiple tests --- src/lumo/contrib/scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo/contrib/scan.py b/src/lumo/contrib/scan.py index b277b0f..b850109 100644 --- a/src/lumo/contrib/scan.py +++ b/src/lumo/contrib/scan.py @@ -17,7 +17,7 @@ class ScanBaseParams(Params): def __init__(self): super().__init__() self.gpus = None - self.group_code = None + self.group = None self.interval = 5 # sleep interval between two tests From 5b0a60cc74d05a50e3f9f5e38eb3bca422a44745 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 20:07:34 +0800 Subject: [PATCH 20/95] Add scripts for generate shell scripts for multiple tests --- src/lumo/contrib/scan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lumo/contrib/scan.py b/src/lumo/contrib/scan.py index b850109..247cb65 100644 --- a/src/lumo/contrib/scan.py +++ b/src/lumo/contrib/scan.py @@ -29,7 +29,7 @@ def base_main(pm: ScanBaseParams, files: List[str], dics: List[dict]): assert isinstance(pm, ScanBaseParams) log = Logger() log.use_stdout = False - log.add_log_dir(f'./log_{pm.group_code}') + log.add_log_dir(f'./log_{pm.group}') base = ("sleep {sleep} ; " + sys.executable + @@ -57,8 +57,8 @@ def base_main(pm: ScanBaseParams, files: List[str], dics: List[dict]): print('wait', flush=True) device = next(gpus) - if pm.group_code is not None: - group = f" --group={pm.group_code} " + if pm.group is not None: + group = f" --group={pm.group} " else: group = '' From 5f529571364d790ca4ea831ddf83285fdc1ffc97 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 20:28:43 +0800 Subject: [PATCH 21/95] Update --- src/lumo/exp/experiment.py | 2 +- src/lumo/exp/watch.py | 7 +++++-- src/lumo/trainer/trainer.py | 9 +++++---- src/lumo/utils/repository.py | 2 +- src/lumo/utils/safe_io.py | 16 ++++++++++++++++ 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/lumo/exp/experiment.py b/src/lumo/exp/experiment.py index f2b7999..8ec1646 100644 --- a/src/lumo/exp/experiment.py +++ b/src/lumo/exp/experiment.py @@ -449,9 +449,9 @@ def dump_progress(self, ratio: float, update_from=None): update_from: The process from which the progress update came from. """ res = {'ratio': max(min(ratio, 1), 0)} + res['last_edit_time'] = strftime() if update_from is None: res['update_from'] = update_from - res['last_edit_time'] = strftime() self.dump_info('progress', res, append=True) def dump_info(self, key: str, info: Any, append=False): diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index a7f2e82..44a4dc3 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -295,7 +295,7 @@ def valid_row(dic): else: return res - def progress(self, is_alive=True): + def progress(self, is_alive=True, with_pandas=True): """ Returns a DataFrame of alive experiments. @@ -323,7 +323,10 @@ def progress(self, is_alive=True): ) except: continue - return pd.DataFrame(res) + if with_pandas: + return pd.DataFrame(res) + else: + return res def interactive(self): """interactive, mark, label, note in ipython environment.""" diff --git a/src/lumo/trainer/trainer.py b/src/lumo/trainer/trainer.py index 93b700d..1024a04 100644 --- a/src/lumo/trainer/trainer.py +++ b/src/lumo/trainer/trainer.py @@ -22,6 +22,7 @@ from lumo.data.loader import DataLoaderType, DataLoaderSide from lumo.proc import dist from lumo.proc import glob +from lumo.utils import safe_io as IO from lumo.trainer.rnd import RndManager from lumo.utils.logger import Logger from .base import _BaseTrainer @@ -1036,8 +1037,8 @@ def save_best_model(self): torch.save(self.state_dict(), file) - with open(file_info, 'w') as w: - w.write(json.dumps({'global_steps': self.global_steps, 'metric': self.exp.metric.value})) + res = {'global_steps': self.global_steps, 'metric': self.exp.metric.value} + IO.dump_json(IO.filter_unserializable_values(res), file_info) self.logger.info(f'saved best model at {file}') self.wait_for_everyone() @@ -1067,8 +1068,8 @@ def save_last_model(self): torch.save(self.state_dict(), file) - with open(file_info, 'w') as w: - w.write(json.dumps({'global_steps': self.global_steps, 'metric': self.exp.metric.value})) + res = {'global_steps': self.global_steps, 'metric': self.exp.metric.value} + IO.dump_json(IO.filter_unserializable_values(res), file_info) self.logger.info(f'saved last model at {file}') self.wait_for_everyone() diff --git a/src/lumo/utils/repository.py b/src/lumo/utils/repository.py index 6466ab7..fce12f8 100644 --- a/src/lumo/utils/repository.py +++ b/src/lumo/utils/repository.py @@ -55,7 +55,7 @@ class branch: def __init__(self, repo: Repo, branch: str): self.repo = repo - self.lock = Lock(f'{hash(repo.git_dir)}_{branch}') + self.lock = Lock(f'{hash(repo.git_dir)}_lumo_auto_commit') self.lock.abtain() self.old_branch = self.repo.head.reference self.branch = branch diff --git a/src/lumo/utils/safe_io.py b/src/lumo/utils/safe_io.py index b87d588..2fbaf69 100644 --- a/src/lumo/utils/safe_io.py +++ b/src/lumo/utils/safe_io.py @@ -16,6 +16,22 @@ load_nd = load_nd +def filter_unserializable_values(self, d): + for key, value in list(d.items()): + if isinstance(value, dict): + filter_unserializable_values(value) + elif isinstance(value, list): + for i in range(len(value)): + if isinstance(value[i], dict): + filter_unserializable_values(value[i]) + elif not json.dumps(value[i], default=lambda x: None): + value[i] = None + elif not json.dumps(value, default=lambda x: None): + d[key] = None + d = {key: value for key, value in d.items() if value is not None} + return d + + def dump_json(obj, fn): """ Dumps the given object to a JSON file at the given file path. From ce772dba0673ae6e87d89cc0abe76957b9a539a8 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 20:31:32 +0800 Subject: [PATCH 22/95] Update --- src/lumo/utils/safe_io.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lumo/utils/safe_io.py b/src/lumo/utils/safe_io.py index 2fbaf69..51306d2 100644 --- a/src/lumo/utils/safe_io.py +++ b/src/lumo/utils/safe_io.py @@ -16,8 +16,8 @@ load_nd = load_nd -def filter_unserializable_values(self, d): - for key, value in list(d.items()): +def filter_unserializable_values(dic): + for key, value in list(dic.items()): if isinstance(value, dict): filter_unserializable_values(value) elif isinstance(value, list): @@ -27,9 +27,9 @@ def filter_unserializable_values(self, d): elif not json.dumps(value[i], default=lambda x: None): value[i] = None elif not json.dumps(value, default=lambda x: None): - d[key] = None - d = {key: value for key, value in d.items() if value is not None} - return d + dic[key] = None + dic = {key: value for key, value in dic.items() if value is not None} + return dic def dump_json(obj, fn): From 40c735c8b51052a9b531137404e1e5aeee5e6b3c Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 20:38:17 +0800 Subject: [PATCH 23/95] Update --- src/lumo/exp/metric.py | 12 +++++++++++- src/lumo/trainer/trainer.py | 2 +- src/lumo/utils/safe_io.py | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/lumo/exp/metric.py b/src/lumo/exp/metric.py index e7e9fdd..56cc574 100644 --- a/src/lumo/exp/metric.py +++ b/src/lumo/exp/metric.py @@ -27,10 +27,16 @@ def __init__(self, metric_fn, persistent=True): os.makedirs(os.path.dirname(os.path.abspath(metric_fn)), exist_ok=True) self.fn = metric_fn self._metric = {} + self._last = {} if os.path.exists(metric_fn): self._metric = IO.load_pkl(metric_fn) + self.persistent = persistent + @property + def current(self): + return self._last + @property def value(self): """ @@ -63,7 +69,7 @@ def dump_metric(self, key, value, cmp: str, flush=True, **kwargs): Raises: NotImplementedError: If cmp is not 'max' or 'min'. """ - dic = self.value + dic = self._metric older = dic.setdefault(key, None) update = False @@ -86,6 +92,10 @@ def dump_metric(self, key, value, cmp: str, flush=True, **kwargs): else: value = older + self._last[key] = value + for kk, vv in kwargs.items(): + self._last[kk] = vv + if flush: self.flush() return value diff --git a/src/lumo/trainer/trainer.py b/src/lumo/trainer/trainer.py index 1024a04..98ee368 100644 --- a/src/lumo/trainer/trainer.py +++ b/src/lumo/trainer/trainer.py @@ -1068,7 +1068,7 @@ def save_last_model(self): torch.save(self.state_dict(), file) - res = {'global_steps': self.global_steps, 'metric': self.exp.metric.value} + res = {'global_steps': self.global_steps, 'metric': self.exp.metric.current} IO.dump_json(IO.filter_unserializable_values(res), file_info) self.logger.info(f'saved last model at {file}') diff --git a/src/lumo/utils/safe_io.py b/src/lumo/utils/safe_io.py index 51306d2..434f189 100644 --- a/src/lumo/utils/safe_io.py +++ b/src/lumo/utils/safe_io.py @@ -26,7 +26,7 @@ def filter_unserializable_values(dic): filter_unserializable_values(value[i]) elif not json.dumps(value[i], default=lambda x: None): value[i] = None - elif not json.dumps(value, default=lambda x: None): + elif json.dumps(value, default=lambda x: None) == 'null': dic[key] = None dic = {key: value for key, value in dic.items() if value is not None} return dic From 58834764abb9ede4c93f18d8abe0bb1893330167 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 20:47:19 +0800 Subject: [PATCH 24/95] Add timezone --- src/lumo/exp/watch.py | 3 ++- src/lumo/proc/tz.py | 6 ++++++ src/lumo/trainer/callbacks.py | 4 ++-- src/lumo/utils/fmt.py | 3 ++- src/lumo/utils/logger.py | 2 +- 5 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 src/lumo/proc/tz.py diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index 44a4dc3..f13dc2e 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -33,6 +33,7 @@ from .experiment import Experiment from lumo.utils import safe_io as IO from lumo.utils.fmt import format_timedelta, strptime, strftime +from lumo.proc.tz import timezone PID_ROOT = os.path.join(progressroot(), 'pid') HB_ROOT = os.path.join(progressroot(), 'hb') @@ -394,7 +395,7 @@ def on_tag_update(sender): tag_ui = widgets.TagsInput(value=tags) tag_ui.observe(on_tag_update, names='value', type='change') - now = datetime.now() + now = datetime.now(timezone()) start = strptime(datestr=dic['progress']['start']) end = strptime(datestr=dic['progress']['last_edit_time']) diff --git a/src/lumo/proc/tz.py b/src/lumo/proc/tz.py new file mode 100644 index 0000000..9af8dac --- /dev/null +++ b/src/lumo/proc/tz.py @@ -0,0 +1,6 @@ +from .config import glob +import pytz + + +def timezone(): + return pytz.timezone(glob.get('timezone'), 'Asia/Shanghai') diff --git a/src/lumo/trainer/callbacks.py b/src/lumo/trainer/callbacks.py index 1018254..5889016 100644 --- a/src/lumo/trainer/callbacks.py +++ b/src/lumo/trainer/callbacks.py @@ -9,7 +9,7 @@ from datetime import datetime from functools import wraps from typing import NewType, Any, Optional, Dict, Union - +from lumo.proc.tz import timezone import psutil from torch.utils.data import DataLoader @@ -707,7 +707,7 @@ def request(self, data: dict): task = self.executor.submit(self.req.post, self.url, json={'data': data, 'type': 'timeevent', 'from': 'lumo.RemoteCallback', - 'datetime': datetime.now().isoformat()}) + 'datetime': datetime.now(timezone()).isoformat()}) self.submits.append(task) def on_hooked(self, source: Trainer, params: ParamsType): diff --git a/src/lumo/utils/fmt.py b/src/lumo/utils/fmt.py index ef6c537..8a52711 100644 --- a/src/lumo/utils/fmt.py +++ b/src/lumo/utils/fmt.py @@ -6,6 +6,7 @@ import numpy as np import torch +from lumo.proc.tz import timezone from . import re @@ -42,7 +43,7 @@ def strftime(fmt='%y-%m-%d-%H%M%S', dateobj: datetime = None): """get current date with formatted""" if dateobj is not None: return dateobj.strftime(fmt) - return datetime.now().strftime(fmt) + return datetime.now(timezone()).strftime(fmt) def strptime(datestr: str = None, fmt='%y-%m-%d-%H%M%S', ): diff --git a/src/lumo/utils/logger.py b/src/lumo/utils/logger.py index 04c3e4f..893db2e 100644 --- a/src/lumo/utils/logger.py +++ b/src/lumo/utils/logger.py @@ -128,7 +128,7 @@ def _format(self, *values, inline=False, fix=0, raw=False, append=False, level=V return None, None if self.adddate and not raw: - cur_date = datetime.now().strftime(self.datefmt) + cur_date = strftime(self.datefmt) else: cur_date = "" From 8cdcc72aa1c7443490ff0479e6e3529f27eab0cc Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 20:52:13 +0800 Subject: [PATCH 25/95] Add timezone --- src/lumo/proc/tz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo/proc/tz.py b/src/lumo/proc/tz.py index 9af8dac..fb89bec 100644 --- a/src/lumo/proc/tz.py +++ b/src/lumo/proc/tz.py @@ -3,4 +3,4 @@ def timezone(): - return pytz.timezone(glob.get('timezone'), 'Asia/Shanghai') + return pytz.timezone(glob.get('timezone', 'Asia/Shanghai')) From 7ec89348cde5cdfd589659de6d0a113b92eef10a Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 20:57:39 +0800 Subject: [PATCH 26/95] Add timezone --- src/lumo/contrib/scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo/contrib/scan.py b/src/lumo/contrib/scan.py index 247cb65..2155572 100644 --- a/src/lumo/contrib/scan.py +++ b/src/lumo/contrib/scan.py @@ -33,7 +33,7 @@ def base_main(pm: ScanBaseParams, files: List[str], dics: List[dict]): base = ("sleep {sleep} ; " + sys.executable + - " {file}.py {kwargs} --device={device}{group} & \n" + " {file} {kwargs} --device={device}{group} & \n" ) if not torch.cuda.is_available(): From 56ff87d9cb0c04815e3dacbb0fad74d001708430 Mon Sep 17 00:00:00 2001 From: sailist Date: Mon, 13 Mar 2023 23:09:35 +0800 Subject: [PATCH 27/95] Flake8 error fixed --- src/lumo/exp/watch.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index f13dc2e..e6ffa0c 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -172,6 +172,7 @@ def mask(self, df): Returns: A boolean mask of the given DataFrame based on the condition. """ + import pandas as pd names = self.name.split('.') value = df for i in names: @@ -224,7 +225,7 @@ def update(self): """Diff & Update""" updates = {} if not os.path.exists(self.hb_root): - return pd.DataFrame() + return {} for root, dirs, fs in os.walk(self.hb_root): if root == self.hb_root: continue @@ -325,6 +326,7 @@ def progress(self, is_alive=True, with_pandas=True): except: continue if with_pandas: + import pandas as pd return pd.DataFrame(res) else: return res @@ -359,7 +361,7 @@ def widget(self, assert params_filter is None or isinstance(params_filter, list) assert metric_filter is None or isinstance(metric_filter, list) - from ipywidgets import widgets, interact, Label + from ipywidgets import widgets, interact from IPython.display import display def make_row(dic: dict): From 03ea68a0f05ef4f142698d90918e6d1cb07c57ae Mon Sep 17 00:00:00 2001 From: sailist Date: Tue, 14 Mar 2023 10:45:23 +0800 Subject: [PATCH 28/95] rerun cli find test by watcher --- src/lumo/cli/__init__.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/lumo/cli/__init__.py b/src/lumo/cli/__init__.py index 714545e..d48da03 100644 --- a/src/lumo/cli/__init__.py +++ b/src/lumo/cli/__init__.py @@ -13,12 +13,17 @@ def rerun(test_name, **kwarg): Returns: """ - from lumo.exp.finder import retrieval_experiment - exp = retrieval_experiment(test_name) - if exp is not None: - exp.rerun([f'--{k}={v}' for k, v in kwarg.items()]) - else: + from lumo.exp.watch import Watcher + from lumo import Experiment + w = Watcher() + df = w.load() + df = df[df['test_name'] == test_name] + if len(df) == 0: + print(f'{test_name} not found') exit(1) + else: + exp = Experiment.from_cache(df.iloc[0].to_dict()) + exp.rerun([f'--{k}={v}' for k, v in kwarg.items()]) def note(test_name, description): From cc983380c444b25f69b8da0c2c43f303c71af367 Mon Sep 17 00:00:00 2001 From: sailist Date: Tue, 14 Mar 2023 11:02:35 +0800 Subject: [PATCH 29/95] rerun cli find test by watcher --- src/lumo/exp/experiment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lumo/exp/experiment.py b/src/lumo/exp/experiment.py index 8ec1646..e949811 100644 --- a/src/lumo/exp/experiment.py +++ b/src/lumo/exp/experiment.py @@ -697,10 +697,10 @@ def rerun(self, arg_list: List[str]): new_test_name = self._create_test_name(self.exp_dir) new_exp = Experiment(self.exp_name, test_name=new_test_name) self.dump_info('deprecated', {'rerun_at': {new_exp.test_name: True}}, append=True) - old_rerun_info = self.properties.get('rerun', None) + old_rerun_info = self.properties.get('rerun', {}) count = 1 - if old_rerun_info is not None: - count += old_rerun_info['count'] + if isinstance(old_rerun_info, dict): + count += old_rerun_info.get('repeat', 0) new_exp.dump_info('rerun', {'from': self.test_name, 'repeat': count}) from lumo.utils.subprocess import run_command old_exec = self.properties['execute'] From fa0ac339257c71b3f1841f523d4d201ff948470c Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 10:19:41 +0800 Subject: [PATCH 30/95] Add panel for Experiment --- src/lumo/exp/experiment.py | 13 ++- src/lumo/exp/exphook.py | 10 +- src/lumo/exp/lazy_panel.py | 169 ++++++++++++++++++++++++++++++++++ src/lumo/exp/watch.py | 31 +++---- src/lumo/trainer/callbacks.py | 1 + src/lumo/utils/subprocess.py | 14 ++- 6 files changed, 213 insertions(+), 25 deletions(-) create mode 100644 src/lumo/exp/lazy_panel.py diff --git a/src/lumo/exp/experiment.py b/src/lumo/exp/experiment.py index e949811..b941a69 100644 --- a/src/lumo/exp/experiment.py +++ b/src/lumo/exp/experiment.py @@ -122,6 +122,7 @@ def __init__(self, exp_name: str, test_name=None, paths=None): self.dump_string = self._trigger_change(self.dump_string) self.dump_note = self._trigger_change(self.dump_note) self.dump_info = self._trigger_change(self.dump_info) + self.trigger = self._trigger_change(self.trigger) self.logger = Logger() @@ -365,6 +366,9 @@ def exec_argv(self): except: return [] + def trigger(self): + pass + def _trigger_change(self, func): """ Decorator function that updates the heartbeat file before executing the decorated function. @@ -384,6 +388,7 @@ def inner(*args, **kwargs): """wrap function""" fn = self.heartbeat_fn io.dump_text(self.info_dir, fn) + io.dump_text(self.info_dir, self.pid_fn) return func(*args, **kwargs) return inner @@ -731,11 +736,13 @@ def initial(self): 'obj': runtime_pid_obj(), }) + self.dump_tags([]) + # register start self.dump_info('progress', {'start': strftime(), 'finished': False}, append=True) self.dump_progress(0) # register progress - io.dump_text(self.info_dir, self.pid_fn) + self.add_exit_hook(self.end) @call_on_main_process_wrap @@ -746,7 +753,6 @@ def start(self): if self.properties.get('progress', None) is not None: return self.initial() - self.set_prop('start', True) for hook in self._hooks.values(): # type: BaseExpHook hook.on_start(self) return self @@ -761,14 +767,11 @@ def end(self, end_code=0, *args, **extra): *args: Additional arguments to pass to the end hooks. **extra: Additional keyword arguments to pass to the end hooks. """ - if not self.is_alive: - return if not self.properties.get('progress', None) is None: return if self.properties['progress'].get('end', False): return - self.set_prop('end', True) if end_code == 0: self.dump_progress(1) diff --git a/src/lumo/exp/exphook.py b/src/lumo/exp/exphook.py index 68500fb..2184b3e 100644 --- a/src/lumo/exp/exphook.py +++ b/src/lumo/exp/exphook.py @@ -89,10 +89,14 @@ def exc_end(self, exc_type, exc_val, exc_tb): import traceback res = traceback.format_exception(exc_type, exc_val, exc_tb) res = [i for i in res if 'in _newfunc' not in i] - self.exp.dump_string('exception', "".join(res)) + + self.exp.dump_info('exception', { + 'exception_type': traceback.format_exception_only(exc_type, exc_val)[-1].strip(), + 'exception_content': "".join(res) + }) + self.exp.end( - end_code=1, - exc_type=traceback.format_exception_only(exc_type, exc_val)[-1].strip() + end_code=1 ) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py new file mode 100644 index 0000000..819978e --- /dev/null +++ b/src/lumo/exp/lazy_panel.py @@ -0,0 +1,169 @@ +try: + import panel as pn +except ImportError as e: + raise ImportError('') from e +from typing import Any + +from bokeh.core.property.primitive import String +from bokeh.plotting import figure +import pandas as pd +from panel.models.tabulator import TableEditEvent + +from lumo.exp.watch import Watcher +import datetime as dt +import numpy as np +from bokeh.models.widgets.tables import HTMLTemplateFormatter, NumberFormatter, TextEditor, StringEditor +from lumo import Experiment + +css = ''' +.tabulator-tableholder { + height: fit-content !important; +} + +.tabulator-row .tabulator-cell { + overflow: visible !important; + vertical-align: top; + min-height: 20px; +} +.bk { + +} +.tabulator .tabulator-col-resize-handle { + height: fit-content !important; + +} + +.tabulator-cell .tabulator-editable { + width: fit-content !important; + +} + +''' + +pn.extension('tabulator', raw_css=[css], css_files=[pn.io.resources.CSS_URLS['font-awesome']]) + + +def DictFormatter(column_name): + base_template = """ +
+ {column_name} + <% _.each(value, function(vv, key) { %> +
  • <%= key %>: <%= vv %>
  • + <% }); %> +
    """ + return HTMLTemplateFormatter(template=base_template.replace('{column_name}', column_name)) + + +class ExceptionFormatter: + base_template = """ +
    + + <%= value['exception_type'] %> + +

    + <%= value['exception_content'] %> +

    +
    """ + + +long_text = HTMLTemplateFormatter(template= + """ +
    + < % value['exception_type'] % > < / summary > + < p + style = 'display: inherit;white-space: break-spaces;word-wrap: normal;word-break: break-all;' > + < %- exception_content % > + < / p > + < / details > + """) + +tabulator_formatters = { + 'metrics': DictFormatter(column_name='metrics'), + 'progress': DictFormatter(column_name='progress'), + 'params': DictFormatter(column_name='params'), + 'exception': HTMLTemplateFormatter(template=ExceptionFormatter.base_template), +} + +tabulator_editors = { + 'exp_name': None, + 'test_name': None, + 'params': None, + 'metrics': None, + 'progress': None, + 'exception': None, + 'str': {'type': 'list', 'valuesLookup': True}, + # 'note': TextEditor(), + # 'tags': StringEditor(), +} + + +def make_experiment_tabular(df: pd.DataFrame, reload_fn): + """ + {'start': '23-03-14-224651', + 'finished': False, + 'ratio': 0, + 'last_edit_time': '23-03-14-224651', + 'update_from': None} + + Args: + df: + reload_fn: + + Returns: + + """ + # process exception + if 'exception' not in df.columns: + df['exception'] = {} + else: + df['exception'].fillna({}) + + # process progress + ratio = df['progress'].apply(lambda x: x.get('ratio', 0)) + end_code = df['progress'].apply(lambda x: x.get('end_code', None)) + + # ratio = 1 + # ratio < 1, no-end-code -> + + df = df[[ + 'exp_name', 'test_name', + 'progress', + 'exception', + 'params', 'metrics', + 'note', 'tags', + ]] + + def on_cell_change(e: TableEditEvent): + nonlocal df + if e.column == 'note': + Experiment.from_cache(df.iloc[e.row].to_dict()).dump_note(e.value) + else: + tags = [i.strip() for i in e.value.split(',')] + Experiment.from_cache(df.iloc[e.row].to_dict()).dump_tags(tags) + + df = reload_fn() + + df_widget = pn.widgets.Tabulator( + df, + groupby=['exp_name'], + hidden_columns=['git', 'paths', 'pinfo', + 'execute', 'lock', 'params.yaml', + 'params_hash', 'table_row', 'hooks', 'logger_args', + 'metric_board'], + pagination='local', + formatters=tabulator_formatters, + editors=tabulator_editors, + selection=[], + show_index=False, + configuration={ + 'clipboard': True, + # 'rowHeight': 100, + # 'columnDefaults': { + # 'headerSort': False, + # }, + } + ) + + df_widget.on_edit(on_cell_change) + return df_widget diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index e6ffa0c..fe88874 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -34,6 +34,7 @@ from lumo.utils import safe_io as IO from lumo.utils.fmt import format_timedelta, strptime, strftime from lumo.proc.tz import timezone +from lumo.proc import glob PID_ROOT = os.path.join(progressroot(), 'pid') HB_ROOT = os.path.join(progressroot(), 'hb') @@ -240,6 +241,8 @@ def update(self): raise e except: continue + finally: + os.remove(hb_file) for exp_name, tests in updates.items(): dic = PDict(os.path.join(self.db_root, f'{exp_name}.sqlite')) @@ -297,7 +300,7 @@ def valid_row(dic): else: return res - def progress(self, is_alive=True, with_pandas=True): + def progress(self, with_pandas=True): """ Returns a DataFrame of alive experiments. @@ -313,16 +316,23 @@ def progress(self, is_alive=True, with_pandas=True): if not f.endswith('.pid'): continue try: - test_root = IO.load_text(os.path.join(root, f)) + pid_f = os.path.join(root, f) + test_root = IO.load_text(pid_f) exp = Experiment.from_disk(test_root) - if exp.is_alive == is_alive: + + if exp.is_alive: res.append(exp.dict()) - if not exp.is_alive and exp.properties['progress'].get('finished', None) is None: + elif exp.properties['progress'].get('finished', None) is None: + if (datetime.timestamp(datetime.now()) - os.stat(pid_f).st_mtime) < glob.get( + 'ALIVE_SECONDS', 1800): + res.append(exp.dict()) + else: exp.dump_info('progress', { - 'end': strftime(), 'finished': True, 'end_code': -1, + 'end': strftime(), 'finished': True, 'end_code': -10, 'msg': 'ended by watcher'} ) + os.remove(pid_f) except: continue if with_pandas: @@ -339,17 +349,6 @@ def server(self): """simple server which make you note your experiments""" pass - def list_all(self, exp_root=None, limit=100) -> Dict[str, List[Experiment]]: - """ - Returns a dictionary of all experiments under exp_root directory. - - Args: - exp_root: The root directory to search for experiments. Default is None, which uses the default experiment root directory. - - Returns: - A dictionary of all experiments, where the keys are the names of the experiments and the values are lists of corresponding Experiment objects. - """ - def widget(self, is_finished: bool = None, is_alive: bool = None, diff --git a/src/lumo/trainer/callbacks.py b/src/lumo/trainer/callbacks.py index 5889016..b842d40 100644 --- a/src/lumo/trainer/callbacks.py +++ b/src/lumo/trainer/callbacks.py @@ -404,6 +404,7 @@ def update(self, trainer: Trainer): TrainStage.train in self.stage and ((trainer.idx + 1) == self.stage[TrainStage.train])): trainer.logger.inline(self.cur_tqdm.full_str()) trainer.logger.newline() + trainer.exp.trigger() def flush(self, trainer: Trainer): """Flush""" diff --git a/src/lumo/utils/subprocess.py b/src/lumo/utils/subprocess.py index f1e073d..c610905 100644 --- a/src/lumo/utils/subprocess.py +++ b/src/lumo/utils/subprocess.py @@ -4,7 +4,16 @@ import signal -def run_command(command, cwd=None, env=None): +def consume(p: subprocess.Popen): + for stream in [p.stdout, p.stderr]: + while True: + line = stream.readline().decode('utf-8') + if not line: + break + print(line, end='') + + +def run_command(command, cwd=None, env=None, non_block=False): """ Executes a command in the shell and captures its standard output and standard error. @@ -20,6 +29,9 @@ def run_command(command, cwd=None, env=None): cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if non_block: + return proc + try: while proc.poll() is None: # Wait for output from the process From 10ca8e577490fc015b552b552ba0c1512819a2ea Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 14:59:24 +0800 Subject: [PATCH 31/95] Give full string for better debug experience --- src/lumo/utils/safe_io.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lumo/utils/safe_io.py b/src/lumo/utils/safe_io.py index 434f189..0daffc7 100644 --- a/src/lumo/utils/safe_io.py +++ b/src/lumo/utils/safe_io.py @@ -43,8 +43,11 @@ def dump_json(obj, fn): Notes: The JSON data will be written with an indentation of 2 spaces. """ - with open(fn, 'w', encoding='utf-8') as w: - json.dump(obj, w, indent=2) + try: + with open(fn, 'w', encoding='utf-8') as w: + json.dump(obj, w, indent=2) + except TypeError as e: + raise TypeError(str(obj)) from e def dump_yaml(obj, fn): From f1a81219f95d4dc01ba945d62d9697582fbd949e Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 14:59:34 +0800 Subject: [PATCH 32/95] Panel for experiment --- src/lumo/exp/agent.py | 34 +++++++ src/lumo/exp/base.py | 2 +- src/lumo/exp/experiment.py | 65 +++++++------ src/lumo/exp/exphook.py | 6 +- src/lumo/exp/lazy_panel.py | 58 ++++++++---- src/lumo/exp/watch.py | 187 ++----------------------------------- 6 files changed, 125 insertions(+), 227 deletions(-) create mode 100644 src/lumo/exp/agent.py diff --git a/src/lumo/exp/agent.py b/src/lumo/exp/agent.py new file mode 100644 index 0000000..2ccf33f --- /dev/null +++ b/src/lumo/exp/agent.py @@ -0,0 +1,34 @@ +""" +heartbeat mechanism +""" +import time + +import psutil +from joblib import hash + +from lumo.core import BaseParams +from lumo.utils import safe_io as IO +from lumo.utils.fmt import strftime +from lumo.exp import Experiment + + +def wait_pid_stop(info_dir=None): + params = BaseParams() + params.info_dir = info_dir + params.from_args() + + exp = Experiment.from_disk(params.info_dir) + pid = exp.properties['pinfo'].get('pid') + info_dir = exp.info_dir + try: + while pid is not None and psutil.pid_exists(pid): + exp.trigger() + exp.dump_info('agent', {'last_edit_time': strftime()}) + + time.sleep(10) + except: + pass + + +if __name__ == '__main__': + wait_pid_stop() diff --git a/src/lumo/exp/base.py b/src/lumo/exp/base.py index fbfb056..4fd231a 100644 --- a/src/lumo/exp/base.py +++ b/src/lumo/exp/base.py @@ -88,7 +88,7 @@ def on_newpath(self, exp, *args, **kwargs): """ - def __repr__(self): + def __str__(self): """Return a string representation of the hook. Returns: diff --git a/src/lumo/exp/experiment.py b/src/lumo/exp/experiment.py index b941a69..203e08b 100644 --- a/src/lumo/exp/experiment.py +++ b/src/lumo/exp/experiment.py @@ -19,6 +19,7 @@ from lumo.utils import safe_io as io from lumo.utils.fmt import can_be_filename, strftime from lumo.utils.logger import Logger +from lumo.utils.subprocess import run_command from .base import BaseExpHook from ..proc.pid import pid_hash, runtime_pid_obj from .metric import Metric @@ -91,7 +92,7 @@ class Experiment: ENV_TEST_NAME_KEY = 'LUMO_EXP_TEST_NAME' - def __init__(self, exp_name: str, test_name=None, paths=None): + def __init__(self, exp_name: str = None, test_name=None, paths=None, info_dir=None): """ Initializes a new instance of the Experiment class. @@ -106,17 +107,24 @@ def __init__(self, exp_name: str, test_name=None, paths=None): raise ValueError(f'Experiment name should be a ligal filename(bettor only contain letter or underline),' f'but got {exp_name}.') - self._prop = {} - self._prop['exp_name'] = exp_name - if test_name is None: - test_name = os.environ.get(Experiment.ENV_TEST_NAME_KEY, None) - self._prop['test_name'] = test_name - if paths is None: - paths = {} - self._prop['paths'] = paths + if info_dir is not None: + exp = self.__class__.from_disk(info_dir=info_dir) + self._prop = exp._prop + self._hooks = exp._hooks + self._metric = exp._metric + else: + assert exp_name is not None + self._prop = {'exp_name': exp_name} + if test_name is None: + test_name = os.environ.get(Experiment.ENV_TEST_NAME_KEY, None) + self._prop['test_name'] = test_name + if paths is None: + paths = {} + self._prop['paths'] = paths + self._prop['note'] = '' - self._hooks = {} - self._metric = None + self._hooks = {} + self._metric = None # wrap self.dump_string = self._trigger_change(self.dump_string) @@ -183,7 +191,7 @@ def __repr__(self): Returns: str: A string representation of the Experiment object. """ - return f'{self.exp_name}->({self.test_name})' + return f'{self.__class__.__name__}(info_dir={self.info_dir})' def __str__(self): """ @@ -192,7 +200,7 @@ def __str__(self): Returns: str: A string representation of the Experiment object. """ - return self.__repr__() + return f'{self.__class__.__name__}(info_dir={self.info_dir})' def _repr_html_(self): """Return a html representation for a particular DataFrame.""" @@ -739,9 +747,10 @@ def initial(self): self.dump_tags([]) # register start - self.dump_info('progress', {'start': strftime(), 'finished': False}, append=True) + self.dump_info('progress', {'start': strftime()}, append=True) self.dump_progress(0) # register progress + self.proc = run_command(f'python3 -m lumo.exp.agent --info_dir={self.info_dir}', non_block=True) self.add_exit_hook(self.end) @@ -767,7 +776,7 @@ def end(self, end_code=0, *args, **extra): *args: Additional arguments to pass to the end hooks. **extra: Additional keyword arguments to pass to the end hooks. """ - if not self.properties.get('progress', None) is None: + if self.properties.get('progress', None) is None: return if self.properties['progress'].get('end', False): return @@ -775,9 +784,11 @@ def end(self, end_code=0, *args, **extra): if end_code == 0: self.dump_progress(1) - self.dump_info('progress', {'end': strftime(), 'finished': True, 'end_code': end_code}, append=True) + self.dump_info('progress', {'end': strftime(), 'end_code': end_code}, append=True) for hook in self._hooks.values(): # type: BaseExpHook hook.on_end(self, end_code=end_code, *args, **extra) + + self.proc.terminate() return self @call_on_main_process_wrap @@ -813,7 +824,7 @@ def add_exit_hook(self, func): import atexit def exp_func(): """Function executed before process exit.""" - func(self) + func() atexit.register(exp_func) @@ -838,12 +849,12 @@ def from_cache(cls, dic: dict): return self @classmethod - def from_disk(cls, path): + def from_disk(cls, info_dir): """ Creates an Experiment object from a test root directory on disk. Args: - path (str): The path to the test root directory. + info_dir (str): The path to the test root directory. Returns: Experiment: An Experiment object created from the test root directory. @@ -852,12 +863,12 @@ def from_disk(cls, path): ValueError: If the path is not a valid test root directory. """ from lumo.exp.watch import is_test_root - if not is_test_root(path): - raise ValueError(f'{path} is not a valid test_root') - path = os.path.abspath(path) - exp_dir = os.path.dirname(path) + if not is_test_root(info_dir): + raise ValueError(f'{info_dir} is not a valid test_root') + info_dir = os.path.abspath(info_dir) + exp_dir = os.path.dirname(info_dir) - paths_fn = os.path.join(path, 'info', f'paths.json') + paths_fn = os.path.join(info_dir, 'info', f'paths.json') if os.path.exists(paths_fn): try: paths = io.load_json(paths_fn) @@ -866,7 +877,7 @@ def from_disk(cls, path): else: paths = {} - self = cls(os.path.basename(exp_dir), test_name=os.path.basename(path), paths=paths) + self = cls(os.path.basename(exp_dir), test_name=os.path.basename(info_dir), paths=paths) # load prop for f in os.listdir(self.mk_ipath('info', is_dir=True)): @@ -903,8 +914,8 @@ class SimpleExperiment(Experiment): execute before and after the experiment. """ - def __init__(self, exp_name: str, root=None): - super().__init__(exp_name, root) + def __init__(self, exp_name: str = None, test_name=None, paths=None, info_dir=None): + super().__init__(exp_name, test_name, paths, info_dir) from . import exphook self.set_hook(exphook.LastCmd()) self.set_hook(exphook.LockFile()) diff --git a/src/lumo/exp/exphook.py b/src/lumo/exp/exphook.py index 2184b3e..90f2c0b 100644 --- a/src/lumo/exp/exphook.py +++ b/src/lumo/exp/exphook.py @@ -91,13 +91,11 @@ def exc_end(self, exc_type, exc_val, exc_tb): res = [i for i in res if 'in _newfunc' not in i] self.exp.dump_info('exception', { - 'exception_type': traceback.format_exception_only(exc_type, exc_val)[-1].strip(), + 'exception_type': exc_type.__name__, 'exception_content': "".join(res) }) - self.exp.end( - end_code=1 - ) + self.exp.end(end_code=1) # class TimeMonitor(ExpHook): diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index 819978e..1af7398 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -1,7 +1,9 @@ try: import panel as pn except ImportError as e: - raise ImportError('') from e + raise ImportError('The experiment panel is supported by panel, ' + 'you should use it by `pip install panel` first.') from e + from typing import Any from bokeh.core.property.primitive import String @@ -66,21 +68,16 @@ class ExceptionFormatter:
    """ -long_text = HTMLTemplateFormatter(template= - """ -
    - < % value['exception_type'] % > < / summary > - < p - style = 'display: inherit;white-space: break-spaces;word-wrap: normal;word-break: break-all;' > - < %- exception_content % > - < / p > - < / details > - """) +progress_formatter = HTMLTemplateFormatter( + template=""" +
    + """ +) tabulator_formatters = { 'metrics': DictFormatter(column_name='metrics'), - 'progress': DictFormatter(column_name='progress'), + 'progress_': DictFormatter(column_name='progress'), + 'progress': progress_formatter, 'params': DictFormatter(column_name='params'), 'exception': HTMLTemplateFormatter(template=ExceptionFormatter.base_template), } @@ -91,10 +88,9 @@ class ExceptionFormatter: 'params': None, 'metrics': None, 'progress': None, + 'progress_': None, 'exception': None, 'str': {'type': 'list', 'valuesLookup': True}, - # 'note': TextEditor(), - # 'tags': StringEditor(), } @@ -119,9 +115,34 @@ def make_experiment_tabular(df: pd.DataFrame, reload_fn): else: df['exception'].fillna({}) - # process progress - ratio = df['progress'].apply(lambda x: x.get('ratio', 0)) - end_code = df['progress'].apply(lambda x: x.get('end_code', None)) + def reformat_progress(dic): + ratio = dic.get('ratio', 0) + end_code = dic.get('end_code', None) + + # normal end -> end_code == 0 -> green + # interrupt by exception -> end_code > 0 -> red + # running -> end_code is None -> blue + # killed without any signal ( in 5 minute ) -> end_code is None -> blue + # killed and dumped by watcher -> end_code == -10 -> red + + if end_code is None: + color = 'blue' + elif ratio == 1: + color = 'green' + elif end_code == 0: + color = 'green' + elif end_code > 0: + color = 'red' + if ratio == 0: + ratio = 0.01 + + return { + 'ratio': f'{ratio:2%}', + 'color': color, + } + + df['progress_'] = df['progress'] + df['progress'] = df['progress'].apply(reformat_progress) # ratio = 1 # ratio < 1, no-end-code -> @@ -158,6 +179,7 @@ def on_cell_change(e: TableEditEvent): show_index=False, configuration={ 'clipboard': True, + 'tooltip': True, # 'rowHeight': 100, # 'columnDefaults': { # 'headerSort': False, diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index fe88874..6465b89 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -133,7 +133,7 @@ def __lt__(self, other): return self def __repr__(self): - return f'C({self.name} {self.op} {self.value})' + return f'C({self.name}, {self.value}, {self.op})' def in_(self, lis): """ @@ -239,7 +239,8 @@ def update(self): updates.setdefault(exp.exp_name, []).append(exp.cache()) except KeyboardInterrupt as e: raise e - except: + except Exception as e: + print(e) continue finally: os.remove(hb_file) @@ -304,9 +305,6 @@ def progress(self, with_pandas=True): """ Returns a DataFrame of alive experiments. - Args: - is_alive (bool): A boolean flag indicating whether to return only alive experiments. - Returns: A pandas DataFrame containing the experiment information of alive experiments. """ @@ -322,14 +320,14 @@ def progress(self, with_pandas=True): if exp.is_alive: res.append(exp.dict()) - elif exp.properties['progress'].get('finished', None) is None: + elif exp.properties['progress'].get('end_code', None) is None: if (datetime.timestamp(datetime.now()) - os.stat(pid_f).st_mtime) < glob.get( - 'ALIVE_SECONDS', 1800): + 'ALIVE_SECONDS', 60 * 5): # powered by exp/agent.py res.append(exp.dict()) else: exp.dump_info('progress', { - 'end': strftime(), 'finished': True, 'end_code': -10, + 'end': strftime(), 'end_code': -10, 'msg': 'ended by watcher'} ) os.remove(pid_f) @@ -349,175 +347,10 @@ def server(self): """simple server which make you note your experiments""" pass - def widget(self, - is_finished: bool = None, - is_alive: bool = None, - time_filter: list = None, - params_filter: list = None, - metric_filter: list = None - ): - """Create a user interface in jupyter with ipywidget""" - assert params_filter is None or isinstance(params_filter, list) - assert metric_filter is None or isinstance(metric_filter, list) - - from ipywidgets import widgets, interact - from IPython.display import display - - def make_row(dic: dict): - """ - Helper function for creating a row in the widgets grid. - - Args: - dic (dict): The dictionary containing the experiment information. - - Returns: - A list of ipywidgets objects for the row. - """ - exp = Experiment.from_cache(dic.copy()) - - def on_note_update(sender): - """when note textarea update""" - exp.dump_note(sender['new']) - - def on_tag_update(sender): - """when tag component update""" - exp.dump_tags(*sender['new']) - - note_ui = widgets.Textarea(dic['note']) - - note_ui.continuous_update = False - note_ui.observe(on_note_update, names='value', type='change') - - tags = dic.get('tags', []) - try: - tags = list(tags) - except: - tags = [] - tag_ui = widgets.TagsInput(value=tags) - tag_ui.observe(on_tag_update, names='value', type='change') - - now = datetime.now(timezone()) - start = strptime(datestr=dic['progress']['start']) - end = strptime(datestr=dic['progress']['last_edit_time']) - - human = widgets.VBox([ - - ]) - return [ - widgets.Label(dic['exp_name']), - widgets.Label(dic['test_name']), - widgets.Label(f"""{strftime('%y-%m-%d %H:%M:%S', dateobj=start)}"""), - widgets.Label(f"""{strftime('%y-%m-%d %H:%M:%S', dateobj=end)}"""), - widgets.HTML('\n'.join([ - f'{k}: {v}' - for k, v in dic['metrics'].items() - if isinstance(v, numbers.Number) - ])), - widgets.HBox([note_ui, - tag_ui, ]) - - ] - - test_status = widgets.RadioButtons(options=['full', 'running', 'failed', 'succeed', 'finished']) - start_filter = widgets.DatetimePicker() - end_filter = widgets.DatetimePicker() - - def status_filter(sender): - """when filter condition changed""" - print(sender) - make() - - test_status.observe(status_filter, names='value', type='change') - - # display() - - @interact - def make( - status=widgets.RadioButtons(options=['full', 'running', 'failed', 'succeed', 'finished']), - start=widgets.DatetimePicker(), - end=widgets.DatetimePicker(), - ): - """make widgets with filter condition.""" - if status == 'running': - df = self.progress() - elif status == 'finished': - df = self.progress(is_alive=False) - else: - df = self.load() - if status == 'succeed': - df = df[df['progress'].apply(lambda x: x['finished'])] - elif status == 'failed': - df = df[df['exception'].isna() == False] - - if start: - df = df.pipe( - lambda x: x[x['progress'].apply(lambda y: strptime(datestr=y['start'])) > start] - ) - if end: - df = df.pipe( - lambda x: x[x['progress'].apply(lambda y: strptime(datestr=y['end'])) < end] - ) - - if params_filter is not None: - df_params = df['params'] - masks = None - for condition in params_filter: - mask = condition.mask(df_params) - if masks is None: - masks = mask - else: - masks *= mask - df = df[masks] - - if metric_filter is not None: - df_params = df['metrics'] - masks = None - for condition in metric_filter: - mask = condition.mask(df_params) - if masks is None: - masks = mask - else: - masks *= mask - df = df[masks] - - exps = df.to_dict(orient='records') - # grid = widgets.GridspecLayout(len(exps) + 1, 7) - - children = [ - widgets.Label('exp_name'), - widgets.Label('test_name'), - widgets.Label('start'), - widgets.Label('end'), - widgets.Label('metrics'), - widgets.Label('note & tags'), - ] - # grid[0, 0] = widgets.Label('Meta') - # grid[0, 1] = widgets.Label('Metrics') - # grid[0, 2] = widgets.Label('Notes') - for i, exp in enumerate(exps, start=1): - row = make_row(exp) - children.extend(row) - # display(widgets.HBox(row)) - # for j, item in enumerate(row): - # grid[i, j] = item - - grid = widgets.GridBox(children=children, - - layout=widgets.Layout( - width='100%', - grid_template_columns=' '.join(['auto'] * 5) + ' auto', - # grid_template_rows='80px auto 80px', - grid_gap='5px 10px') - ) - display( - widgets.HTML(""" - - """), - grid, - - ) + def panel(self): + from .lazy_panel import make_experiment_tabular + widget = make_experiment_tabular(self.load(), self.load) + return widget.servable() def is_test_root(path: str) -> bool: From a6c388da0ac5ef842d68ef444066799e668ffb8b Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 14:59:45 +0800 Subject: [PATCH 33/95] Refine code --- src/lumo/trainer/components.py | 1 - src/lumo/trainer/trainer.py | 38 +++++++++++++++++++++------------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/lumo/trainer/components.py b/src/lumo/trainer/components.py index 37d5bf7..30dc078 100644 --- a/src/lumo/trainer/components.py +++ b/src/lumo/trainer/components.py @@ -15,7 +15,6 @@ def log_dir(self): @property def params_fn(self): res = self.mk_ipath('params.yaml') - self.dump_string('params.yaml', res) return res @property diff --git a/src/lumo/trainer/trainer.py b/src/lumo/trainer/trainer.py index 98ee368..87bf43b 100644 --- a/src/lumo/trainer/trainer.py +++ b/src/lumo/trainer/trainer.py @@ -4,6 +4,7 @@ import warnings from datetime import datetime from functools import lru_cache +from pprint import pformat from typing import Union, Dict, Any, Optional, Sequence, Mapping, Callable import numpy as np @@ -32,7 +33,7 @@ # overwrite send_to_device to resolve https://github.com/pytorch/pytorch/issues/83015 # from accelerate import Accelerator # from accelerate.utils import send_to_device -from ..utils.fmt import strftime +from ..utils.fmt import strftime, indent_print ParamsType = TrainerParams @@ -86,14 +87,13 @@ def __init__(self, params: ParamsType, dm: DataModule = None): self._saver = None self.params.iparams() - self.exp = TrainerExperiment(self.generate_exp_name()) + self.exp = TrainerExperiment(exp_name=self.generate_exp_name()) self._database = TableRow(self.exp.mk_ipath('metric.pkl'), persistent=self.is_main) self.metric_board = Metrics(self.exp.mk_bpath('board.sqlite'), persistent=self.is_main) self.metric = self.exp.metric - self.exp.dump_info('metric_board', self.metric_board.fpath) - self.exp.dump_info('table_row', self._database.fpath) + # self.exp.dump_info('table_row', self._database.fpath) self.rnd = RndManager() self.train_epoch_toggle = False @@ -110,6 +110,14 @@ def __init__(self, params: ParamsType, dm: DataModule = None): if dist.is_main(): self.params.to_yaml(self.exp.params_fn) + params_hash = self.params.hash() + self.exp.dump_info('trainer', { + 'params_meta': { + 'fn': self.exp.params_fn, + 'hash': params_hash + }, + 'board_fn': self.metric_board.fpath + }, append=True) self.exp.dump_info('params', self.params.to_dict()) self.set_global_steps(0) @@ -170,7 +178,9 @@ def logger(self): self._logger.debug('Enable debug log.') if self.is_main: fn = self._logger.add_log_dir(self.exp.log_dir) - self.exp.dump_info('logger_args', {'log_dir': fn}) + self.exp.dump_info('trainer', { + 'logger_fn': fn + }, append=True) return self._logger @@ -517,10 +527,10 @@ def to_device(self, item: Optional[Union[nn.Module, torch.Tensor, Sequence, Mapp def on_trainer_exception(self, func: Callable, exception: BaseException): """Updates database with error information when an exception occurs during training.""" - self.exp.dump_info('exception', dict(end=strftime(), - finished=False, - error=str(exception), - trainer_frame=str(func))) + # self.exp.dump_info('exception', dict(end=strftime(), + # finished=False, + # error=str(exception), + # trainer_frame=str(func)), append=True) @property def is_initialized(self): @@ -542,15 +552,15 @@ def initialize(self): return self.exp.start() - params_hash = self.params.hash() - self.exp.dump_string('params_hash', params_hash) - self.icallbacks(self.params) self.set_property('initial.callbacks', True) self.imodels(self.params) self.set_property('initial.model', True) self.set_property('initial', True) + self.logger.info('Use Experiment') + self.logger.info(self.exp) + def stop_train(self): """Toggle to stop train.""" self.train_toggle = True @@ -613,7 +623,6 @@ def train(self, dm: Union[DataModule, DataLoaderType] = None, params: ParamsType for eidx in range(params.epoch): # update training progress - self.exp.dump_train_eidx(eidx, params.epoch) self.set_epoch_idx(eidx) # train loop @@ -632,7 +641,8 @@ def train(self, dm: Union[DataModule, DataLoaderType] = None, params: ParamsType self.set_property('early_stop', f'meet limit_global_steps {limit_global_steps}') break - # update when train finished + self.exp.dump_train_eidx(eidx, params.epoch) + self.exp.end() return self._prop From 5c969105e5f94d6ab3f7a760b52e728628db5963 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 14:59:56 +0800 Subject: [PATCH 34/95] wraps --- src/lumo/decorators/process.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lumo/decorators/process.py b/src/lumo/decorators/process.py index 5a48812..0e1f7dc 100644 --- a/src/lumo/decorators/process.py +++ b/src/lumo/decorators/process.py @@ -1,5 +1,5 @@ from typing import Callable - +from functools import wraps from lumo.proc.dist import is_main @@ -18,6 +18,7 @@ def call_on_main_process_wrap(func) -> Callable: """ + @wraps(func) def inner(*args, **kwargs): if is_main(): return func(*args, **kwargs) From 45d219140139ac0681e5fc59c6243248e4156cd4 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 15:00:11 +0800 Subject: [PATCH 35/95] __repr__ to __str__ for pythonic --- src/lumo/data/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo/data/builder.py b/src/lumo/data/builder.py index 1522bc6..9f87243 100644 --- a/src/lumo/data/builder.py +++ b/src/lumo/data/builder.py @@ -117,7 +117,7 @@ def __init__(self): self._iter_cache = {} - def __repr__(self): + def __str__(self): if self.sized: return f'Builder(flow={pformat(self._outs)}, sized={self.sized}, size={len(self)}, iterable={self.iterable})' From 0ed78eb6359db4650135f0334ef37ab2dd1595ef Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 15:37:33 +0800 Subject: [PATCH 36/95] Panel --- src/lumo/exp/experiment.py | 11 +++++------ src/lumo/exp/lazy_panel.py | 22 +++++++++++++++++++++- src/lumo/exp/watch.py | 2 +- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/lumo/exp/experiment.py b/src/lumo/exp/experiment.py index 203e08b..6367358 100644 --- a/src/lumo/exp/experiment.py +++ b/src/lumo/exp/experiment.py @@ -103,10 +103,6 @@ def __init__(self, exp_name: str = None, test_name=None, paths=None, info_dir=No Raises: ValueError: If the experiment name is not a legal filename. """ - if not can_be_filename(exp_name): - raise ValueError(f'Experiment name should be a ligal filename(bettor only contain letter or underline),' - f'but got {exp_name}.') - if info_dir is not None: exp = self.__class__.from_disk(info_dir=info_dir) self._prop = exp._prop @@ -114,6 +110,9 @@ def __init__(self, exp_name: str = None, test_name=None, paths=None, info_dir=No self._metric = exp._metric else: assert exp_name is not None + if not can_be_filename(exp_name): + raise ValueError(f'Experiment name should be a ligal filename(bettor only contain letter or underline),' + f'but got {exp_name}.') self._prop = {'exp_name': exp_name} if test_name is None: test_name = os.environ.get(Experiment.ENV_TEST_NAME_KEY, None) @@ -191,7 +190,7 @@ def __repr__(self): Returns: str: A string representation of the Experiment object. """ - return f'{self.__class__.__name__}(info_dir={self.info_dir})' + return f'{self.__class__.__name__}(info_dir="{self.info_dir})"' def __str__(self): """ @@ -744,7 +743,7 @@ def initial(self): 'obj': runtime_pid_obj(), }) - self.dump_tags([]) + self.dump_tags() # register start self.dump_info('progress', {'start': strftime()}, append=True) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index 1af7398..967bb6b 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -28,7 +28,7 @@ min-height: 20px; } .bk { - + } .tabulator .tabulator-col-resize-handle { height: fit-content !important; @@ -157,6 +157,7 @@ def reformat_progress(dic): def on_cell_change(e: TableEditEvent): nonlocal df + if e.column == 'note': Experiment.from_cache(df.iloc[e.row].to_dict()).dump_note(e.value) else: @@ -187,5 +188,24 @@ def on_cell_change(e: TableEditEvent): } ) + # def on_cell_click(e): + # nonlocal enable_update + # if e.column in {'note', 'tags'}: + # enable_update = False + + # enable_update = True + # df_widget.on_click(on_cell_click) df_widget.on_edit(on_cell_change) + # + # def update_time(): + # if enable_update: + # time_now = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + # time_component.object = f"

    {time_now}

    " + # + # time_component = pn.pane.HTML("current time is") + # pn.state.onload(update_time) + # pn.state.add_periodic_callback(update_time, 1000) # 每1000毫秒执行一次更新时间的操作 + # + # widget = pn.Column(time_component, df_widget) # .servable() + return df_widget diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index 6465b89..75f2b5a 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -350,7 +350,7 @@ def server(self): def panel(self): from .lazy_panel import make_experiment_tabular widget = make_experiment_tabular(self.load(), self.load) - return widget.servable() + return widget def is_test_root(path: str) -> bool: From 3881c75e8332a440534ac7e848da84947c1c4cb9 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 15:41:36 +0800 Subject: [PATCH 37/95] change css --- src/lumo/exp/lazy_panel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index 967bb6b..e33c400 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -28,7 +28,7 @@ min-height: 20px; } .bk { - + height: fit-content !important; } .tabulator .tabulator-col-resize-handle { height: fit-content !important; From eeac87f7b3b6b48c5937a4fc045db444072922ef Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 15:58:30 +0800 Subject: [PATCH 38/95] Version fix --- src/lumo/exp/lazy_panel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index e33c400..741803e 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -114,6 +114,8 @@ def make_experiment_tabular(df: pd.DataFrame, reload_fn): df['exception'] = {} else: df['exception'].fillna({}) + if 'tags' not in df.columns: + df['tags'] = [] def reformat_progress(dic): ratio = dic.get('ratio', 0) From 24a83949093fa708f2ce7c384d046b31a993a1df Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 16:02:43 +0800 Subject: [PATCH 39/95] Version fix --- src/lumo/exp/lazy_panel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index 741803e..eab81e4 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -115,7 +115,7 @@ def make_experiment_tabular(df: pd.DataFrame, reload_fn): else: df['exception'].fillna({}) if 'tags' not in df.columns: - df['tags'] = [] + df['tags'] = np.empty((len(df.index), 0)).tolist() def reformat_progress(dic): ratio = dic.get('ratio', 0) From 9ccd0635d80c54b3d90db99f921fc3cb26f0df14 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 16:06:30 +0800 Subject: [PATCH 40/95] Version fix --- src/lumo/exp/watch.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index 75f2b5a..b0dfd73 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -253,6 +253,31 @@ def update(self): dic.flush() return updates + def fullupdate(self): + updates = {} + for root, dirs, fs in os.walk(self.exp_root): + for f in dirs: + if is_test_name(f): + info_dir = os.path.join(root, f) + try: + exp = Experiment.from_disk(info_dir) + updates.setdefault(exp.exp_name, []).append(exp.cache()) + except KeyboardInterrupt as e: + raise e + except Exception as e: + print(e) + continue + + for exp_name, tests in updates.items(): + dic = PDict(os.path.join(self.db_root, f'{exp_name}.sqlite')) + dic.clear() + + for test in tests: + dic[test['test_name']] = test + dic.flush() + + return updates + def load(self, with_pandas=True): """ Loads the experiment information from heartbeat files and the experiment databases. From e23460ff5472d8e75fc276faa41e6e7f31d249d0 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 16:08:01 +0800 Subject: [PATCH 41/95] Version fix --- src/lumo/exp/watch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index b0dfd73..c885ff7 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -204,7 +204,7 @@ class Watcher: def __init__(self, exp_root=None, hb_root=None, pid_root=None, db_root=None): if exp_root is None: - exp_root = os.path.join(exproot(), 'hb') + exp_root = exproot() if hb_root is None: hb_root = os.path.join(cache_dir(), 'heartbeat') From 9b9032b460b5e1a52c560ff13e1991f65376cc87 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 16:33:20 +0800 Subject: [PATCH 42/95] Version fix --- src/lumo/exp/lazy_panel.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index eab81e4..60faca4 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -118,6 +118,12 @@ def make_experiment_tabular(df: pd.DataFrame, reload_fn): df['tags'] = np.empty((len(df.index), 0)).tolist() def reformat_progress(dic): + if not isinstance(dic, dict): + return { + 'ratio': '100%', + 'color': 'yellow', + } + ratio = dic.get('ratio', 0) end_code = dic.get('end_code', None) From a04981d64c3aa93c546746c0a81ec119848ec846 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 16:38:29 +0800 Subject: [PATCH 43/95] Version fix --- src/lumo/exp/lazy_panel.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index 60faca4..ced0942 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -26,10 +26,13 @@ overflow: visible !important; vertical-align: top; min-height: 20px; + height: fit-content !important; } + .bk { height: fit-content !important; } + .tabulator .tabulator-col-resize-handle { height: fit-content !important; From d1c588dee010db00712333f50830561633047c65 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 16:42:39 +0800 Subject: [PATCH 44/95] Version fix --- src/lumo/exp/lazy_panel.py | 5 +---- src/lumo/exp/watch.py | 10 ++++------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index ced0942..cca8289 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -97,7 +97,7 @@ class ExceptionFormatter: } -def make_experiment_tabular(df: pd.DataFrame, reload_fn): +def make_experiment_tabular(df: pd.DataFrame): """ {'start': '23-03-14-224651', 'finished': False, @@ -107,7 +107,6 @@ def make_experiment_tabular(df: pd.DataFrame, reload_fn): Args: df: - reload_fn: Returns: @@ -175,8 +174,6 @@ def on_cell_change(e: TableEditEvent): tags = [i.strip() for i in e.value.split(',')] Experiment.from_cache(df.iloc[e.row].to_dict()).dump_tags(tags) - df = reload_fn() - df_widget = pn.widgets.Tabulator( df, groupby=['exp_name'], diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index c885ff7..24af30d 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -364,17 +364,15 @@ def progress(self, with_pandas=True): else: return res - def interactive(self): - """interactive, mark, label, note in ipython environment.""" - pass - def server(self): """simple server which make you note your experiments""" pass - def panel(self): + def panel(self, df=None): from .lazy_panel import make_experiment_tabular - widget = make_experiment_tabular(self.load(), self.load) + if df is None: + df = self.load() + widget = make_experiment_tabular(df) return widget From 2ad758462173217e9eaf69231a2dd553f871f529 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 16:46:12 +0800 Subject: [PATCH 45/95] Enable extra columns --- src/lumo/exp/lazy_panel.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index cca8289..93cef5c 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -157,13 +157,21 @@ def reformat_progress(dic): # ratio = 1 # ratio < 1, no-end-code -> - df = df[[ + columns = set(df.columns) - {'git', 'paths', 'pinfo', + 'execute', 'lock', 'params.yaml', + 'params_hash', 'table_row', 'hooks', 'logger_args', + 'metric_board'} + top_columns = [ 'exp_name', 'test_name', 'progress', 'exception', 'params', 'metrics', 'note', 'tags', - ]] + ] + + columns = columns - set(top_columns) + + df = df[top_columns + list(columns)] def on_cell_change(e: TableEditEvent): nonlocal df @@ -177,10 +185,7 @@ def on_cell_change(e: TableEditEvent): df_widget = pn.widgets.Tabulator( df, groupby=['exp_name'], - hidden_columns=['git', 'paths', 'pinfo', - 'execute', 'lock', 'params.yaml', - 'params_hash', 'table_row', 'hooks', 'logger_args', - 'metric_board'], + # hidden_columns=[], pagination='local', formatters=tabulator_formatters, editors=tabulator_editors, From b09268592b6985e19993140c00b2d6ee3d2a6daf Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 16:49:41 +0800 Subject: [PATCH 46/95] Enable extra columns --- src/lumo/exp/lazy_panel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index 93cef5c..c16ada1 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -151,7 +151,7 @@ def reformat_progress(dic): 'color': color, } - df['progress_'] = df['progress'] + # df['progress_'] = df['progress'] df['progress'] = df['progress'].apply(reformat_progress) # ratio = 1 @@ -160,6 +160,7 @@ def reformat_progress(dic): columns = set(df.columns) - {'git', 'paths', 'pinfo', 'execute', 'lock', 'params.yaml', 'params_hash', 'table_row', 'hooks', 'logger_args', + 'agent', 'trainer', 'progress_', 'metric_board'} top_columns = [ 'exp_name', 'test_name', From 8995e707e7ddeb0dc19dde5e30ee6b529f8a7526 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 16:50:38 +0800 Subject: [PATCH 47/95] Copy mode in panel --- src/lumo/exp/lazy_panel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index c16ada1..f223d0d 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -111,6 +111,7 @@ def make_experiment_tabular(df: pd.DataFrame): Returns: """ + df = df.reset_index(drop=True) # process exception if 'exception' not in df.columns: df['exception'] = {} From 018d12414f479c67242944d15c34bc66a57baec2 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 20:06:26 +0800 Subject: [PATCH 48/95] Boundcheck --- src/lumo/exp/lazy_panel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index f223d0d..aeca0d9 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -142,7 +142,7 @@ def reformat_progress(dic): color = 'green' elif end_code == 0: color = 'green' - elif end_code > 0: + elif end_code > 0 or end_code < 0: color = 'red' if ratio == 0: ratio = 0.01 From 012aa47d7fd58cfa2b887aa60768fd69f5312345 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 20:06:37 +0800 Subject: [PATCH 49/95] Add example to use Condition --- src/lumo/exp/watch.py | 111 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 107 insertions(+), 4 deletions(-) diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index 24af30d..727c6f3 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -78,7 +78,52 @@ def not_in_(ser, value): class Condition: - """Represents a condition to filter data based on a certain criteria.""" + """ + Represents a condition to filter data based on a certain criteria. + + row filter: + ``` + from lumo import C + import pandas as pd + + # create a sample DataFrame + data = {'name': ['Alice', 'Bob', 'Charlie', 'David', 'Emily'], + 'age': [25, 30, 35, 40, 45], + 'city': ['New York', 'Los Angeles', 'Chicago', 'Houston', 'Miami']} + df = pd.DataFrame(data) + + # create and apply the condition to filter the DataFrame + filtered_df = (C['age'] >= 35).apply(df) + + # print the filtered DataFrame + print(filtered_df) + ``` + + column edit: + ``` + (C+{'city.index':'cindex'}).apply(df).columns + # Index(['name', 'age', 'city', 'cindex'], dtype='object') + + (-C['city']).apply(df).columns + # Index(['name', 'age'], dtype='object') + + (C-['city']).apply(df).columns + (C-'city').apply(df).columns + # Index(['name', 'age'], dtype='object') + + (C-['city','name']).apply(df).columns + # Index(['age'], dtype='object') + ``` + + pipeline: + ``` + C.pipe(df,[ + (C['age']>35), + C+{'city.index':'cindex'}, + C-['city','name'] + ]) + ``` + """ def __init__(self, name: str = None, value=None, op=None): self.name = name @@ -91,8 +136,32 @@ def __getattr__(self, item): def __getitem__(self, item): return Condition(item) + def __add__(self, other): + self.op = 'add_column' + self.value = {} + self.op + if isinstance(other, str): + self.value[other] = other + elif isinstance(other, dict): + self.value.update(other) + else: + raise NotImplementedError() + return self + + def __sub__(self, other): + self.name = None + self.op = 'drop_column' + self.value = {} + if isinstance(other, str): + self.value.update({other: None}) + elif isinstance(other, (list, set, dict)): + self.value.update({k: None for k in other}) + else: + raise NotImplementedError() + return self + def __neg__(self): - self.drop = True + self.op = 'drop_column' return self def __ge__(self, other): @@ -180,12 +249,46 @@ def mask(self, df): if isinstance(value, pd.DataFrame): value = value[i] else: - value = df.apply(lambda x: x[i]) + value = value.apply(lambda x: x[i]) return mapping[self.op](value, self.value) + def capply(self, df): + import pandas as pd + df = df.reset_index(drop=True) + if self.op == 'drop_column': + if isinstance(self.name, str): + var = [self.name] + elif isinstance(self.value, str): + var = [self.value] + else: + var = list(self.value) + df = df.drop(var, axis=1) + else: + assert isinstance(self.value, dict) + for name, aim in self.value.items(): + names = name.split('.') + value = df + for i in names: + if isinstance(value, pd.DataFrame): + value = value[i] + else: + value = value.apply(lambda x: x[i]) + df[aim] = value + return df + def apply(self, df): """Returns a new DataFrame with only the rows that meet the condition.""" - return df[self.mask(df)] + if not self.op.endswith('column'): + return df[self.mask(df)].reset_index(drop=True) + else: + return self.capply(df) + + def pipe(self, df, conditions: List['Condition']): + """Applies a list of conditions to a DataFrame using the pipe method.""" + filtered_df = df + for condition in conditions: + filtered_df = condition.apply(filtered_df) # .reset_index(drop=True) + return filtered_df C = Condition() From 71ee67042b8ff503e987de9e9fc31bfdbb230868 Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 21:10:07 +0800 Subject: [PATCH 50/95] import --- src/lumo/exp/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lumo/exp/__init__.py b/src/lumo/exp/__init__.py index da726c3..b6de703 100644 --- a/src/lumo/exp/__init__.py +++ b/src/lumo/exp/__init__.py @@ -1 +1,3 @@ from .experiment import SimpleExperiment, Experiment +from .watch import Watcher, C +from .metric import Metric From b5f7f19838f814e6fafe7c9bd2afb57a62de8afc Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 21:10:12 +0800 Subject: [PATCH 51/95] import --- src/lumo/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lumo/__init__.py b/src/lumo/__init__.py index 5893105..5dd93f3 100644 --- a/src/lumo/__init__.py +++ b/src/lumo/__init__.py @@ -4,8 +4,9 @@ __version__ = "0.15.0" from .core import Params, ParamsType, MetricType, Meter, Record, TrainStage, BaseParams +from .proc import glob from .data import DataLoader, DataModule, DatasetBuilder, LumoDataLoader, CollateBase, DataLoaderSide -from .exp import SimpleExperiment, Experiment +from .exp import SimpleExperiment, Experiment, Metric, Watcher, C from .trainer import Trainer, TrainerParams, callbacks, RndManager from .utils import Logger From 0160c3c70eadc87a941d6390edc2f1f0105f1f0e Mon Sep 17 00:00:00 2001 From: sailist Date: Wed, 15 Mar 2023 21:10:20 +0800 Subject: [PATCH 52/95] Add board command --- src/lumo/cli/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lumo/cli/__init__.py b/src/lumo/cli/__init__.py index d48da03..1a213d1 100644 --- a/src/lumo/cli/__init__.py +++ b/src/lumo/cli/__init__.py @@ -41,7 +41,7 @@ def note(test_name, description): print(f"Adding note '{description}' to {test_name}") -def server(port=8080): +def board(port=11606, address=None, open=True): """ Args: @@ -50,6 +50,9 @@ def server(port=8080): Returns: """ + from lumo import Watcher + w = Watcher() + w.panel().show(port=port, address=address, open=open) print(f"Starting server on port {port}") @@ -57,5 +60,5 @@ def main(): fire.Fire({ 'rerun': rerun, 'note': note, - 'server': server, + 'board': board, }) From 2ef8c1656fea336b1baffbb87f3b34d58eea934e Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 14:19:32 +0800 Subject: [PATCH 53/95] Add one-file examples --- examples/imagenet.py | 253 +++++++++++++++++++++++++++++++++++++ examples/mnist.py | 117 ++++++++++++++++++ examples/moco.py | 288 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 658 insertions(+) create mode 100644 examples/imagenet.py create mode 100644 examples/mnist.py create mode 100644 examples/moco.py diff --git a/examples/imagenet.py b/examples/imagenet.py new file mode 100644 index 0000000..4a61e57 --- /dev/null +++ b/examples/imagenet.py @@ -0,0 +1,253 @@ +import sys +from pathlib import Path +from typing import Union + +import torch +from PIL import Image +from torch.utils.data import DataLoader +import os + +from torchvision.datasets.folder import default_loader + +from lumo import DatasetBuilder, MetricType, Trainer, TrainerParams, Meter, callbacks, DataModule +from torchvision.datasets import FakeData, ImageFolder +from torchvision import transforms +from torchvision.models.resnet import resnet18 +from torch import nn +from lumo.proc.dist import is_dist, is_main +from torch.nn import functional as F +from lumo.proc import glob +from lumo.utils.subprocess import run_command + +"""define transforms""" + + +def none(mean, std): + return transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize(mean, std) + ]) + + +def standard(mean, std, resize=None): + return transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize(mean, std) + ]) + + +"""create datasets""" + + +def imagenet(split='train'): + """ + download from https://www.kaggle.com/c/imagenet-object-localization-challenge/overview/description + ``` + mkdir imagenet + cd ./imagenet + kaggle competitions download -c imagenet-object-localization-challenge + unzip imagenet-object-localization-challenge.zip + tar -xvf imagenet_object_localization_patched2019.tar.gz + ls + >>> ILSVRC LOC_synset_mapping.txt LOC_val_solution.csv imagenet_object_localization_patched2019.tar.gz + >>> LOC_sample_submission.csv LOC_train_solution.csv imagenet-object-localization-challenge.zip + ``` + """ + root = glob['imagenet'] + if split == 'train': + file = Path(root).joinpath('ILSVRC', 'ImageSets', 'CLS-LOC', 'train_cls.txt') + train_root = os.path.join(root, 'ILSVRC/Data/CLS-LOC/train') + with file.open('r') as r: + lines = r.readlines() + imgs = [line.split(' ')[0] for line in lines] + name_cls_map = {name: i for i, name in enumerate(sorted(set([i.split('/')[0] for i in imgs])))} + xs = [os.path.join(train_root, f'{i}.JPEG') for i in imgs] + ys = [name_cls_map[i.split('/')[0]] for i in imgs] + else: + file = Path(root).joinpath('LOC_val_solution.csv') + val_root = os.path.join(root, 'ILSVRC/Data/CLS-LOC/val') + + with file.open('r') as r: + r.readline() + lines = r.readlines() + lines = [line.split(',') for line in lines] + lines = [[img, res.split(' ')[0]] for img, res in lines] + + name_cls_map = {name: i for i, name in enumerate(sorted(set([i[1] for i in lines])))} + xs = [os.path.join(val_root, f'{img}.JPEG') for img, _ in lines] + ys = [name_cls_map[res] for _, res in lines] + + return list(xs), list(ys) + + +def take_first(item): + return item[0] + + +def take_second(item): + return item[1] + + +def make_dataset(dummy=False): + if dummy: + train_dataset = FakeData(1281167, (3, 224, 224), 1000, transforms.ToTensor()) + val_dataset = FakeData(50000, (3, 224, 224), 1000, transforms.ToTensor()) + ds = ( + DatasetBuilder() + .add_input('fake', train_dataset) + .add_output('fake', 'xs', transform=take_first) + .add_output('fake', 'ys', transform=take_second) + ) + test_ds = ( + DatasetBuilder() + .add_input('fake', val_dataset) + .add_output('fake', 'xs', transform=take_first) + .add_output('fake', 'ys', transform=take_second) + ) + else: + train_dataset = ImageFolder(os.path.join(glob['imagenet'], 'train')) + val_dataset = ImageFolder(os.path.join(glob['imagenet'], 'val')) + + xs, ys = list(zip(*train_dataset.samples)) + test_xs, test_ys = list(zip(*val_dataset.samples)) + + mean = [0.485, 0.456, 0.406] + std = [0.229, 0.224, 0.225] + + ds = ( + DatasetBuilder() + .add_input('xs', xs, transform=default_loader) # 注册样本来源,命名为 'xs' + .add_input('ys', ys) # 注册标签来源,命名为 'ys' + .add_output('xs', 'xs', transform=standard(mean, std)) # 添加一个弱增广输出 'xs0' + .add_output('ys', 'ys') # 添加标签输出 + ) + + print(ds) + print(ds[0].keys()) + + test_ds = ( + DatasetBuilder() + .add_input('xs', test_xs, transform=default_loader) # 注册样本来源,命名为 'xs' + .add_input('ys', test_ys) # 注册标签来源,命名为 'ys' + .add_output('xs', 'xs', transform=none(mean, std)) # 测试样本不使用增广 + .add_output('ys', 'ys') # 添加标签输出 + ) + + print(test_ds) + print(test_ds[0].keys()) + return ds, test_ds + + +class LargeParams(TrainerParams): + def __init__(self): + super().__init__() + self.optim = self.OPTIM.create_optim('SGD', + lr=0.06, + momentum=0.9, + weight_decay=5e-5, + ) + self.lr_decay_end = 0.00001 + self.batch_size = 512 + self.dummy = False + self.multiprocessing_distributed = True + + +ParamsType = LargeParams + + +class LargeModel(nn.Module): + + def __init__(self, feature_dim) -> None: + super().__init__() + self.backbone = resnet18() + in_feature = self.backbone.fc.in_features + self.backbone.fc = nn.Identity() + self.head = nn.Linear(in_feature, feature_dim, bias=True) + + def forward(self, xs): + feature_map = self.backbone(xs) + feature = self.head(feature_map) + return feature + + +class LargeTrainer(Trainer): + + def icallbacks(self, params: ParamsType): + callbacks.LoggerCallback().hook(self) + + def imodels(self, params: ParamsType): + self.model = resnet18(num_classes=1000) + self.optim = params.optim.build(self.model.parameters()) + + self.lr_sche = params.SCHE.Cos( + start=params.optim.lr, end=params.lr_decay_end, + left=0, + right=len(self.train_dataloader) * params.epoch + ) + # manually trigger send_to_device method + self.to_device() + + def train_step(self, batch, params: ParamsType = None) -> MetricType: + super().train_step(batch, params) + m = Meter() + eval_xs, xs, ys = batch['xs0'], batch['xs1'], batch['ys'] + logits = self.model(xs) + + Lall = F.cross_entropy(logits, ys) + self.optim.zero_grad() + self.accelerate.backward(Lall) + self.optim.step() + + # change lr by training epoch + cur_lr = self.lr_sche.apply(self.optim, self.eidx) + + with torch.no_grad(): + m.mean.Lall = Lall + eval_logits = self.model(eval_xs) + m.mean.Ax = torch.eq(eval_logits.argmax(dim=-1), ys).float().mean() + m.last.lr = cur_lr + return m + + def test_step(self, batch, params: ParamsType = None) -> MetricType: + m = Meter() + xs, ys = batch['xs'], batch['ys'] + logits = self.model(xs) + + all_logits = self.accelerate.gather(logits) + all_ys = self.accelerate.gather(ys) + + m.test_mean.Ax = torch.eq(all_logits.argmax(dim=-1), all_ys).float() + return m + + +def main(): + params = LargeParams() + params.from_args() + + if params.multiprocessing_distributed and not is_dist(): + command = ' '.join([ + 'accelerate', 'launch', + *sys.argv, + ]) + run_command(command) + else: # not distributed or in distribution environment + # create datamodule to contain dataloader + ds, test_ds = make_dataset(dummy=params.dummy) + dl = ds.DataLoader(batch_size=params.batch_size) + test_dl = test_ds.DataLoader(batch_size=params.batch_size) + dm = DataModule() + dm.regist_dataloader(train=dl, test=test_dl) + + # with the input of params and dataloader, the initialization of models and optimizers in Trainer, + # then the output will be the trained parameters, metrics and logs. + trainer = LargeTrainer(params, dm=dm) + + trainer.train() # or trainer.train(dm=dl) if dm are not given above + trainer.test() # or trainer.test(dm=dl) + trainer.save_last_model() + + +if __name__ == '__main__': + main() diff --git a/examples/mnist.py b/examples/mnist.py new file mode 100644 index 0000000..45cc6a6 --- /dev/null +++ b/examples/mnist.py @@ -0,0 +1,117 @@ +import torch +from lumo import DatasetBuilder, MetricType, Trainer, TrainerParams, Meter, callbacks, DataModule +from lumo.proc.path import cache_dir +from torchvision.datasets.mnist import MNIST +from torchvision.transforms import Compose, RandomCrop, Normalize +from torch import nn +from torch.nn import functional as F + + +def make_dataset(): + data = MNIST(root=cache_dir(), train=True, download=True) + test_data = MNIST(root=cache_dir(), train=False, download=True) + + mean = torch.mean(data.data / 255.) + std = torch.std(data.data / 255.) + + ds = ( + DatasetBuilder() + .add_input('xs', data.data.float().unsqueeze(1)) # 注册样本来源,命名为 'xs' + .add_input('ys', data.targets) # 注册标签来源,命名为 'ys' + .add_output('xs', 'xs0', transform=Normalize(mean=(mean,), std=(std,))) # 添加一个弱增广输出 'xs0' + .add_output('xs', 'xs1', + transform=Compose( + [RandomCrop(28, padding=4), Normalize(mean=(mean,), std=(std,))])) # 添加一个强增广输出 'xs1' + .add_output('ys', 'ys') # 添加标签输出 + ) + print(ds) + print(ds[0].keys()) + + test_ds = ( + DatasetBuilder() + .add_input('xs', test_data.data.float().unsqueeze(1)) # 注册样本来源,命名为 'xs' + .add_input('ys', test_data.targets) # 注册标签来源,命名为 'ys' + .add_output('xs', 'xs', transform=Normalize(mean=(mean,), std=(std,))) # 测试样本不使用增广 + .add_output('ys', 'ys') # 添加标签输出 + ) + + print(test_ds) + print(test_ds[0].keys()) + return ds, test_ds + + +class MNISTParams(TrainerParams): + def __init__(self): + super().__init__() + self.batch_size = 128 + self.optim = self.OPTIM.create_optim('SGD', lr=0.0001, momentum=0.9) + + +ParamsType = MNISTParams + + +class MNISTTrainer(Trainer): + + def icallbacks(self, params: ParamsType): + super().icallbacks(params) + callbacks.LoggerCallback().hook(self) + + def imodels(self, params: ParamsType): + super().imodels(params) + self.model = nn.Sequential( + nn.Flatten(), + nn.Linear(28 * 28, 128), + nn.ReLU(), + nn.Linear(128, 10), + ) + self.optim = params.optim.build(self.model.parameters()) + + # manually trigger send_to_device method + self.to_device() + + def train_step(self, batch, params: ParamsType = None) -> MetricType: + super().train_step(batch, params) + m = Meter() + eval_xs, xs, ys = batch['xs0'], batch['xs1'], batch['ys'] + logits = self.model(xs) + Lall = F.cross_entropy(logits, ys) + self.optim.zero_grad() + Lall.backward() + self.optim.step() + with torch.no_grad(): + m.mean.Lall = Lall + eval_logits = self.model(eval_xs) + m.mean.Ax = torch.eq(eval_logits.argmax(dim=-1), ys).float().mean() + return m + + def test_step(self, batch, params: ParamsType = None) -> MetricType: + m = Meter() + xs, ys = batch['xs'], batch['ys'] + logits = self.model(xs) + m.test_mean.Ax = torch.eq(logits.argmax(dim=-1), ys).float() + return m + + +def main(): + ds, test_ds = make_dataset() + + # create datamodule to contain dataloader + dl = ds.DataLoader(batch_size=32) + test_dl = test_ds.DataLoader(batch_size=32) + dm = DataModule() + dm.regist_dataloader(train=dl, test=test_dl) + + params = MNISTParams() + params.epoch = 10 + params.from_args() + # with the input of params and dataloader, the initialization of models and optimizers in Trainer, + # then the output will be the trained parameters, metrics and logs. + trainer = MNISTTrainer(params, dm=dm) + + trainer.train() # or trainer.train(dm=dl) if dm are not given above + trainer.test() # or trainer.test(dm=dl) + trainer.save_last_model() + + +if __name__ == '__main__': + main() diff --git a/examples/moco.py b/examples/moco.py new file mode 100644 index 0000000..f911367 --- /dev/null +++ b/examples/moco.py @@ -0,0 +1,288 @@ +""" +refer to +https://colab.research.google.com/github/facebookresearch/moco/blob/colab-notebook/colab/moco_cifar10_demo.ipynb +""" +from typing import Union + +import torch +from PIL import Image +from torch.utils.data import DataLoader + +from lumo import DatasetBuilder, MetricType, Trainer, TrainerParams, Meter, callbacks, DataModule +from lumo.contrib import EMA, MemoryBank, StorageBank +from lumo.contrib.accelerate.utils import send_to_device +from lumo.contrib.nn.loss import contrastive_loss2 +from lumo.proc.path import cache_dir +from torchvision.datasets.cifar import CIFAR10 +from torchvision import transforms +from torchvision.models.resnet import resnet18 +from torch import nn +from torch.nn import functional as F + +"""define transforms""" + + +def none(mean, std): + return transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize(mean, std) + ]) + + +def simclr(mean, std): + return transforms.Compose([ + transforms.RandomResizedCrop(32), + transforms.RandomHorizontalFlip(), + transforms.RandomApply([ + transforms.ColorJitter(0.4, 0.4, 0.4, 0.1) + ], p=0.8), + transforms.RandomGrayscale(0.2), + transforms.ToTensor(), + transforms.Normalize(mean, std) + ]) + + +"""create datasets""" + + +def make_dataset(): + data = CIFAR10(root=cache_dir(), train=True, download=True) + data.data = [Image.fromarray(img) for img in data.data] + test_data = CIFAR10(root=cache_dir(), train=False, download=True) + + mean = [0.4914, 0.4822, 0.4465] + std = [0.2023, 0.1994, 0.2010] + + ds = ( + DatasetBuilder() + .add_input('xs', data.data) # 注册样本来源,命名为 'xs' + .add_input('ys', data.targets) # 注册标签来源,命名为 'ys' + .add_output('xs', 'xs0', transform=simclr(mean, std)) # 添加一个弱增广输出 'xs0' + .add_output('xs', 'xs1', transform=simclr(mean, std)) # 添加一个强增广输出 'xs1' + .add_output('ys', 'ys') # 添加标签输出 + ) + + # for knn test + memo_ds = ( + DatasetBuilder() + .add_input('xs', data.data) # 注册样本来源,命名为 'xs' + .add_input('ys', data.targets) # 注册标签来源,命名为 'ys' + .add_output('xs', 'xs', transform=none(mean, std)) # 添加一个弱增广输出 'xs0' + .add_output('ys', 'ys') # 添加标签输出 + ) + print(ds) + print(ds[0].keys()) + + test_ds = ( + DatasetBuilder() + .add_idx('idx') # add index key for sample + .add_input('xs', test_data.data) # 注册样本来源,命名为 'xs' + .add_input('ys', test_data.targets) # 注册标签来源,命名为 'ys' + .add_output('xs', 'xs', transform=none(mean, std)) # 测试样本不使用增广 + .add_output('ys', 'ys') # 添加标签输出 + ) + + print(test_ds) + print(test_ds[0].keys()) + return ds, memo_ds, test_ds + + +class MocoParams(TrainerParams): + def __init__(self): + super().__init__() + self.optim = self.OPTIM.create_optim('SGD', + lr=0.06, + momentum=0.9, + weight_decay=5e-5, + ) + self.lr_decay_end = 0.00001 + self.temperature = 0.1 + self.ema_alpha = 0.99 + self.feature_dim = 129 + self.queue_size = 4096 + self.batch_size = 512 + self.symmetric = False + + +ParamsType = MocoParams + + +class MocoModel(nn.Module): + + def __init__(self, feature_dim) -> None: + super().__init__() + self.backbone = resnet18() + in_feature = self.backbone.fc.in_features + self.backbone.fc = nn.Identity() + self.head = nn.Linear(in_feature, feature_dim, bias=True) + + def forward(self, xs): + feature_map = self.backbone(xs) + feature = self.head(feature_map) + return feature + + +def knn_predict(feature, feature_bank, feature_labels, classes, knn_k, knn_t): + # compute cos similarity between each feature vector and feature bank ---> [B, N] + sim_matrix = torch.mm(feature, feature_bank) + # [B, K] + sim_weight, sim_indices = sim_matrix.topk(k=knn_k, dim=-1) + # [B, K] + sim_labels = torch.gather(feature_labels.expand(feature.size(0), -1), dim=-1, index=sim_indices) + sim_weight = (sim_weight / knn_t).exp() + + # counts for each class + one_hot_label = torch.zeros(feature.size(0) * knn_k, classes, device=sim_labels.device) + # [B*K, C] + one_hot_label = one_hot_label.scatter(dim=-1, index=sim_labels.view(-1, 1), value=1.0) + # weighted score ---> [B, C] + pred_scores = torch.sum(one_hot_label.view(feature.size(0), -1, classes) * sim_weight.unsqueeze(dim=-1), dim=1) + + pred_labels = pred_scores.argsort(dim=-1, descending=True) + return pred_labels + + +class MocoTrainer(Trainer): + + def icallbacks(self, params: ParamsType): + callbacks.LoggerCallback().hook(self) + + def imodels(self, params: ParamsType): + self.model = MocoModel(params.feature_dim) + self.ema_model = EMA(self.model, alpha=params.ema_alpha) + + self.optim = params.optim.build(self.model.parameters()) + + self.tensors = StorageBank() + self.tensors.register('test_feature', dim=params.feature_dim, k=len(self.dm.test_dataset)) + self.tensors.register('test_ys', dim=-1, k=len(self.dm.test_dataset), dtype=torch.long) + + self.mem = MemoryBank() + # do not need normalize because normalize is applied in contrastive_loss2 function + self.mem.register('negative', dim=params.feature_dim, k=params.queue_size) + self.mem['negative'] = F.normalize(self.mem['negative'], dim=-1) + + self.lr_sche = params.SCHE.Cos( + start=params.optim.lr, end=params.lr_decay_end, + left=0, + right=len(self.train_dataloader) * params.epoch + ) + # manually trigger send_to_device method + self.to_device() + + def train_step(self, batch, params: ParamsType = None) -> MetricType: + m = Meter() + im_query, im_key, ys = batch['xs0'], batch['xs1'], batch['ys'] + feat_query = self.model.forward(im_query) + + with torch.no_grad(): + # shuffle for making use of BN + feat_key = self.ema_model.forward(im_key) # keys: NxC + feat_key = F.normalize(feat_key, dim=1) # already normalized + + feat_query = F.normalize(feat_query, dim=1) + + if params.symmetric: + Lcsa = contrastive_loss2(query=feat_query, key=feat_key, + memory=self.mem['negative'], + query_neg=False, key_neg=False, + temperature=params.temperature, + norm=False) + Lcsb = contrastive_loss2(query=feat_key, key=feat_query, + memory=self.mem['negative'], + query_neg=False, key_neg=False, + temperature=params.temperature, + norm=False) + Lcs = Lcsa + Lcsb + else: + + Lcs = contrastive_loss2(query=feat_query, key=feat_key.detach(), + memory=self.mem['negative'].clone().detach(), + query_neg=False, key_neg=False, + temperature=params.temperature, + norm=False) + + # memory bank + with torch.no_grad(): + if params.symmetric: + self.mem.push('negative', torch.cat([feat_query, feat_key], dim=0)) + else: + self.mem.push('negative', feat_key) + + self.optim.zero_grad() + self.accelerate.backward(Lcs) + self.optim.step() + cur_lr = self.lr_sche.apply(self.optim, self.global_steps) + + # metrics + with torch.no_grad(): + m.mean.Lcs = Lcs + m.last.lr = cur_lr + return m + + def test_step(self, batch, params: ParamsType = None) -> MetricType: + idx = batch['idx'] + xs, ys = batch['xs'], batch['ys'] + feature = self.model(xs) + self.tensors.scatter('test_feature', feature, idx) + self.tensors.scatter('test_ys', ys, idx) + + def test(self, dm: Union[DataModule, DataLoader] = None, params: ParamsType = None, limit_step=None): + super().test(dm, params, limit_step) # run default test loop + self.save_last_model() + + feature_bank = [] + with torch.no_grad(): + # generate feature bank + for batch in self.dm['memo']: + batch = send_to_device(batch, self.device) + data, target = batch['xs'], batch['ys'] + feature = self.model(data) + feature = F.normalize(feature, dim=1) + feature_bank.append(feature) + + feature_bank = torch.cat(feature_bank, dim=0).t().contiguous() + # [N] + feature_labels = torch.tensor(self.dm['memo'].dataset.inputs['ys'], device=feature_bank.device) + # loop test data to predict the label by weighted knn search + + pred_labels = knn_predict(self.tensors['test_feature'], + feature_bank, feature_labels, params.n_classes, params.knn_k, params.knn_t) + total_num = pred_labels.shape[0] + total_top1 = torch.eq(pred_labels[:, 0], self.tensors['test_ys']).float().sum().item() + + knn_acc = total_top1 / total_num * 100 + + max_knn_acc = self.metric.dump_metric('Knn', knn_acc, cmp='max', flush=True) + self.logger.info(f'Best Knn Top-1 acc: {max_knn_acc}, current: {knn_acc}') + + if knn_acc >= max_knn_acc: + self.save_best_model() + + +def main(): + ds, memo_ds, test_ds = make_dataset() + + params = MocoParams() + params.from_args() + + # create datamodule to contain dataloader + dl = ds.DataLoader(batch_size=params.batch_size) + memo_dl = memo_ds.DataLoader(batch_size=params.batch_size) + test_dl = test_ds.DataLoader(batch_size=params.batch_size) + dm = DataModule() + dm.regist_dataloader(train=dl, + test=test_dl, + memo=memo_dl) # add extra dataloader with any name + + # with the input of params and dataloader, the initialization of models and optimizers in Trainer, + # then the output will be the trained parameters, metrics and logs. + trainer = MocoTrainer(params, dm=dm) + + trainer.train() # or trainer.train(dm=dl) if dm are not given above + trainer.test() # or trainer.test(dm=dl) + trainer.save_last_model() + + +if __name__ == '__main__': + main() From 7973f91050864e4acb5b7aef39d0e713ea1a373c Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 14:20:01 +0800 Subject: [PATCH 54/95] Meter: agg array --- src/lumo/core/meter.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/lumo/core/meter.py b/src/lumo/core/meter.py index 44bc21b..7c52bc4 100644 --- a/src/lumo/core/meter.py +++ b/src/lumo/core/meter.py @@ -126,9 +126,10 @@ def __setitem__(self, key, value): stg = self._avg.get(key, None) isscalar = value.size == 1 - if stg is None: + if stg is None: # Auto infer a stg method for the value dtype = value.dtype.name + # sanity check if self._stage in {'min', 'max'} and not isscalar: raise ValueError( f'Only support min/max(a) operator on scalar metrics, but got data of shape {value.shape}.') @@ -145,9 +146,14 @@ def __setitem__(self, key, value): self._stage = 'last' self._avg[key] = self._stage + stg = self._stage if isscalar: value = value.item() + elif stg == {'min', 'max', 'sum'}: + value = getattr(np, stg)(value).item() + elif stg == 'mean': + value = [getattr(np, stg)(value).item(), value.size] self._rec[key] = value self._stage = 'default' @@ -192,6 +198,17 @@ def mean(self): self._stage = 'mean' return self + @property + def test_mean(self): + """ + Sets the aggregation method to 'mean'. + + Returns: + The meter itself. + """ + self._stage = 'mean' + return self + @property def last(self): """ @@ -405,6 +422,10 @@ def __repr__(self): def update(self, item): """Updates the reduction value with a new item.""" self.cur = item + count = 1 + if isinstance(item, list) and len(item) == 2: + item, count = item + item = detach(item) avg = self.gb_method @@ -419,8 +440,8 @@ def update(self, item): elif avg in {'mean', 'sum'}: if len(self.acc) == 0: self.acc.append(0) - self.acc[0] = self.acc[0] + item - self.c += 1 + self.acc[0] = self.acc[0] + item * count + self.c += count elif avg == 'max': self._res = max(self.cur, self._res) elif avg == 'min': From d2c1ae2f46591c7070f42f1af92ac0097d7d6fa7 Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 14:20:28 +0800 Subject: [PATCH 55/95] DatasetBuilder: fix typehint of pycharm --- src/lumo/data/builder.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/lumo/data/builder.py b/src/lumo/data/builder.py index 9f87243..7515aa2 100644 --- a/src/lumo/data/builder.py +++ b/src/lumo/data/builder.py @@ -360,7 +360,7 @@ def scale_to_size(self, size: int): size (int): the size to scale the dataset builder to. Returns: - self (DatasetBuilder): the scaled dataset builder. + the scaled dataset builder. """ assert isinstance(size, int) assert 'pseudo_repeat' not in self._prop @@ -378,7 +378,7 @@ def repeat(self, multiple: int): multiple (int): the number of times to repeat the dataset builder. Returns: - self (DatasetBuilder): the repeated dataset builder. + the repeated dataset builder. """ assert isinstance(multiple, int) assert 'pseudo_length' not in self._prop @@ -424,7 +424,7 @@ def chain(self): Set the mode of the dataset builder to chain. Returns: - self (DatasetBuilder): the dataset builder with the chain mode set. + the dataset builder with the chain mode set. """ self._prop['mode'] = 'chain' return self @@ -434,7 +434,7 @@ def item(self): Set the mode of the dataset builder to item. Returns: - self (DatasetBuilder): the dataset builder with the item mode set. + the dataset builder with the item mode set. """ self._prop['mode'] = 'item' return self @@ -444,7 +444,7 @@ def zip(self): Set the mode of the dataset builder to zip. Returns: - self (DatasetBuilder): the dataset builder with the zip mode set. + the dataset builder with the zip mode set. """ self._prop['mode'] = 'zip' return self @@ -489,7 +489,7 @@ def add_idx(self, name): name (str): the name of the index. Returns: - self (DatasetBuilder): the dataset builder with the index added. + the dataset builder with the index added. """ outkeys = self._outs.setdefault(f"::idx::", []) assert name not in self._outkeys, f'Output key {name} duplicated.' @@ -507,7 +507,7 @@ def add_input(self, name: str, source, transform: SingleValueTransform = None): transform (SingleValueTransform): the transform to apply to the input source. Returns: - self (DatasetBuilder): the dataset builder with the input source added. + the dataset builder with the input source added. Notes: Iterable object without `__len__` method currently are not well-tested. Be careful to use them in DatasetBuilder. @@ -529,7 +529,7 @@ def add_input_transform(self, name: str, transform: SingleValueTransform = None) transform (SingleValueTransform): the transform to add. Returns: - self (DatasetBuilder): the dataset builder with the transform added. + the dataset builder with the transform added. """ assert name in self._data, f'Source {name} should be added.' warnings.warn('`add` may cause confusion, use set_input_transform ') @@ -545,7 +545,7 @@ def add_output(self, name: str, outkey: str, transform: SingleValueTransform = N transform (SingleValueTransform): the transform to apply to the output. Returns: - self (DatasetBuilder): the dataset builder with the output added. + the dataset builder with the output added. """ assert name in self._data, f'Must have data source {name} first.' @@ -567,7 +567,7 @@ def add_output_transform(self, outkey: str, transform: SingleValueTransform = No transform (SingleValueTransform): the transform to add. Returns: - self (DatasetBuilder): the dataset builder with the transform added. + the dataset builder with the transform added. """ assert outkey in self._outkeys, f'Output key {outkey} should be added.' warnings.warn('add may cause confusion, use set_output_transform ') @@ -581,7 +581,7 @@ def add_global_transform(self, transform: DictTransform): transform (DictTransform): the global transform to apply to the dataset. Returns: - self (DatasetBuilder): the dataset builder with the global transform added. + the dataset builder with the global transform added. """ self._transforms['::global::'] = transform return self @@ -595,7 +595,7 @@ def set_input_transform(self, name, transform: SingleValueTransform = None): transform (SingleValueTransform): the transform to set. Returns: - self (DatasetBuilder): the dataset builder with the transform set. + the dataset builder with the transform set. """ self._transforms[name] = transform return self @@ -609,7 +609,7 @@ def set_output_transform(self, outkey, transform: SingleValueTransform = None): transform (SingleValueTransform): the transform to set. Returns: - self (DatasetBuilder): the dataset builder with the transform set. + the dataset builder with the transform set. """ self._transforms[f'::{outkey}'] = transform return self From f13f1c3daf00cbb22706210b693c3377db6aff91 Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 14:20:49 +0800 Subject: [PATCH 56/95] panel: beautify --- src/lumo/exp/lazy_panel.py | 124 ++++++++++++++++++++++--------------- 1 file changed, 75 insertions(+), 49 deletions(-) diff --git a/src/lumo/exp/lazy_panel.py b/src/lumo/exp/lazy_panel.py index aeca0d9..a035bf3 100644 --- a/src/lumo/exp/lazy_panel.py +++ b/src/lumo/exp/lazy_panel.py @@ -1,20 +1,16 @@ +import numbers + try: import panel as pn except ImportError as e: raise ImportError('The experiment panel is supported by panel, ' 'you should use it by `pip install panel` first.') from e -from typing import Any - -from bokeh.core.property.primitive import String -from bokeh.plotting import figure import pandas as pd from panel.models.tabulator import TableEditEvent -from lumo.exp.watch import Watcher -import datetime as dt import numpy as np -from bokeh.models.widgets.tables import HTMLTemplateFormatter, NumberFormatter, TextEditor, StringEditor +from bokeh.models.widgets.tables import HTMLTemplateFormatter from lumo import Experiment css = ''' @@ -48,7 +44,7 @@ pn.extension('tabulator', raw_css=[css], css_files=[pn.io.resources.CSS_URLS['font-awesome']]) -def DictFormatter(column_name): +def FoldDictFormatter(column_name): base_template = """
    {column_name} @@ -59,6 +55,15 @@ def DictFormatter(column_name): return HTMLTemplateFormatter(template=base_template.replace('{column_name}', column_name)) +def DictFormatter(column_name): + base_template = """ + <% _.each(value, function(vv, key) { %> +
  • <%= key %>: <%= vv %>
  • + <% }); %> + """ + return HTMLTemplateFormatter(template=base_template) + + class ExceptionFormatter: base_template = """
    @@ -79,9 +84,9 @@ class ExceptionFormatter: tabulator_formatters = { 'metrics': DictFormatter(column_name='metrics'), - 'progress_': DictFormatter(column_name='progress'), + 'progress_': FoldDictFormatter(column_name='progress'), 'progress': progress_formatter, - 'params': DictFormatter(column_name='params'), + 'params': FoldDictFormatter(column_name='params'), 'exception': HTMLTemplateFormatter(template=ExceptionFormatter.base_template), } @@ -93,10 +98,56 @@ class ExceptionFormatter: 'progress': None, 'progress_': None, 'exception': None, - 'str': {'type': 'list', 'valuesLookup': True}, + 'note': {'type': 'list', 'valuesLookup': True}, + 'tags': {'type': 'list', 'valuesLookup': True}, } +def drop_nonscalar_metric(dic): + if not isinstance(dic, dict): + return + + for k, v in list(dic.items()): + if not isinstance(v, numbers.Number): + dic.pop(k) + return dic + + +def reformat_progress(dic): + if not isinstance(dic, dict): + return { + 'ratio': '100%', + 'color': 'yellow', + } + + ratio = dic.get('ratio', 0) + end_code = dic.get('end_code', None) + + # normal end -> end_code == 0 -> green + # interrupt by exception -> end_code > 0 -> red + # running -> end_code is None -> blue + # killed without any signal ( in 5 minute ) -> end_code is None -> blue + # killed and dumped by watcher -> end_code == -10 -> red + + if end_code is None: + color = 'blue' + elif ratio == 1: + color = 'green' + elif end_code == 0: + color = 'green' + elif end_code > 0 or end_code < 0: + color = 'red' + if ratio == 0: + ratio = 0.01 + else: + color = 'black' + + return { + 'ratio': f'{ratio:2%}', + 'color': color, + } + + def make_experiment_tabular(df: pd.DataFrame): """ {'start': '23-03-14-224651', @@ -117,40 +168,11 @@ def make_experiment_tabular(df: pd.DataFrame): df['exception'] = {} else: df['exception'].fillna({}) + if 'tags' not in df.columns: df['tags'] = np.empty((len(df.index), 0)).tolist() - def reformat_progress(dic): - if not isinstance(dic, dict): - return { - 'ratio': '100%', - 'color': 'yellow', - } - - ratio = dic.get('ratio', 0) - end_code = dic.get('end_code', None) - - # normal end -> end_code == 0 -> green - # interrupt by exception -> end_code > 0 -> red - # running -> end_code is None -> blue - # killed without any signal ( in 5 minute ) -> end_code is None -> blue - # killed and dumped by watcher -> end_code == -10 -> red - - if end_code is None: - color = 'blue' - elif ratio == 1: - color = 'green' - elif end_code == 0: - color = 'green' - elif end_code > 0 or end_code < 0: - color = 'red' - if ratio == 0: - ratio = 0.01 - - return { - 'ratio': f'{ratio:2%}', - 'color': color, - } + df['metrics'] = df['metrics'].apply(drop_nonscalar_metric) # df['progress_'] = df['progress'] df['progress'] = df['progress'].apply(reformat_progress) @@ -158,11 +180,14 @@ def reformat_progress(dic): # ratio = 1 # ratio < 1, no-end-code -> - columns = set(df.columns) - {'git', 'paths', 'pinfo', - 'execute', 'lock', 'params.yaml', - 'params_hash', 'table_row', 'hooks', 'logger_args', - 'agent', 'trainer', 'progress_', - 'metric_board'} + extra_columns = set(df.columns) - {'git', 'paths', 'pinfo', + 'execute', 'lock', 'params.yaml', + 'params_hash', 'table_row', 'hooks', 'logger_args', + 'agent', 'trainer', 'progress_', + 'start', 'end', + 'tensorboard_args', + 'state', + 'metric_board'} top_columns = [ 'exp_name', 'test_name', 'progress', @@ -171,9 +196,10 @@ def reformat_progress(dic): 'note', 'tags', ] - columns = columns - set(top_columns) + extra_columns = extra_columns - set(top_columns) + [tabulator_editors.setdefault(k, None) for k in extra_columns] - df = df[top_columns + list(columns)] + df = df[top_columns + list(extra_columns)] def on_cell_change(e: TableEditEvent): nonlocal df From 4eedf1694bb90ea6f2ee9acac2878905c85c4275 Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 14:21:22 +0800 Subject: [PATCH 57/95] Watcher: avoid replacement when dump_info --- src/lumo/exp/watch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lumo/exp/watch.py b/src/lumo/exp/watch.py index 727c6f3..97950f3 100644 --- a/src/lumo/exp/watch.py +++ b/src/lumo/exp/watch.py @@ -457,6 +457,7 @@ def progress(self, with_pandas=True): { 'end': strftime(), 'end_code': -10, 'msg': 'ended by watcher'} + , append=True ) os.remove(pid_f) except: From b798e19623af164d9fb7fa1b2423151d91c15536 Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 14:21:40 +0800 Subject: [PATCH 58/95] trainer: fix small bugs --- src/lumo/trainer/base.py | 2 +- src/lumo/trainer/callbacks.py | 6 ++++-- src/lumo/trainer/trainer.py | 8 +++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/lumo/trainer/base.py b/src/lumo/trainer/base.py index bab315f..26a5c06 100644 --- a/src/lumo/trainer/base.py +++ b/src/lumo/trainer/base.py @@ -134,7 +134,7 @@ def inner(dm=None, params=None, *args, **kwargs): process_loader = getattr(self, 'process_loader', None) if process_loader is not None: process_loader(dm, TrainStage.create_from_str(func.__name__)) - func(*args, **kwargs) + func(dm, params, *args, **kwargs) return inner diff --git a/src/lumo/trainer/callbacks.py b/src/lumo/trainer/callbacks.py index b842d40..dad473e 100644 --- a/src/lumo/trainer/callbacks.py +++ b/src/lumo/trainer/callbacks.py @@ -353,6 +353,7 @@ def __init__(self, step_frequence=3, break_in=1000): self.step = step_frequence file = tempfile.TemporaryFile('w') self.temp = file + self.cur_tqdm = None def on_imodels_end(self, trainer: Trainer, func, params: ParamsType, result: Any, *args, **kwargs): super().on_imodels_end(trainer, func, params, result, *args, **kwargs) @@ -409,8 +410,9 @@ def update(self, trainer: Trainer): def flush(self, trainer: Trainer): """Flush""" self.c = 0 - trainer.logger.inline(self.cur_tqdm) - trainer.logger.newline() + if self.cur_tqdm is not None: + trainer.logger.inline(self.cur_tqdm) + trainer.logger.newline() def on_train_epoch_begin(self, trainer: Trainer, func, params: ParamsType, *args, **kwargs): super().on_train_epoch_begin(trainer, func, params, *args, **kwargs) diff --git a/src/lumo/trainer/trainer.py b/src/lumo/trainer/trainer.py index 87bf43b..b714be0 100644 --- a/src/lumo/trainer/trainer.py +++ b/src/lumo/trainer/trainer.py @@ -610,6 +610,8 @@ def train(self, dm: Union[DataModule, DataLoaderType] = None, params: ParamsType ValueError: If no data loader is available for training. """ + self.change_stage(TrainStage.train) + loader = self.select_loader(dm) if not loader: loader = self.train_dataloader @@ -816,8 +818,7 @@ def change_stage(self, stage: TrainStage): else: v.eval() - @classmethod - def select_loader(cls, dm=None): + def select_loader(self, dm=None, stage=None): """ Selects the appropriate loader based on the given data module. @@ -830,7 +831,8 @@ def select_loader(cls, dm=None): loader = None if dm: if isinstance(dm, DataModule): - loader = dm.train_dataloader + loader = dm.get_loader_with_stage(stage=self.trainstage) + # loader = dm.train_dataloader elif isinstance(dm, DataLoader) or isinstance(dm, DataLoaderSide): loader = dm else: From d7c4fa9ce3d4962664fa8a4ef6297299dc48e547 Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 14:21:50 +0800 Subject: [PATCH 59/95] run_command: typehint --- src/lumo/utils/subprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo/utils/subprocess.py b/src/lumo/utils/subprocess.py index c610905..749a793 100644 --- a/src/lumo/utils/subprocess.py +++ b/src/lumo/utils/subprocess.py @@ -13,7 +13,7 @@ def consume(p: subprocess.Popen): print(line, end='') -def run_command(command, cwd=None, env=None, non_block=False): +def run_command(command: str, cwd=None, env=None, non_block=False): """ Executes a command in the shell and captures its standard output and standard error. From 82655dad16ed27be965921c281b4b9877969ce01 Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 14:21:59 +0800 Subject: [PATCH 60/95] contrib: importing --- src/lumo/contrib/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lumo/contrib/__init__.py b/src/lumo/contrib/__init__.py index a4f53fd..e4f1925 100644 --- a/src/lumo/contrib/__init__.py +++ b/src/lumo/contrib/__init__.py @@ -3,3 +3,4 @@ """ from .module.ema import EMA from .optim.grouper import ParamGrouper +from .module.memoty_bank import MemoryBank, StorageBank From ca8c10d762c2352c024c4acaa506bddee50944da Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 14:26:01 +0800 Subject: [PATCH 61/95] imagenet example --- examples/imagenet.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/imagenet.py b/examples/imagenet.py index 4a61e57..463ff14 100644 --- a/examples/imagenet.py +++ b/examples/imagenet.py @@ -192,7 +192,7 @@ def imodels(self, params: ParamsType): def train_step(self, batch, params: ParamsType = None) -> MetricType: super().train_step(batch, params) m = Meter() - eval_xs, xs, ys = batch['xs0'], batch['xs1'], batch['ys'] + xs, ys = batch['xs'], batch['ys'] logits = self.model(xs) Lall = F.cross_entropy(logits, ys) @@ -205,8 +205,7 @@ def train_step(self, batch, params: ParamsType = None) -> MetricType: with torch.no_grad(): m.mean.Lall = Lall - eval_logits = self.model(eval_xs) - m.mean.Ax = torch.eq(eval_logits.argmax(dim=-1), ys).float().mean() + m.mean.Ax = torch.eq(logits.argmax(dim=-1), ys).float().mean() m.last.lr = cur_lr return m From a8b6f4b800f1186983a625ddc48f090592e9b556 Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 14:27:31 +0800 Subject: [PATCH 62/95] example --- examples/imagenet.py | 4 ++-- examples/moco.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/imagenet.py b/examples/imagenet.py index 463ff14..31208f3 100644 --- a/examples/imagenet.py +++ b/examples/imagenet.py @@ -234,8 +234,8 @@ def main(): else: # not distributed or in distribution environment # create datamodule to contain dataloader ds, test_ds = make_dataset(dummy=params.dummy) - dl = ds.DataLoader(batch_size=params.batch_size) - test_dl = test_ds.DataLoader(batch_size=params.batch_size) + dl = ds.DataLoader(batch_size=params.batch_size, num_workers=4) + test_dl = test_ds.DataLoader(batch_size=params.batch_size, num_workers=4) dm = DataModule() dm.regist_dataloader(train=dl, test=test_dl) diff --git a/examples/moco.py b/examples/moco.py index f911367..169aff9 100644 --- a/examples/moco.py +++ b/examples/moco.py @@ -267,9 +267,9 @@ def main(): params.from_args() # create datamodule to contain dataloader - dl = ds.DataLoader(batch_size=params.batch_size) - memo_dl = memo_ds.DataLoader(batch_size=params.batch_size) - test_dl = test_ds.DataLoader(batch_size=params.batch_size) + dl = ds.DataLoader(batch_size=params.batch_size, num_workers=2) + memo_dl = memo_ds.DataLoader(batch_size=params.batch_size, num_workers=2) + test_dl = test_ds.DataLoader(batch_size=params.batch_size, num_workers=2) dm = DataModule() dm.regist_dataloader(train=dl, test=test_dl, From ddc59b570f8d815f7496955e344f13120bcec55f Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 15:08:28 +0800 Subject: [PATCH 63/95] test: Fix precision error --- tests/trainer/test_meter.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/trainer/test_meter.py b/tests/trainer/test_meter.py index edf8fae..2956eb0 100644 --- a/tests/trainer/test_meter.py +++ b/tests/trainer/test_meter.py @@ -9,11 +9,10 @@ def test_ReduceItem(): item.update(i) assert item.res == np.mean(range(200)[-ReduceItem.SLIDE_WINDOW_SIZE:], dtype=float).item() - item = ReduceItem([1, 2, 3], 'last') + item = ReduceItem(3, 'last') for i in range(4): - res = [j for j in range(i)] - item.update(res) - assert item.res == res + item.update(i) + assert item.res == i rand = [np.random.rand(4) for i in range(4)] avg = ReduceItem(rand[0], 'sum') @@ -63,4 +62,13 @@ def test_avgmeter(): def test_meter(): - pass + from lumo import Meter, Record + import numpy as np + rnd = np.random.rand(1000) + r = Record() + m = Meter() + m.mean.rnd = rnd[:500] + r.record(m) + m.mean.rnd = rnd[500:] + r.record(m) + assert np.isclose(r.agg()['rnd'], rnd.mean()) From 81993b17683fa227c62c390640824fb67c7621d8 Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 15:42:26 +0800 Subject: [PATCH 64/95] Make huggingface.accelerator an optional choice --- src/lumo/contrib/accelerate/__init__.py | 1 - src/lumo/contrib/accelerate/data_loader.py | 10 -- src/lumo/contrib/accelerate/utils.py | 30 ----- src/lumo/contrib/module/memoty_bank.py | 16 +-- src/lumo/exp/exphook.py | 2 - src/lumo/trainer/accelerator.py | 109 ++++++++++++++++++ .../augments => trainer/backend}/__init__.py | 0 .../backend}/accelerator.py | 2 +- src/lumo/trainer/trainer.py | 40 ++----- src/lumo/{contrib => utils}/data/__init__.py | 0 src/lumo/utils/data/augments/__init__.py | 0 .../{contrib => utils}/data/augments/mix.py | 0 .../data/augments/transforms.py | 0 src/lumo/{contrib => utils}/data/splits.py | 0 src/lumo/utils/device.py | 96 +++++++++++++++ tests/trainer/test_skip.py | 5 +- tests/trainer/test_trainer.py | 5 +- 17 files changed, 227 insertions(+), 89 deletions(-) delete mode 100644 src/lumo/contrib/accelerate/__init__.py delete mode 100644 src/lumo/contrib/accelerate/data_loader.py delete mode 100644 src/lumo/contrib/accelerate/utils.py create mode 100644 src/lumo/trainer/accelerator.py rename src/lumo/{contrib/data/augments => trainer/backend}/__init__.py (100%) rename src/lumo/{contrib/accelerate => trainer/backend}/accelerator.py (93%) rename src/lumo/{contrib => utils}/data/__init__.py (100%) create mode 100644 src/lumo/utils/data/augments/__init__.py rename src/lumo/{contrib => utils}/data/augments/mix.py (100%) rename src/lumo/{contrib => utils}/data/augments/transforms.py (100%) rename src/lumo/{contrib => utils}/data/splits.py (100%) create mode 100644 src/lumo/utils/device.py diff --git a/src/lumo/contrib/accelerate/__init__.py b/src/lumo/contrib/accelerate/__init__.py deleted file mode 100644 index 59d25ad..0000000 --- a/src/lumo/contrib/accelerate/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .accelerator import Accelerator \ No newline at end of file diff --git a/src/lumo/contrib/accelerate/data_loader.py b/src/lumo/contrib/accelerate/data_loader.py deleted file mode 100644 index f05ac57..0000000 --- a/src/lumo/contrib/accelerate/data_loader.py +++ /dev/null @@ -1,10 +0,0 @@ -from accelerate import synchronize_rng_states, DistributedType -from accelerate.data_loader import DataLoaderDispatcher as _DataLoaderDispatcher, DataLoaderShard as _DataLoaderShard -from accelerate.state import AcceleratorState -from accelerate.utils import send_to_device - -from lumo import LumoDataLoader - - -class DataLoaderDispatcher(_DataLoaderDispatcher): - pass diff --git a/src/lumo/contrib/accelerate/utils.py b/src/lumo/contrib/accelerate/utils.py deleted file mode 100644 index 44d29cc..0000000 --- a/src/lumo/contrib/accelerate/utils.py +++ /dev/null @@ -1,30 +0,0 @@ -from accelerate.utils import recursively_apply -import torch - - -def send_to_device(tensor, device, non_blocking=False): - """ - Recursively sends the elements in a nested list/tuple/dictionary of tensors to a given device. - - Args: - tensor (nested list/tuple/dictionary of `torch.Tensor`): - The data to send to a given device. - device (`torch.device`): - The device to send the data to. - - Returns: - The same data structure as `tensor` with all tensors sent to the proper device. - """ - if not isinstance(device, torch.device): - device = torch.device(device) - - if 'mps' in device.type: - non_blocking = False - - def _send_to_device(t, device): - return t.to(device, non_blocking=non_blocking) - - def _has_to_method(t): - return hasattr(t, "to") - - return recursively_apply(_send_to_device, tensor, device, test_type=_has_to_method) diff --git a/src/lumo/contrib/module/memoty_bank.py b/src/lumo/contrib/module/memoty_bank.py index d080c87..7c7477f 100644 --- a/src/lumo/contrib/module/memoty_bank.py +++ b/src/lumo/contrib/module/memoty_bank.py @@ -1,4 +1,4 @@ -from accelerate.utils import gather +from torch import distributed from lumo.proc.dist import is_dist import torch.distributed from torch import nn @@ -26,9 +26,9 @@ def __getitem__(self, item): @torch.no_grad() def scatter(self, name, value, index): value = value.detach() - value = gather(value) + value = distributed.gather(value) if isinstance(index, torch.Tensor): - index = gather(index) + index = distributed.gather(index) self[name][index] = value @@ -61,7 +61,7 @@ def push(self, name, value): raise AssertionError() value = value.detach() - value = gather(value) + value = distributed.gather(value) ptr = self.offsets[name] k = self.sizes[name] batch_size = value.shape[0] @@ -80,9 +80,9 @@ def batch_shuffle_ddp(x): """ if not is_dist(): return x, torch.arange(len(x)) - # gather from all gpus + # distributed.gather from all gpus batch_size_this = x.shape[0] - x_gather = gather(x) + x_gather = distributed.gather(x) batch_size_all = x_gather.shape[0] num_gpus = batch_size_all // batch_size_this @@ -111,9 +111,9 @@ def batch_unshuffle_ddp(x, idx_unshuffle): """ if not is_dist(): return x - # gather from all gpus + # distributed.gather from all gpus batch_size_this = x.shape[0] - x_gather = gather(x) + x_gather = distributed.gather(x) batch_size_all = x_gather.shape[0] num_gpus = batch_size_all // batch_size_this diff --git a/src/lumo/exp/exphook.py b/src/lumo/exp/exphook.py index 90f2c0b..f0c25d0 100644 --- a/src/lumo/exp/exphook.py +++ b/src/lumo/exp/exphook.py @@ -198,11 +198,9 @@ def on_start(self, exp: Experiment, *args, **kwargs): 'joblib', 'fire', 'psutil', - 'accelerate', 'hydra', 'omegaconf', 'decorator', - 'numpy', 'torch', ) diff --git a/src/lumo/trainer/accelerator.py b/src/lumo/trainer/accelerator.py new file mode 100644 index 0000000..ab28ca3 --- /dev/null +++ b/src/lumo/trainer/accelerator.py @@ -0,0 +1,109 @@ +import warnings +from torch import nn +import torch +from torch import distributed +from torch.utils.data import DataLoader +from lumo.data.loader import DataLoaderSide + + +class Accelerator: + def __init__(self, **kwargs): + self._prop = kwargs + + @property + def device(self): + return self._prop.get('device', None) + + def set_device(self, device): + assert isinstance(device, torch.device) + self._prop['device'] = device + + def prepare_data_loader(self, dataloader): + return dataloader + + def prepare_model(self, model: torch.nn.Module): + return model.to(self.device) + + def prepare_optimizer(self, optimizer: torch.optim.Optimizer): + return optimizer + + def unwrap_model(self, model): + return model + + def prepare(self, *args): + res = [] + for item in args: + if isinstance(item, nn.Module): + res.append(self.prepare_model(item)) + elif isinstance(item, (DataLoader, DataLoaderSide)): + res.append(self.prepare_data_loader(item)) + elif isinstance(item, torch.optim.Optimizer): + res.append(self.prepare_optimizer(item)) + else: + raise NotImplementedError() + return res + + def wait_for_everyone(self): + torch.distributed.barrier() + + def gather(self, tensor: torch.Tensor): + if tensor.ndim == 0: + tensor = tensor.clone()[None] + output_tensors = [tensor.clone() for _ in range(torch.distributed.get_world_size())] + distributed.all_gather(output_tensors, tensor) + return torch.cat(output_tensors, dim=0) + + +class HugAccelerator(Accelerator): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + from .backend.accelerator import Accelerator + self._backbone = Accelerator() + + @property + def device(self): + return self._backbone.device + + def set_device(self, device: torch.device): + assert isinstance(device, torch.device) + self._backbone.state.device = device + + def prepare_data_loader(self, loader): + from accelerate.data_loader import DataLoaderShard, DataLoaderDispatcher + if isinstance(loader, (DataLoaderShard, DataLoaderDispatcher)): + warnings.warn('Duplicated prepare a same DataLoader twice, check your code.') + return loader + return self._backbone.prepare_data_loader(loader) + + def prepare_model(self, model): + return self._backbone.prepare_model(model) + + def prepare_optimizer(self, optimizer): + return self._backbone.prepare_optimizer(optimizer) + + def unwrap_model(self, model): + return self._backbone.unwrap_model(model) + + def prepare(self, *args): + return self._backbone.prepare(*args) + + def wait_for_everyone(self): + self._backbone.wait_for_everyone() + + def gather(self, tensor): + return self._backbone.gather(tensor) + + +register = { + + 'none': Accelerator, + 'accelerator': HugAccelerator, + 'deepspeed': None, + 'horovod': None, +} + + +def get_accelerator(name: str, **kwargs): + assert name not in register, ', '.join(register.keys()) + return register[name](**kwargs) diff --git a/src/lumo/contrib/data/augments/__init__.py b/src/lumo/trainer/backend/__init__.py similarity index 100% rename from src/lumo/contrib/data/augments/__init__.py rename to src/lumo/trainer/backend/__init__.py diff --git a/src/lumo/contrib/accelerate/accelerator.py b/src/lumo/trainer/backend/accelerator.py similarity index 93% rename from src/lumo/contrib/accelerate/accelerator.py rename to src/lumo/trainer/backend/accelerator.py index 79d4d3b..488e539 100644 --- a/src/lumo/contrib/accelerate/accelerator.py +++ b/src/lumo/trainer/backend/accelerator.py @@ -10,7 +10,7 @@ class Accelerator(_Accelerator): the device of data will be controlled by Trainer rather than Accelerator. """ - def prepare_data_loader(self, data_loader): + def prepare_data_loader(self, data_loader, **kwargs): return prepare_data_loader( data_loader, None, # None instead of self.device, diff --git a/src/lumo/trainer/trainer.py b/src/lumo/trainer/trainer.py index b714be0..6489f8a 100644 --- a/src/lumo/trainer/trainer.py +++ b/src/lumo/trainer/trainer.py @@ -1,22 +1,16 @@ import bisect import os -import sys import warnings -from datetime import datetime from functools import lru_cache -from pprint import pformat from typing import Union, Dict, Any, Optional, Sequence, Mapping, Callable import numpy as np import torch -from accelerate import DistributedDataParallelKwargs -from accelerate.data_loader import DataLoaderDispatcher, DataLoaderShard +from .accelerator import get_accelerator from torch import nn from torch.optim import Optimizer from torch.utils.data import DataLoader -import json -from lumo.contrib.accelerate import Accelerator -from lumo.contrib.accelerate.utils import send_to_device +from lumo.utils.device import send_to_device from lumo.core import TrainStage, Record, MetricType, Meter from lumo.core.disk import TableRow, Metrics from lumo.data import DataModule @@ -30,11 +24,6 @@ from .components import TrainerExperiment, TrainerParams from .saver import Saver -# overwrite send_to_device to resolve https://github.com/pytorch/pytorch/issues/83015 -# from accelerate import Accelerator -# from accelerate.utils import send_to_device -from ..utils.fmt import strftime, indent_print - ParamsType = TrainerParams @@ -74,7 +63,7 @@ def __init_subclass__(cls, **kwargs): raise TypeError( f"Can't instantiate abstract class {cls.__name__} directly, please create a subclass of it.") - def __init__(self, params: ParamsType, dm: DataModule = None): + def __init__(self, params: ParamsType, dm: DataModule = None, accelerator='accelerator'): if dm is None: dm = DataModule(params) else: @@ -100,13 +89,12 @@ def __init__(self, params: ParamsType, dm: DataModule = None): self.train_toggle = False device = params.get('device', None) if not self.is_dist else None + # self.accelerate = Accelerator(kwargs_handlers=[ + # DistributedDataParallelKwargs(find_unused_parameters=params.get('find_unused_parameters', False)) + # ]) + self.accelerate = get_accelerator(accelerator) - self.accelerate = Accelerator(kwargs_handlers=[ - DistributedDataParallelKwargs(find_unused_parameters=params.get('find_unused_parameters', False)) - ]) - - if self.accelerate.state.distributed_type == self.accelerate.state.distributed_type.NO: - self.accelerate.state.device = torch.device(device) + self.accelerate.set_device(torch.device(device)) if dist.is_main(): self.params.to_yaml(self.exp.params_fn) @@ -405,7 +393,7 @@ def device(self): Returns: The device used for training. """ - return self.accelerate.device + return torch.device(self.params.device) def _load_fun_state_dict(self, src: dict): """ @@ -577,17 +565,7 @@ def prepare_dataloader(self, loader: DataLoaderType, stage: TrainStage = None): :param stage: :return: """ - if isinstance(loader, (DataLoaderShard, DataLoaderDispatcher)): - warnings.warn('Duplicated prepare a same DataLoader twice, check your code.') - return loader - - split_batches = self.params.get('split_batches', None) - if stage is not None and not stage.is_train(): - split_batches = True - - """do not change original loader stage""" if isinstance(loader, DataLoader): - self.accelerate.split_batches = split_batches loader = self.accelerate.prepare_data_loader(loader) elif isinstance(loader, DataLoaderSide): loader = loader.copy() diff --git a/src/lumo/contrib/data/__init__.py b/src/lumo/utils/data/__init__.py similarity index 100% rename from src/lumo/contrib/data/__init__.py rename to src/lumo/utils/data/__init__.py diff --git a/src/lumo/utils/data/augments/__init__.py b/src/lumo/utils/data/augments/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/lumo/contrib/data/augments/mix.py b/src/lumo/utils/data/augments/mix.py similarity index 100% rename from src/lumo/contrib/data/augments/mix.py rename to src/lumo/utils/data/augments/mix.py diff --git a/src/lumo/contrib/data/augments/transforms.py b/src/lumo/utils/data/augments/transforms.py similarity index 100% rename from src/lumo/contrib/data/augments/transforms.py rename to src/lumo/utils/data/augments/transforms.py diff --git a/src/lumo/contrib/data/splits.py b/src/lumo/utils/data/splits.py similarity index 100% rename from src/lumo/contrib/data/splits.py rename to src/lumo/utils/data/splits.py diff --git a/src/lumo/utils/device.py b/src/lumo/utils/device.py new file mode 100644 index 0000000..2e49881 --- /dev/null +++ b/src/lumo/utils/device.py @@ -0,0 +1,96 @@ +from typing import Any, Mapping +import torch + + +def honor_type(obj, generator): + """ + Cast a generator to the same type as obj (list, tuple, or namedtuple) + """ + try: + return type(obj)(generator) + except TypeError: + # Some objects may not be able to instantiate from a generator directly + return type(obj)(*list(generator)) + + +def is_torch_tensor(tensor): + return isinstance(tensor, torch.Tensor) + + +def recursively_apply(func, data, *args, test_type=is_torch_tensor, error_on_other_type=False, **kwargs): + """ + Recursively apply a function on a data structure that is a nested list/tuple/dictionary of a given base type. + + Args: + func (`callable`): + The function to recursively apply. + data (nested list/tuple/dictionary of `main_type`): + The data on which to apply `func` + *args: + Positional arguments that will be passed to `func` when applied on the unpacked data. + main_type (`type`, *optional*, defaults to `torch.Tensor`): + The base type of the objects to which apply `func`. + error_on_other_type (`bool`, *optional*, defaults to `False`): + Whether to return an error or not if after unpacking `data`, we get on an object that is not of type + `main_type`. If `False`, the function will leave objects of types different than `main_type` unchanged. + **kwargs: + Keyword arguments that will be passed to `func` when applied on the unpacked data. + + Returns: + The same data structure as `data` with `func` applied to every object of type `main_type`. + """ + if isinstance(data, (tuple, list)): + return honor_type( + data, + ( + recursively_apply( + func, o, *args, test_type=test_type, error_on_other_type=error_on_other_type, **kwargs + ) + for o in data + ), + ) + elif isinstance(data, Mapping): + return type(data)( + { + k: recursively_apply( + func, v, *args, test_type=test_type, error_on_other_type=error_on_other_type, **kwargs + ) + for k, v in data.items() + } + ) + elif test_type(data): + return func(data, *args, **kwargs) + elif error_on_other_type: + raise TypeError( + f"Can't apply {func.__name__} on object of type {type(data)}, only of nested list/tuple/dicts of objects " + f"that satisfy {test_type.__name__}." + ) + return data + + +def send_to_device(tensor, device, non_blocking=False): + """ + Recursively sends the elements in a nested list/tuple/dictionary of tensors to a given device. + + Args: + tensor (nested list/tuple/dictionary of `torch.Tensor`): + The data to send to a given device. + device (`torch.device`): + The device to send the data to. + + Returns: + The same data structure as `tensor` with all tensors sent to the proper device. + """ + if not isinstance(device, torch.device): + device = torch.device(device) + + if 'mps' in device.type: + non_blocking = False + + def _send_to_device(t, device): + return t.to(device, non_blocking=non_blocking) + + def _has_to_method(t): + return hasattr(t, "to") + + return recursively_apply(_send_to_device, tensor, device, test_type=_has_to_method) diff --git a/tests/trainer/test_skip.py b/tests/trainer/test_skip.py index 966af2a..2fa2c27 100644 --- a/tests/trainer/test_skip.py +++ b/tests/trainer/test_skip.py @@ -57,9 +57,8 @@ def to_device(self, item: Optional[Union[nn.Module, torch.Tensor, Sequence, Mapp device_args_kwargs=None): return super().to_device(item, device_args_kwargs) - def prepare_dataloader(self, stage: TrainStage, dataloader=None): - # assert self.context == 'prepare_dataloader' - return super().prepare_dataloader(stage, dataloader) + def prepare_dataloader(self, loader: DataLoaderType, stage: TrainStage = None): + return super().prepare_dataloader(loader, stage) def initialize(self): if not self.is_initialized: diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 38c015c..a75fa94 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -72,9 +72,8 @@ def to_device(self, item: Optional[Union[nn.Module, torch.Tensor, Sequence, Mapp device_args_kwargs=None): return super().to_device(item, device_args_kwargs) - def prepare_dataloader(self, stage: TrainStage, dataloader=None): - # assert self.context == 'prepare_dataloader' - return super().prepare_dataloader(stage, dataloader) + def prepare_dataloader(self, loader: DataLoaderType, stage: TrainStage = None): + return super().prepare_dataloader(loader, stage) def initialize(self): if not self.is_initialized: From 492aaf4bde7d6e939f5cbc1687dd8c66d5d48490 Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 15:45:55 +0800 Subject: [PATCH 65/95] Make huggingface.accelerator an optional choice --- src/lumo/trainer/accelerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo/trainer/accelerator.py b/src/lumo/trainer/accelerator.py index ab28ca3..12eb0ea 100644 --- a/src/lumo/trainer/accelerator.py +++ b/src/lumo/trainer/accelerator.py @@ -105,5 +105,5 @@ def gather(self, tensor): def get_accelerator(name: str, **kwargs): - assert name not in register, ', '.join(register.keys()) + assert name in register, ', '.join(register.keys()) return register[name](**kwargs) From b5be9cc41541b06208b0bd966c76e016f3de15a8 Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 15:48:52 +0800 Subject: [PATCH 66/95] Make huggingface.accelerator an optional choice --- src/lumo/trainer/accelerator.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lumo/trainer/accelerator.py b/src/lumo/trainer/accelerator.py index 12eb0ea..2458933 100644 --- a/src/lumo/trainer/accelerator.py +++ b/src/lumo/trainer/accelerator.py @@ -53,6 +53,9 @@ def gather(self, tensor: torch.Tensor): distributed.all_gather(output_tensors, tensor) return torch.cat(output_tensors, dim=0) + def backward(self, loss: torch.Tensor, **kwargs): + loss.backward(**kwargs) + class HugAccelerator(Accelerator): @@ -94,6 +97,9 @@ def wait_for_everyone(self): def gather(self, tensor): return self._backbone.gather(tensor) + def backward(self, loss: torch.Tensor, **kwargs): + self._backbone.backward(loss, **kwargs) + register = { @@ -104,6 +110,6 @@ def gather(self, tensor): } -def get_accelerator(name: str, **kwargs): +def get_accelerator(name: str, **kwargs) -> Accelerator: assert name in register, ', '.join(register.keys()) return register[name](**kwargs) From c25cd01b80378cae148b98ffbe08fe99bb71b9ef Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 16:36:07 +0800 Subject: [PATCH 67/95] Gather method --- src/lumo/contrib/module/memoty_bank.py | 17 +++++++++-------- src/lumo/proc/dist.py | 10 ++++++++++ src/lumo/trainer/accelerator.py | 7 ++----- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/lumo/contrib/module/memoty_bank.py b/src/lumo/contrib/module/memoty_bank.py index 7c7477f..a73dfe5 100644 --- a/src/lumo/contrib/module/memoty_bank.py +++ b/src/lumo/contrib/module/memoty_bank.py @@ -1,6 +1,7 @@ from torch import distributed -from lumo.proc.dist import is_dist +from lumo.proc.dist import is_dist, gather import torch.distributed + from torch import nn import torch @@ -26,9 +27,9 @@ def __getitem__(self, item): @torch.no_grad() def scatter(self, name, value, index): value = value.detach() - value = distributed.gather(value) + value = gather(value) if isinstance(index, torch.Tensor): - index = distributed.gather(index) + index = gather(index) self[name][index] = value @@ -61,7 +62,7 @@ def push(self, name, value): raise AssertionError() value = value.detach() - value = distributed.gather(value) + value = gather(value) ptr = self.offsets[name] k = self.sizes[name] batch_size = value.shape[0] @@ -80,9 +81,9 @@ def batch_shuffle_ddp(x): """ if not is_dist(): return x, torch.arange(len(x)) - # distributed.gather from all gpus + # gather from all gpus batch_size_this = x.shape[0] - x_gather = distributed.gather(x) + x_gather = gather(x) batch_size_all = x_gather.shape[0] num_gpus = batch_size_all // batch_size_this @@ -111,9 +112,9 @@ def batch_unshuffle_ddp(x, idx_unshuffle): """ if not is_dist(): return x - # distributed.gather from all gpus + # gather from all gpus batch_size_this = x.shape[0] - x_gather = distributed.gather(x) + x_gather = gather(x) batch_size_all = x_gather.shape[0] num_gpus = batch_size_all // batch_size_this diff --git a/src/lumo/proc/dist.py b/src/lumo/proc/dist.py index f960bff..ad48d14 100644 --- a/src/lumo/proc/dist.py +++ b/src/lumo/proc/dist.py @@ -75,3 +75,13 @@ def is_main(): """ return local_rank() <= 0 + + +def gather(tensor): + if dist.is_initialized(): + if tensor.ndim == 0: + tensor = tensor.clone()[None] + output_tensors = [tensor.clone() for _ in range(torch.distributed.get_world_size())] + torch.distributed.all_gather(output_tensors, tensor) + return torch.cat(output_tensors, dim=0) + return tensor diff --git a/src/lumo/trainer/accelerator.py b/src/lumo/trainer/accelerator.py index 2458933..c7e873f 100644 --- a/src/lumo/trainer/accelerator.py +++ b/src/lumo/trainer/accelerator.py @@ -4,6 +4,7 @@ from torch import distributed from torch.utils.data import DataLoader from lumo.data.loader import DataLoaderSide +from lumo.proc.dist import gather class Accelerator: @@ -47,11 +48,7 @@ def wait_for_everyone(self): torch.distributed.barrier() def gather(self, tensor: torch.Tensor): - if tensor.ndim == 0: - tensor = tensor.clone()[None] - output_tensors = [tensor.clone() for _ in range(torch.distributed.get_world_size())] - distributed.all_gather(output_tensors, tensor) - return torch.cat(output_tensors, dim=0) + return gather(tensor) def backward(self, loss: torch.Tensor, **kwargs): loss.backward(**kwargs) From 1fe265cbe0e39f54ae223ad60a0070688e9e20ed Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 16:45:47 +0800 Subject: [PATCH 68/95] Document --- README.ch.md | 229 ++++++++++++++++++++++----------------------------- 1 file changed, 98 insertions(+), 131 deletions(-) diff --git a/README.ch.md b/README.ch.md index afbb779..209904b 100644 --- a/README.ch.md +++ b/README.ch.md @@ -2,184 +2,151 @@ [![PyPI version](https://badge.fury.io/py/lumo.svg)](https://badge.fury.io/py/lumo) ![Python-Test](https://github.com/pytorch-lumo/lumo/actions/workflows/python-test.yml/badge.svg) -[![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/pytorch-lumo/lumo/blob/master/LICENSE) +[![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/Lightning-AI/lightning/blob/master/LICENSE) +![Python-doc](./images/docstr_coverage_badge.svg) +`lumo` 是一个精简高效的库,简化了实验所需的所有组件的管理,并特别关注增强深度学习实践者的体验。 -`lumo`:轻量、可扩展、功能解耦合的 Pytorch 实验框架。 +- 实验管理:: 为每次运行分配唯一路径,区分不同类型的文件并存储;并通过 git 管理代码快照。 +- 参数管理:基于 fire 提供比 argparser 更便捷的参数管理 +- 运行时配置:提供多级作用域下的配置管理 +- 可视化:基于 [Panel](https://panel.holoviz.org/index.html) 提供可交互的 jupyter 实验管理面板 +- 为深度学习提供额外的优化 + - 训练:基于 Trainer 提供可任意扩展的训练逻辑简化,并提供完善的回调逻辑 + - 优化器:参数与优化器构建一体化 + - 数据: 数据集构建流程抽象、组合多个 DataLoader、... + - 分布式训练:同样支持多种训练加速框架,统一抽象,方便随时切换 +- 更多工具类... -lumo 的设计理念: +# :book: 目录 -- 模块解耦合:所有模块可以单独作为您现在使用的框架中的一个插件使用(而不像其他框架几乎耦合在一起) -- 恰到好处的抽象:和模型相关的细节完全由使用者掌控,lumo 只封装了外部通用逻辑(而不像其他一些框架会代理模型初始化或损失迭代) -- 覆盖整个生命周期:数据集构建、模型初始化、随机种子、训练/测试...,lumo 为所有步骤提供了功能包或流程简化 -- 极强的可扩展性:从单文件到包含多个领域多个方法的项目,lumo 都可以提供舒适的使用体验。已在两个领域有复现项目的最佳实践示例(见[Related Work](#Related Work))。 +- [安装](#安装) +- [快速开始](#快速开始) -# 如何使用 +# :cloud: 安装 -## 安装 - -从 pypi 或 github 主页安装最新的稳定版本: +安装已发布的通过了所有测试的版本 ```bash pip install -U lumo -pip install git+https://github.com/pytorch-lumo/lumo ``` +或从 dev1 分支安装最新版本: -## 快速开始 - -本节包含 lumo 最常用的几个子功能,帮助使用者快速利用这些功能减少已有项目中的冗余代码,这些功能包括: - -- 一些常用功能的更优替代,如 [Params](#参数控制)(arguments 的平替),[Logger](#变量&日志记录)(logging 的平替) -- 一些训练过程中部份流程的优化,如 [Experiment](#路径管理&版本控制)(提供无重复的实验路径管理、基于 git 的版本控制),[DatasetBuilder](#数据集构建)(更快构建数据集), - -### 参数控制 - -`argparse` 的更优替代。`Params` 底层依托于 [omegaconf](https://github.com/omry/omegaconf) 和 [fire](https://github.com/google/python-fire) -,只需要简单的配置,就可以从文件、命令行中读取参数。 - -直接基于 Params 类定义参数: - -```python -# python main.py --epoch=30 --dataset=100 -from lumo import Params +```bash +pip install git+https://github.com/pytorch-lumo/lumo +``` -params = Params() -params.epoch = 20 -# 集成优化器参数,自带补全提示 -params.optim = params.OPTIM.create_optim('Adam', lr=0.0001, weight_decay=4e-5) -# 数据集只能从 cifar10/cifar100 中选择,且默认为 cifar10,其他的选择会报错 -params.dataset = params.choice('cifar10', 'cifar100') +实验面板依赖于 panel,需要额外安装: -# 从命令行参数中更新 -params.from_args() -print(params.epoch) # -> 30 -print(params.dataset) # -> cifar100 - -# 保存到文件 -params.to_json('./config.json') -params.to_yaml('./config.yaml') -# 从文件中更新 -params.from_json('./config.json') -params.from_yaml('./config.yaml') +``` +pip install panel ``` -也可以通过继承、多重继承来嵌套,组合参数。即使在命令行中输入了不存在的参数,Params 也会正常读取。 +# :book: 快速开始 -### 变量&日志记录 +以下是两个经典场景: -`logging` 的更优替代。通过 Meter、Record 和 Logger,可以实现变量的记录和格式化输出。其中: +## :small_orange_diamond: 已有项目嵌入 -- Meter 记录单次的值 -- Record 以一定规则归约 Meter 实例(如 mean、sum 等) -- Logger 用于代替 logging,除常用的 info、warn 等方法外,还提供了 inline 方法,可以在屏幕能单行更新(实际中,屏幕打印时间远小于训练时间,因此单行更新带来的时间开销可以忽略不计)。 +对已有项目,可以通过以下方式快速嵌入 ```python import random -import time +from lumo import SimpleExperiment, Params, Logger, Meter, Record -from lumo import Record, Meter, Logger +logger = Logger() +# 定义及使用,无需转换 +exp = SimpleExperiment(exp_name='my_exp_a') # 为每种实验手动定义唯一名称 +exp.start() +logger.add_log_dir(exp.mk_ipath()) -log = Logger() +# 替换基于 argparse 等的参数定义方法 +params = Params() +params.dataset = params.choice('cifar10', 'cifar100') +params.alpha = params.arange(default=1, left=0, right=10) +params.from_args() # python3 train.py --dataset=cifar100 --alpha=0.2 +print(params.to_dict()) # {"dataset": "cifar100", "alpha": 0.2} -record = Record() -for idx in range(256): - meter = Meter() - meter.last.i = idx - meter.sum.acc = idx - meter.mean.loss = random.random() - - record.record(meter) - log.inline(record) # 单行更新 - time.sleep(0.5) - if idx % 50 == 0: - log.newline() - record.clear() - -log.info(record) -``` +# 记录实验参数 +exp.dump_info('params', params.to_dict()) +print(exp.test_name) # 为每次实验自动分配唯一名称 -### 路径管理&版本控制 +# 基于命名空间提供本次实验的唯一路径 +# 元数据和二进制大文件分离,方便清理 +params.to_yaml(exp.mk_ipath('params.yaml')) -`Experiment` 主要提供路径管理,可以为每一次试验根据实验名、日期、次数等自动提供不一样的保存路径。此外,Experiment 还可以通过 hook -提供如代码版本管理、元数据记录等功能。在实验中,可以使用其子类 `SimpleExperiment` 实现大部分需求。 +for i in range(10): + # 记录实验指标 + max_acc = exp.dump_metric('Acc', random.random(), cmp='max') + logger.info(f'Max acc {max_acc}') -```python -from lumo import SimpleExperiment -from lumo import Params + # 存储大文件/二进制文件(如模型权重) + ckpt_fn = exp.mk_bpath('checkpoints', f'model_{i}.ckpt') + ... # 保存代码 given ckpt_fn -pm = Params() -pm.module = 'example' -pm.from_args() +record = Record() +for batch in range(10): + m = Meter() + m.mean.Lall = random.random() + m.last.lr = batch + record.record(m) + logger.info(record) + +# 主动结束实验,补充元信息。也可以在进程结束后由 hook 自动结束,支持针对异常的记录 +exp.end() +``` -# 注册该次实验,实验名为 `pm.module` -exp = SimpleExperiment(pm.module) -# 实验开始,该方法会调用已注册的 ExpHooks,完成代码版本控制等功能。 -exp.start() +## :small_orange_diamond: 从零开始 -# 小数据通过 `.test_file()` 获得路径 -fn = exp.test_file('params.json') -pm.to_json(fn) +如果从新开始一个深度学习实验,那么可以使用 lumo 全方位的加速代码的构建,下面提供了多个不同规模下使用 lumo 训练的示例: -# 大文件通过 `.blob_file()` 获得路径(这是约定,而不是强制,大文件也可以保存到 `.test_file()` 中) -fn = exp.blob_file('checkpoint.pt') -with open(fn, 'w') as w: - w.write('write big data in blob file') +单文件: -print(exp.test_root) -print(exp.get_prop('git')) # see git commit history -exp.end() -``` +| 示例 | CoLab | 代码行数 | +|----------------------------------------|-------|------| +| [MNIST 示例](./examples/mnist.py) | | 118 | +| [MocoV2 训练 CIFAR10](./examples/moco.py) || 284 | +| [多卡训练 ImageNet]() ||| -### 数据集构建 +实验项目: -![DatasetBuilder](./images/DatasetBuilder.png) +| 项目 | 说明 | +|------------------------|--------------------------| +| [image-classification] | 集成了全监督、半监督、自监督的多个论文的复现代码 | +| [emotion-recognition] | 集成了情感分类、多模态情感分类的多个论文的复现代码 | -`DatasetBuilder` 是采用有向无环图思路设计的数据集构建类,该类提供了一个恰当的抽象逻辑,避免了在一个实验里定义多个重复 Datasets 类。 +## :small_orange_diamond: 可视化界面 -`DatasetBuilder `将数据集的构件划分为输入-输出两阶段,同时提供 `.chain()`(序列格式)和`.zip()`(字典格式) 两种输出方式。 +在 jupyter 中: ```python -from lumo import DatasetBuilder -from torchvision.transforms import transforms -import torch - -# Create a mnist-like dummy dataset -db = ( - DatasetBuilder() - .add_input("xs", torch.rand(500, 28, 28)) - .add_input("ys", torch.randint(0, 10, (500,))) - .add_idx('id') - .add_output("xs", "xs1", transforms.RandomHorizontalFlip()) - .add_output("xs", "xs2", ) - .add_output("ys", "ys") -) -# Watch dataset structure -print(db) -# Builder(flow={'::idx::': ['id'], 'xs': ['xs1', 'xs2'], 'ys': ['ys']}, sized=True, size=500, iterable=True) - -print(db[0]) -# dict_keys(['id', 'xs1', 'xs2', 'ys']) -``` +from lumo import Watcher -# 更多教程 +w = Watcher() +df = w.load() +widget = w.panel(df) +widget.servable() +``` -# Related Work +![Panel](./images/panel-example.png) -- [image-classification](https://github.com/pytorch-lumo/image-classification): supervised/semi-supervised/self-supervised/noisy label learning on image-classfication - field. (suporrted datasets: CIFAR10/CIFAR100/STL-10/SVHN/ImageNet/tinyimagenet) -- [emotion-recognition-in-conversation](https://github.com/pytorch-lumo/emotion-recognition-in-conversation):Multimodel emotional recognition on conversation. (suporrted datasets: IEMOCAP/MELD/MOSEI) +将按手动条件过滤后的实验筛选出来: +![Panel](./images/panel-example2.png) +可以直接使用命令行打开页面查看当前所有实验: -# Acknowledge +``` +lumo board [--port, --address, --open] +``` - 一个人维护一个库四年,背后的动力是我持续不断的使用,感谢 lumo 陪我见证我的学术生涯。lumo 确实不一定适合所有人的习惯,但一定最适合我自己。lumo 取自 lumos,这是哈利波特里魔法杖发光的咒语。torch 是火炬,ignite 是点燃,所以 lumo 也向往着发光发热,希望 lumo 给大家带来美好的使用体验。 +# More -# License +# :pencil: Acknowledge -Distributed under the GNU General Public License 3.0. See [LICENSE](./LICENSE) for more information. +从 2020 年维护至今。感谢 lumo 陪我见证我的学术生涯。 -# Contact +# :scroll: License - - [sailist@outlook.com](mailto:sailist@outlook.com) +采用 [GNU General Public License 3.0 协议](./LICENSE)。 From 05ac71bac84c5d7f17197c7b03d44587bbac551f Mon Sep 17 00:00:00 2001 From: sailist Date: Thu, 16 Mar 2023 16:45:49 +0800 Subject: [PATCH 69/95] Document --- images/panel-example2.png | Bin 0 -> 234161 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 images/panel-example2.png diff --git a/images/panel-example2.png b/images/panel-example2.png new file mode 100644 index 0000000000000000000000000000000000000000..c0116eb93be26ae1cf286a51b70957757de867ab GIT binary patch literal 234161 zcmb5W1ymL57dL$9kWT3kq$H%f4$|F?Af3`(BHi5}-3RFg6_5t$2I-LQ<{j?+*LuJA zu1{yJ!wfU?)ZY8q@!R{HP(^tORAfSA5D0`SB`Ky10wIxuKrp0;@W35}O77Pn5Q?w` z7_2A-29ql~*qK^bn}9%)p@|v@nks#GSvqks!sbx0`5%{HV+n<&5gE^--oBuM5)FNg zp}Nr9JPePr2pca4jb<;hYvV+PRjwTpw;nQXeyTTsN2a zK*c0mA!0nmyr8CUUiQ|Xg}ikgkTcJ4J_(cj@Oz=S_5M_BaC`ebXh8Ay^5lS5IDa44 zvt&x>_R(te=kaMHhA;gi_1{@5QI{1tQqfcTN2e zhpkf0j=czw|GvyV9>l?u5n=c$JG`704)H{cCxhV8jQ7E{ZzvY!`twIMY87HEhD^4> z`Jp{JL*;U4`)n3QR%9c&5Ftc2R?iS6=|U@^^QGLd?7EfL%Y^_f&8SzmQ*C@DkuaPBiXy%)&`g%;HCKf~FGV!WrJIjm# zBpNqAJirL|qM*qgeA>-viaZmG+{z`yCMcFgR-QQ!+l{{UL2vb0DiWCYuGM}8?6`6a z1HIwclXV=3WRIMQp6m6fd99g2*&nJtvsQSQ@s`dGxk^S~B*W-cRth?PzAl*p{)R-) zz*^U{8t$CY6St!ca^lazpieMJZU3xe5<)bR3ISeMWn0;uL3pBPGf3!zQuhL35GMxg`L$sATlz%Llj z1-Ki-)%G7X48~evjRjgckNNiv1b0X5I4;H#tkR?+7cX=k`IRTtC!IClr2Qg0eG$!J z!@XR-l9hd5^aDDVdX;TDznD)i$X9f$PC?qdKH={ zScVz)DoHVbPY(S#6x&jJhd`AQ#7+9ij%tJ_xA~eZeLa@IH zd&LM-5r@8si&dq9!KB18#;XR=f%{|Wjj@%3_e9VrX<{5D@g1JI`<+o%#wySTm44DL zrmzf+5+|U=Mg^~phML3k;xdYbj~19yYlk0*rT@U*hC3sii)(jq#G;@2yA-W2utIso7;^a*!uz^j zt;()qt@0YZIOeVjzY5_Nri?w4bfL0S#t8-Y(Aoub2M><%IPF&w=5ZvMKm3HCLo6*G*-WPAlO$ zL0TGS180M2BeJ@>x}-YTA);E?J~ zUAU$9uCcR>a}uwFca%rnjmSCCE({@ih71Oe_*D_Q!T7cBiK%Tt?+5vLJdD-eN5}2;)@~ zhQHRgUyaX}IqGeVBX(jAV%}kfORz^Y@~^fXNa_jcwcD21<{fY>DtI`x3Ab6d;d%^t zT%Or`>)b!Tx48LruCQ|%IYGF?5yw~Uw7wMEfT{?i6QCK83ZIV3ivo`li4uVNCddj^ zf$)N0=lL~(2|)!bujPc5EZfCvyyumyf~I#F5IM6k$L+*37mtomf2okrF5`HqqCPVf zhH=<&%5m{=Pg5MWAg$(E`Wns}AIEp6h^Jbc=bH_Ty?7|$vf&(fX0MiXw6$JpG1Xev zdzltkb=T9^mNqn7$<e+3SZ!-^`Ynnyco0@fw zq>MOuPED-Fw;R+JCNSRqrG4qaKOhR+2#y8UbPkMe=BbERKKC{~CNkQG&i+`E)p(y zEDkNIELtttvD7{vji8RGA%#Wvjcy_h77+AkcHi_oIKLYhSYnvB3ZAa1Qh#^-t|7vE z-DY>_cxiocAABiwKD#}>Z?!sbTygHbZMNBSI=_3FV9_~fbYR?m=6Z&IOLpWta$-DW zN`n3c`U8wK3@;1^3^sHb^dUkjyb0_i!Z|_<{0NF5sVE}SGc8Pq0Gr;;>R&VWIR`m4 z;Q3C~Z>!%*zmW#b2ZeX)y!a9FBP0>i^#vnkVC;|*c^F9@>#lyS+S$Z5_;pvxcFJxF zP1QRiv+k_Wyl%OudCW_uB_SWJyG+<$uu`Hrx!S8AyPCT$rJ$rXq!!o2qtgWrM5}$O z$(yJH6XH^;IZWPq{k)o7^L*JTPn-JQJ9n~c2YWj;p%@6cn z%gSTQQMC?h@~Rt5&^M_z*f;4n3O7k{ERkyh_f;YaWwXbqtFaS9nu|Q-^#tk+h+i(H z1+U8uTJ=qT5IP~jvg%`wLx2;6n}j1Pq?VuO(tFqZ-lGR)o*>+epAm)r&gP|HJYS7X zQeXLX4e6li@De*hYCLC=4PqGxS*wwfQAv0m-!?zOt@ui6!~j-8rBnc$ijB)w10cyu1DOzc2ok~5fV z8nl_c+34E%wlS@71)0%@DCuiHn45I<7pk496_hPOvRZF+17*5{I1)JGI2IFE5*e+A zrkM)yG9Fd(Z^3sH=m6lFbYGc#e z(|E>Eo#DIhR~5Ep4^!0@1`S9J&UVp@OZr>-8V#Lqd#XKZ^_7=f9KQ|Av_QMmvCnW> zTTWQKnn!h!aS>XMZ{E6D`rKx7`}~&hLT9g%N0?7>?j!q<`L5lt^?|LF?GEpAkIxm8 zW#?Zgg2D4>>BJDhTyNu9rT)>?ynr6$XjEc+l0&{&`&Fg+^?E*yY`ReBQ4 zxB7zi)Ia-o%J$0eA#e15HD|rQ-;6}!YPLJE6>Ld#{584%VLtnCutla($3x*>zuP zh`!eW!?nuVNLum~^X22lrG~^~<3pClszyL%a_gvP&3?tee7*m+|Lvi}vbT@HjmTl= zMRY{8nIN^d$Bp|O{#bHL?P>^5wE+~0I=cXg-}F4_(@PlPfGxIu zx9e*Kb&Ts&wV*G)-%i+a%NwD7fF76M-TYi0LIt_l`S&mvS8_u6nbU)gNkPsoq!6$f z$83|bcE}O&=hsJ8+O~vOTyK0f?aKIH5@DYR}`p>mE6fFqmulvv-P_P9E_FwPF1LvnB2KYR+`TGnL7X(58 z{=))3Zl9q4{WcQ$CzyXR3DN`K*{ZCt{YC37k$@0Flvtcqcwlgweax{^ceAm!b>wvu zp#19%Uf}xaHZvvpU#~b>2~cXvDUyTj98AbLnOK-uCKl2=4j#o zwzC1obQ1h8zy8(ue;@vT@8g=X$ z(BQ&qg~P$I9R-^9MFo;{gyoGbEjr{MDMbRX5o3fu#`q5F9T>Z~a?UQo#(iFfcl3-cX>wcXUs*y`SHJ_{|*N*guiwC8O4Ra*h0a^>=kwf{TbOo z(2`eNe@1b{8Ad-)+@jMA#_!1Me;59l&(CRVm}Fbz&)8+q2+YyoN(jX7^_yGd?;*wB zkoQC7JME0U*|?g)`?INrbOOTIy+TGRPx40%Ujvj;mVJbM|7TO}G6ZX?lE(AqP_D4jq*91Z3bKiA# zmVp0+^W=qa7{XyY*`O%&KOzeED}V>Y4=XDZf22t;#-CVzrU2^sGolbb@c`}d=>6^= zZM+Z*faOLvG=6J;MijIs9t8WkaXbIfX6UK~SpNDR`^)j45heJE2c1Iv86Fd`A3l7b zWoJ(yCL!qqj>61imsSBT1Ox=>1lm}+`8>AYm%%GtSfJaSY~vi!z^+)GdTXi9Ak^@j znyYH``2rbyUY|Q|lfJk>m%SNGx}c5@p`lc^V3)%sIYB`|%SRLf)*nl4o|Epfzpsn^ znWX?3e12mw08$<=l(SgxjeYHeXK-O?gGAou`Q)WO+#a*rVX_XXwW z045+^K|WKuQZ%n#MU3T3fepOR)lf3Klf5-{pe3K-z4}z5QiRNs#iZAKmK+%@@dtZF z5b?=OBO8?a0n@H}*E4pqF%USJ%4SBZlq+UXPDIN6`yBm&>~#Vf+Ziv~S!i-J+Ac*S z;Z>BBM6{Xt>>owKCw_UjTvTf@j)kr;)$DBLU~jK9;=IwHc(yr2)St$|pjN4`r^iHN zyU-}YFN@azk4mW6?6k8^-|TTtPs06Pc<}UxmcPHh)6q{6Ui($p#TM7-ldTa24y&pB zQnj+ud2^bJ{dtM$N&_0;*qy0*;qKuvHkv1~Gf|>qWWbN3ak@D~r;x=zHxy9_oainL zYZX)b5*hFl8Fi+ds(8sJ9h3Q*y{=jJXKN&8POIanWt8hICuzY!D7B7T@}m{X1=8O% zY)^T-(9zKF9^0ApTP5}N^%MA9ZI5S&IIShyJkE31dZJa5UTK;4M3IQ2Yy28!nAx7I zv&vN%$r9-9FlZ79!=c-rx9do(x%vYo-%jhVKq`E5IwE9ou+RjZ3iI0g_5#Z{TwsSZ_T`i8qq+I)K|RclydckwHV=4ma}o>km`qTJP#1 z7UPBN4(pON8xA6>Bbk?-Tf@;JmFL@|0j9$lbn0?x?4i3;6=lxb&(R{+#+n_`+k&={DBYeqdeC%L5LrFs5{{dnb2#vlh0IYA*`;&Y=_}z%s|&{c%4PF& zvee~2X`{TfUUV%=5Sx5Nu7T9xwl#j6nyoQ=Gh#hk9ZIjEi)jIfmN*o98z?R~I9Qt? z%=L@V)_j9qxVQJ^qN}*jJBZ)exPnlq^A+^(8?knF|L3UWuU$%%3(UWLL`o3!_LzJa z&g7#+%tsv-(}1HA?A@EK2@NKSk1V5P)NNE8e7N~lIZE3$qHm zgFI~*`l^&*N5}W?S@u7;{{fHRAV2Y#0SVaZepoc}#B@GRa50V07zk0d`+e~=s%;+5 zRUu%80C?0f2xqBZ7r3d;dUnkG9?&f2U;Lnq2++}c5;G~SE?}P)WoKu1vzcz!Rq{Tt zgjyykB@?f5%V$Bb2sHM^AI1V$$ErP}c5R4s(kr^c4gCEP6ey~a4~M_6lfSq;amQZr zV{l(};TsK5pK`A_{D(79!8v3}&x^gB2m&_3%aG!2t+zp)#7PHIN8dVza9u1|H@E-a= zE8G7TSEGD2Cf%Yt_sbVyj)adOWt>k{bib{i4O)dG;A76{$2D(4#egXJTe_JJ>Klj;SWwlSO!m-Nm6fmSwWQPM1)shBia?SA1PYwYYAPI7;L zUjpo4Bkuy~#9nsAmt5BK^^EqP08ya(b5S4>^h!}KlDC=ElE4$z?(h*fJ~CTqTt(Gr zkF{BBc4oJpVR>yu7jnu2A5+^|rd7(>4emeRpVycXGXq&DG`O8jfFEGXZ?njeluiW< zr<|icX_%WA-<)ohD(K!zRrVRqVF-_ne`Wps9rZR-I0K2i&kw^lo5Lz;Ht%*T+t>W& zY{!TPS`wbrDoEq@a@p5nFy*yTzzDTp7rHoFWfGg2{9L1b9#q$Q8A=_6pD}d6AE@cK z0Z#)SE1FD0^z!)KPcSstS#2I~mmkf4O;s4Z21X?z)sF}!SW^ddb{eZmm|@_i72qX> zFeusIWFyll;o}RpxZVKv#ze8wi`U_xIMuJNf3WtJr>UIp&Bmh< zbCZ*<0Inh(U=nxpywA2VCySN9nG&Er;p2yUPt58j@7sFKFP#$`!^d@ znbCj}9MBP?c-Vzhq`j9EUe6dQg~YlzxVqx|(;slZ{Q?49jJ#1!?3c2i00k3bf=Es< z`WF88W1<-<4730EZea)-rK)s{PD zI?(5O1HbAOU=7*d{}k?zeSxspU-Lx#>+5T!?l4?NL1DC&BSoDBBOTC_$6X5;i101j zW~0&EH>%afQdQigQ29T8C=#-nMs%-nS&V&bWW{P2q4VpC4knxakpp&T{T2jL{>ZP1 z^!v7x66b0iDj#rSOVvE0NCl+nR0<=GVCxtJ`1y+gI-T8JJ33d@)UcrvvP%vQNDDEj zmwyrsL@x2J6~RF}TkngP(4(Pg)E)-BxzGUF8q=Y;@b1fHpW8@I+XYEok!|1&@n3G6 z5BIlSi4l$VfjNMu8)9*7a<)C%wPkFh)9UsbmswxeDSEeQ;2lJ_DXx)l4B$;SC4>?X zz86YwO0A|LMg&5(K6lwFYl?)Avc}X@KGt`R8T{_7jC#!pSIeI`ps4moj>ZaPaG3R5 zM?#UJ;r>9j7?1j3Bwez1f>FVuqT5pyI!3*9fSQgiE3;B{y?$de;zOJjN+#6qwg5P; zpD4gV#pBjDO?96?voyONlDd0(ssLUe9bm{1F}!VsjQL4_oBKwxKxsydk?by;!28FC zdyB2%%+=9MUgxklDyfrHL1q3|zugr%$UmCG4>Z=J7-@1#4EQHca=Eau(EIhPh(L>8 zi_7SU(BrGkC)axc{m&58dLZgC4p%4M*C|${u-_aW9#$>ad@UVI3Afm2AHXjlFey>% zd1?C;qinr~`hCFzVn!<<7_1hji@J=3qG}6P`Xmemh&IZTa+s4?*c$8T?2O95z#vb>!NDQ7+GK!`v2!q)!Ov=8 z>$IlBWk1cWy+|rP=w!X`^qQXqpt+YRv>U|7ROpB1#rrHlZ)q;u1#$@q33|Qe+95O7 zhdXy<6qI~G>#KZ^Ah1~?CHeF7cUF!D74GnP!uckf=6^Y@DeEoXi{Wd(B3FG7@(IW!3Po;P~ z>E*$4hXUZ{vg$0hdAjb7<^}(2kwE;*KqBBVv7He2Qck)^K9g0ukn2>CJ4G}Ax|?2N zBW*K=B}bp@Rk#Us<9F{gvXIAlj1%A6;k~!=NW^r`Ckl_c$ET;*9c9qa(8UHFJ`+uj zW;KqUmk%Ux!OyeCO&Jp5La|@wxE&iES|$H<6**&K7ZW(&7F_1~+HWPgQAf<>^7B4V zguEH9+#Rw@f_x-)ZAR7 z&3B=4XWTd8-i8Dr9WPQKs|sTXbaQtPwSbK|s}&qE@p@6@{A+6U>%YQpA~VurPQNa$ zxW)q6)Hon8C)jz{$mzWibXc=f6+F8jTlUp@Xo z*K}z=fv4LNFJL5rXucr!g~azCS~EVHHXsWCZCPJkH9D8ieMG+|l?L@G+W&k9A9L5r zG`{u%p?hs`DVKvF%F)xl**a@Ly3XwBAD8=>ZfkyJ1cDpaqrEWVY0z8vauCX_e)4jP(I z{^m!WXCGu1pdqW^D9o_q?>i8x>(WZ0P8Jm|3^B=W_@&q@3=(fE(b^i7X4 zeU6)9Ildg(51(5s)s%vUJi$zpmNE0i_MF)u+u?cHAXFTHMrOQZm&yJmE-HSyE8 zWoP(>Z&)ZOuDtkL^aEhN5-0;7j&ftwghfF}Q28mI@ZAqD&{N#QT(X)Kq_De-upJ5K zoWK3GGExl~Nw;Mz$3qqHOgZ!i9ke;LvB{|=S> z8!3Uz0Lp#-6}+dW=JNtT!M5M*mObZvHjjQwtuFkH?Bv}6`gO6N@vY;Trh zos21FVR5m!zv1B#HQ`)`0|$v@ zW`jZj&kICP{qNbm>*=Nhtj3=*`CP+n0UZ%`hMxHvQ`p1(99clnDMsH-KmYRU*Y!R1 z3|7^0QNx_WZMTNu)rIiL=ipS!39RvxXP4eS$It?N&L5x;77M=gMAPYifK3o35M^|S zEF`q+zL74!=6c!i?8ICAvfetO=4^HBjfcl-N{Xw^LIjq@gU#_b_HG%a-EVIu9{4dY z9t7H&ebi6MuH|-Pyx?AU)1xK%UNFt3Z~+N+nJCT@14*YgJ zPOGU1QhqmKulp9aD%r_I6C60hr%iF_$+=`{9|nUyk)N^=08HA<_b@seY+@$z2( zl^+lQb6lq}pX(8+)9h4I3?C=p>0;0L1neBwJq)I@CGfjD*voBMne`{o)dh2;ZT9RR z9@^DLn9-}3OX|Y)DTf0Q*!{s`OY#-3_sTao>1;uPL|UZ}09Gy2-rjD+je5{CeJvJ> z)hFlF;oA?2yJU4m_PmTzUhfB&gyR$-w78C)o^JI%9*{^E-03?o?H2M+|DqjHLs)b~ zReD%FI{$ikr{!_GopZHJC#P-V+mvXf#^i9_4A1jYy1b=Aw|Bl>5mM8S>tEeq%P`zy zHg8(B%+>E4ewj>f0Oepamw{#0OUs2l?cfO+VP&Y5P-O>W-5Q(T-rok-`bF}BNbE}rW4aOh&LV< z-RVzIgX~kk@V>L(C&b6WlHRJk9(z4Pknm;H|HLa%^je5ojV>vcwLmV{ zM7fCBL#SviIXmo{lfwb9E6h7T10|ep`q`ut`6+ORc4X@Yj0^Xe`X+5^v7goJ=f(VB z=`SNLa?4_`J--s*bv!3n^x0N;<4}kZFOwK1K(25vIXR@Q>j%cS@;#!Mojc>B^nyQ*zT^1xKQ_y~| zOk`|pU#_*l7Vx@KB#!EYPh~Y36F6J#4yV&}m!sDKbP}yh(kq9lZNA5>&1H#&lg%Nk z<2Iy6A3*bzZ}(L0T)V0}HF%qmjUiRiIIVnl?2qA%Jt3BpaZcQdIUEq?^}TQ6);%qq zeew(z!3NFOAEFqh)5`kmzTG}B^ylajotVhIIqx^gd9^c`9Xw}+a`^sqw8bdpP1eC| zImnmIQjXrN;H>1;qkL4qVXkpUmc5JzUMYc@6qH7uXii5WU5`iC@ig(%@`{HaZS}5!d0D6-@w7?Pm z18}{X_JszW{Qi0?o89$5ccsfe1yI~!LSw1l>=7*hL}_;wh^|7277TpvIgSNoiI!VQ zS&c(9TdikV>9uQ%=S=<$w+fl`<2=Dijj@$3S1hA||GYa^5WZt{EOp7KQy;Q%XK|S& z;F)&50(aL4DEgw7sK=0AlUX^PeK%HHY?m!bkfF!WKFJw9mFp zDN~v2dM$57rduWKt?jEc1+E3u?A%GnnuX8a8+i1h2(`#gFEjXaY%X2dTb;B9C$Cu{ z$YMtF7D@avk5?Z>50I=0+j4R*^f(exdN_}y6QEXKuUtvZg* zV=Lnq+#ct?PX`z4uGLPPp*`zUu(OK(YS;t#GI}Z?kkhNrm>f+fkxiv&N|zB zb^%T!FH9bwRSh96&19aHBU}skv|oE?cYSKHclwixrn&5%>s?-JYnXIxxG zu+hcIE-u@&qQhwJ&EN>BVirB#@Y{OG``~jUEwzS-vq9z%e!Di7U#HHYvg`*vaS1b0 zLO(t&egRZm=zv1;Kv$m8IGVHLxZl*xih+wEne7I0L6I*!VX0;aTpBa+)Fy=kH^vAq zy|o39rZPRqq=n{>B`MM!rtiW{G@kdd7m%B?b=zl1w&=jWFQsx{;LFn!+le##l(=|V zJRl1rxva4(bs%A;vVE&qhpxa)@4sHM%%u2DvO6;AmcIimkX)}K3ic5na4+;c_h+|S zrgC=wb|qh%*KvI9lri0F-pgB8(+7K4=3a8~Qf5E<{Nkd}uKikk&(9bA=+`CSbrS5y z2=#uc%xSW}G6QNB*0+lv50iw7KiNFqoknX@ziG)Ra-U&ABRM*Bwgn6}=QCW#?GpR7 z@2hoC!NlAcJzr z$cAQdNcG1LLMqwvniD>y0dAbmK|hD-Nw8H!eMi9#HMmr80n2PS*Cw>xT<%OqysdVIaNu+nLumW$7$PAq$Ms%AO~Y=D{U77! zzozjk6)I*>^!@?Z!6<_@Vh7^pjEy3%#f62gB7bi$FR{ANXk6jAn>wrMmyoHiiT=hL zRQe6Rk3M<3QmwL{0qI-U(f6%wH4fl-25;RKmq`3@1xZI-tTqMSlcN0V@&0Q2&mrlI zP*hD7j9bG!>-{gPkL&M3Eg|Jp3}k)I0I=Vs#w34n@KQlR_2ehfD)ZxMZci|J>43W^ zpgW2TZIIOk+XfPSGQyjb#$G{i6qZVI`MQK4t0#&5vj}(aH$&PLz(O&*Cqt>Ny3neI1sO&r(L+B5q zgNQU8bRK+M?+H&bSVdk*r_80cmrg6HS|R&f$3Y(~sb93|mDk{ac!l;fJQ_#dW5Vnp;s!49?_L2A6cY#a~r-tn6UbF`DX&zj1YPhuOT}1^5;N0PFhuY z&1$#@nfy3)-Xk&|it4)fI;f&twP#w4;FINYyTB824_N1irv$~XwNfu2OvE49NS+}Sj@#-p8L4sc(>6kLo)`YQvgzNaS;{_A z!AMrCOhxpCz(^Px8rJTAj3$h*)cl(G-6gU$ExZmY|MIZ2Sp6~6=#kiqo4-boI^>XC z0es*7e%?8)c}!(cVJAm*iq1x#`UohW>8jLgvIl*H*{^5!gq9q4iOe0^($I2Fp@@8` z>|8*I)zAwqndH*=`Bc0=?fd)A1%MJ3MEb)N`EuS-N~<4NaBH;(%jMgpyI)$R>4Qr4 zmz}qBlsWp+>T~C9rL!ql{faqmW4*7O`4Bvsjt}9`aC+?{K7i1_J6c zR^O$-f4aJ)B?~4BHobOkySwk|=;Wp)%8cv;vrQ2|2i;|q7(6gYC$q*DXSh5x=cJ|Z zXX%5a&@#V!yWNW8>h)2}VKtv0aa0I7N!Ut<2dlnpSg@G59)A?bgV{pCEpS^I9>1LY zC3-v8G~wfrYg3Vd$58*~ys5wEGlw5e@^ZtGXF&LxIFj~5X78cSRh@6hN&DK0U;%Hb zvQ$#~&Y?q<;`N71M&%D;EY3f__#1tlsd_ib^3E+fr0D9L4dLYa5M{1INvsuj>YtF7 zi@!=-2!OmD^+R#x{rUkAQ;K1PJbqzHZ~aO#i7eUL0hF9z`_ij(Lc(g_=j43o1gd?c zfsi|~tjpB~6#DqQw-dFgxjD>PL_~xVk=zb;+go|o+>FeW`)&8o6ezA@YZy1vEd;}M z?Rz9Pmuy$W$r~NL&}8pD(AaIv+l`>ZXZCKvACc$`ODBxx#**W`o6ESqE{aF8A%9uW zK4;cT6;xYPW%ld()x?U3p7)N|7_^)sc(HbAbX!}{uef6o1p3en9_m^|wd!*`ajVLY zyY1*#gx%y4VPEWC&#Ws;P(Y-?&&G3ca$waSqR0jBYnO6$SRD#VMum3T&Dd{Y3p9^8 zqJyt;7WS@9!L7$~hZ47!qZ*NfMEmJxZj$-&Z9Z57*Wx05$0`w`D*WtSqhxd8=ltcO z-EcBuEHbZHuB*$`ag``Oc1ql_eJ&y>RScv{|BlVYW<48x5&s1M-89NjXR6VyZakRC zo7qP$K+0_MvO+Rw%EXVWqe+BUDZVbjU7hC8`ivR(KL!yE8~G38Ye9u-*oPF?VihJN zA~jNC8AzouE#*fGIuX$KFR9Wpx|sX>+j#BsIk3%LN|d;4>R zIvkD2gk9CdZ$!YuVf_=Um=51L%Y}Y6u-lRQ>ljS3yt=>=S!A||$VV}b-9gb z>}W+b`JGI2pgM|srk1ESKVtpS032bx@TOB@Sl~pG{L9 zA4%9qkmBf_(}PmsV_rHlzj5w2j%_L2kgiLxmqE4Ve>Hw2Z@bHz^#eX8jJC6Gh(I$d zj#{4lhF>MfX_x!rGw;IKFkKxZ{IhlW3ZR1Ku_?WB?2JR-Xne=JU%77ed!G_D%o|kS zS;nXl|Ih&6$rm)4h6UWUOVuVaRb-?uivax6Xl{;K35lHASz5dfnIqTQ!ci#*hso)& zTW@ZsDHbBY%FiSieAa)0yXX=+{u-5BZBQ@sF5qtG+oHBWWyVY3Qc=)8ry(M~H zP`7r7dm9J~AyV3yL82i`zo@8^tZe>RjH>`kDI7g9M1GN9w4?mMv^I+jMnj6%bu0fl zmV66qE=U<~t6Y`C&Zd8FPr4*N9}_%UA=Hv_xAZ&+YVCf4c`oB-ngFtBl2BBa2qO_m zNF5}&0m4A=b5BvSEvy_kN3*m7gl8)E8R8@-t@@B>j4mlu6byWb7AdK`53fFC`qozAX%|iVd`M%EWz?$f+EaB4K|1K!e}@n=h(`Jq8uG1^ zg2dJf+%jn3OLVDzfOm2vqFeDHT_4ov26Ngp4s-Kk9J{A0tT|XdZb{L+bu0@McRd=B zY;Jdu{Ah+*K(F|tIDj}Sgu!cKp=x=xiNNN}aML^)Z5KmQO~WpqiYaxlxer;cHTNz9 zjEq%-*9uNphFh=+&U!_`*qpWh#?(km;O)k>7#=$%!;NbdTVX8>EJfA~s5 z)nnZD=IsbmGs`Mh6|LE=d_qIn@!I$WN9HC`)O=Y2Zg>-+XkX~KY)gncK)gx~1G_mx z&V}@Ypx9TnPTo`SF}tMv6yM_Kmhs_kQS-}(P&)XAA9#fqSU6A>9h{4wy$A|n`*9Jy zo*>x2tKb}2bFtnc$0IJmL?}v48crAwZasSI$LXg^Z=kJxWaSo`s6;y0-U zpW=n|xujo|h3C+cHFL~GUQB^Ii?^;5j#4^#pahxz!P>nd5~KO$sk8jGC4_EvU#Y~d z)P)x7-1Si9u{90!;?4zeaSEz7(AXTCOo<{MSAY8G6$jSg;m1n~!q!3rg|MXQ0*oQs z$I6Vzq{G6_9D6k?$O3)vhk8zsu>vTR%#Rb)4Ml~Ij~_=i{B!y|9FMV@doR5WWQDd+ zxQtH-eZ)Uu)bRZ}!e!%@HKg=nsgQ{cmE4Z{9+)VGfb0fGPexQUc1E;VpcG5vji6hm1SGPbngK!$I2sbV?T@Umk=T$ z0~3z?BT`HS`_F3XA)A&NbSi|`vQ?mOvUJU&zp%Ux%K2*5lx=N9MY38}o&@B2f1_G;9Tc|GC;om7i;U_r-xb zDz`Oo%P7~+BZD>9(t8$=(Qo7$#zs&tubpfl6e&1bAiAzc4mbf%@vlBU{iKO4`O<3` zKjQO(6fCCaoE1@$U#h2gg<66_gV~>8&kqSJKd%Y}xgcNcy}#M2djz}%Kv)_KR~~CFHCC_&o-R`?AH`TzOGT-_%S{RRq$uyx ztrw8N(Sw)UJl`QkzwdDb`{E|fy+UpJ=3NVwe8! za}K_2VCRTju(%?C!go`c{1~h4C(%e=vqW8JMAj|o^CeWX#$pg0O-?}}Z}Y(xRnUix z%l3^yL_Y?*EZb+IW%yE&3TP0o01+xs``O`Zkdda|N3T(`Y*x`BTElfmv#AkTE`D88 zs65>VigQG!nGYVPtlmCZS&_>um2Fd*iVO zhnVn{dS;tsR#gHCPtGyjE(3 z*!P-~N6+P8Bnuo>^N$%#2UM^UBjC%o%ZDAE^A};5L#{6ziDE`h`F5N2IeZFUQqL8i z`W|MBmdP_JZvk4Pl~v3=Be?P)$CLTsN4y65&Kp1GHjSWE6g6l?Yf2)7jHz>EbpE~# zrF)at#YENs^@jZNqYN5nIY03y3!G~eRoZ^d!9DCl}maN zX%OGhEVjqv$}(ez?<&3I!dX!C_uHf+U+Ski-e%`tVQ_bRAAB`t#@x1hB8gB(UbMS- zm0!-)XJG$R2KWo!U#6VK6@a5HCa?SdGUH7xEh?c{)Pqr4O%5jZD*nNyYb!XlPjNBP zT2xfj++>%pdAUYqmge`wn5*%qSPlaE&(wa#znV@J?fJbh8=4%ql!3(bpu81Oh`me0 z_D%)Z>?|$(HgVj`V=9liwFLudnYRIYeB0l8bBvnw0Y{vR+D`yYXVrYeJUEMN5OXNLuzp9dtoGvE2m3lAjCC_T$K6FM}6GC2(7;iwi7$iUBlDh|@Q z*CHQaW3#0GW9faj6!R>`(Wq;@t{v!=^Fu7~^V~gwd~Oz>Z@+HP$KebfFhCksUi2&i z9tWxl?K58ban-0T0yg0JhUb!i-y!S4CdCP&o85zo-p z_q|7OrZ9E%5RP+wg;!;BKw;Rno6ev0<7Sb0Hm_e7MfQ2{!H>I&>5ww$Hxsf9&w*Wj z`k)cpk0WVT&x1b1ZoH&bbywNGQCzAcUI#=%O&jvy@Zjef`>~m&XLI5s25_N(=ju;s z34CFITeF*j%2szJgn`8VnX>FvBmcVUgLO$)jG|HCFh1#6QtXzf?|NhN@dZ@y0kKsBK4$NLkd!# z$Y8@YQ$k@@RLnp+kx!L0c$37LPsJn&+=f`CLKP@&s6KfR$HVH>c8B4@;cSf{!(Ww1 z%sX%~NyO>kfv6FV(uDkjFUF!oKCD`}8(RB^EK39k{@mm-uissD9MY0?eM75-qMhwd zGh8+Xeh}gE;S_dX=u*+zMY}{fQPPa|o6-Tk`DD9JZZxN#(``=2^Zp-6@5EK@U;S;8 zZQM~7xF8)sKICY3PVG2rXQf?$->PcE{FDR$o9E31mKr`f9wEdVbDsy3`6zsZT>@gH&^<@h0^fUW5N8AdS0?jFEb3W%Zu&Hr8f-o=&A z>%40`)#Kz3Yy!X-0bd}neEwyuJ{XNeIhZ6&4vGpg{K76whmdk_wglU0cT4}n)DzH| zay{NyxIzX&Uf(uGB!ImZR4Mg-ZW}c@Qyq>~I7Wuop9LCA+)>$?D)o)@+4Jh;7hwfA zWdK+fCh|`-z*^;^B$g7MZ-B=4)dt=qQn^RXp-Hit6y|~@ly|><7l|sOi5}m12ug?M zCyrA|tDPHiDq@?4$Taa1+7_p|Tq)C&2L@t!)4nv6##r&X>%xT8TNR*?+ZVw3_-Ky< z-?}j#;b5YlHf!rKv->NWUSYo$$K-0Wac{cdCDehDBq&oaTFs+5+DWSmSmw<0^K^X^ z$OSojNfeA5?P6Q053_LsB&GYMsz^qt29h%AnK?dwo=7Bn30Tq6GENMGbV@6wCc$5D zF{1iyxr^uRc)^Tg=L-Z)3O}v@ha#H8W1TT zy#M`M0Q&1d;o45Ac_xtMph6?NXL&%>JM*zUd)}xVibBAD`V2>e_~5;{xPWx#g4BK) zD!EoER_)dx%=^msj1# zV{YHx3&*?My5>l079OHuzVAir+e-ZHAnu-(E11d(o# z4(aYL=|;L!kOt{aC8R|{TDrSi>5%U3?rt~_-#+`_=N{*bv3@vQEXKQd<9Y6R&w1T0 z+lgGxNz0n4Tu3AgLM%~Rgbyluc7xEt#ex!^WzCf9q)Mij?~v2_WHgD_-E!|Id{8rm z{h=(Wr>YH{{?-w+rk5^kS!ReOEF|w7#T4N%d=Lp=0B7W#j9Rv>j5E3I1A#(vKmP=s zx@9Xn{WI3%jlAGs%t>E9FdyCNA_*bv=dRt|$4kOq6<|efyqju`S}Bxrn6YX>VO;SJ zQ&kBHRuvH?;9u*WOe?uHRKA%K18#zMPY_UCJ~nuYY?y_$#JT@)gre%efChlm&54mr zsF9xBKrfzEC?}V8Fe70Z`ERU-V(f$np>TuS4&>Q-+Kw; z*;df_ZM3*$yqybHEPfHN-s9_+5+3~qcUxR==}=dxwj((3y*=DoEmtFF<^HJe)UwT*pKY*+> zB3UW2_AsK9VnNc@$_9cabD#QHu{K{fJ&Vqde7I|P;KxmO(eZ87#aSq6IOTK$r-ee` z9ez;C_kqm&`#eOF!Fh*>CtaJjS0SQ&k2+Yn?L5kKTwEQ*aW2%AAy85SL24OfJgFUA z56m_D6W)75=dnNW=WC!jidPD+zC&h*inER!JZ(;H&cWf*t{KfCKdC7;^;>IS)c}c% z?`OpkJZL(l&rF}Ks&%a&vs`i$p1G076DBYE#ZXn~IuB{!#)lX%IjobqP%2IlPknG2 zxgE1P1@ zWtPh1RCX4`I{#sJ&LgF!aD9n4f~Fgae7c|>k%U*>CvK%Iq=S^zJ+EBIAf+scf{ymt z2A!~Ei(m~tu4jB1)a5_L$N#URvqBeNxbgENk2lxdFU_R_ecQVRHPR{@8!gm0VCPCN zs!gG~ju<%B5qY&t+fjp8YW1KAe}ylV7>hz>P6z4G)#)99G;IP@F}t2aS>NUM*IqVn zSxw$3hOQ2p#+g+74rS7fNHoAd39fdde$W076{wTLpBqCs29oqi&}xdF&FpXxavXH+ zL6w=bI{EZ?ZSc~YKM9wI%my93A3ga-Yt8g9PSu-FU=ttASDi7vG%Y?EwYOiVP;ZP{7h;<-uMN4|A1<^r6>4hS7GKD zIgjnm+HBt5k!b`IQ!6H#jsIbKG*|Q%y~IJ&Pr3+J;{L&BU-RWq4N;Us+!ca`c6xgUnFKBx#h{*cqNOA` zq-?m-ev-Nr>a-#Wm|Z~nawHG|CtM-SPWw}D^dIv?m)zNCZ>Pg0S*1jiVPa<*t+kuE zNn>q%!4PUc_7*#>!mTai+Q&9eTlCdm<0JbPa z@O4c%mgDef9imXkJYna~CW2t1!ASqEaoC0BS)M`zZ?QovyQx60iSzd6234bAouq}K zWcf^C!~QtDvF~{=Dt9ihRcTwEN-mgX9b#zbQ}_FyyecwyFwTOXAPV#vN-w-kMK`4d zr~gN>S^Wc}U+WqjjBe@p3!AM79|`~3OvjZrg~BVO+gpF${9^0GFCFCf&L1M*Vn<$6 z>7;A7jdIzp`sXAomZ@yR$ZuMnK7NFbR!A2tQRmzghIl?`Fdc}GPzRz~&_@ge!FC+v zQP3F@Czf+Zx-8*z z!S*ki;2SZbp=kVI6rQhdm+`TqXQy+8M+$>O>W#Zz?ngboP^|<@ zj$WGyZV(+#h)NAkE^2fT;31bXuU?ABMP{?lr_oDvP_+y_l~>fc`eybwwmDsZhr;n zaABgNl3@2tL7=DWzVuX!2rTeX*j(ycR)bnNqi^Q!UlSgkl)cZnLK)Pw%J=H(sJB0%smyI@Xa<%6nX-q{n%anY)}Jm0OT{kR!bq@I z^c>@~B8!D6+1NSl9gx$+1{uUSxCV9LuKGrYU$R$`utxK2ujB2(GPuMgE%4Y0_hV$r zC+Jtv?q^45c&^;{ml6fuu*Exa+edHj&P0G1bZkF(??bpLh4ZBQaNX&Z-~QK^rd0GaF43I*e73f_5Yy&q&2sQx0a^U01*|ZBY)(p8*p46N{?g{E`zB@Xd}kWIdMKqZ+`fw{duAqahV% zVAj7)rLnT%o$-%E6-g;vXCj0$2m)KUyTx&$bm2ra<_!<|X|3hl}Y3C4rwG zgKK=1?50}`NVc84Pw2@|QW^ab&n*#_VScC@>G`cjO-fkshaVukj=)m>(zH|yXkDsL zO&QDHw6$#FytZpx6*NeVqV~Hu_)Eh(Sr@!B&~bWF&W~0@H=Z|EqV^oh*wv$u86H=J z=j`jU2c3Gy(;VOjG!0^d%<9#+ch!7vyb2M&jC3+125_p9+?&%epEAR1CF3rq#6D8f zhwJO#M=5U&mNBxXu4HVuN%0_asv6Ys$QV(Md+V^-9S^!&U6mZ3U(qQ2es_qzpR2M> z;Q+EHIi9GVpvw^8?($(<4&4?Y07ltnLs4PcDe4KyY5?x#w!P#3rmttPx8P>q~?BuKE*_tDd}CSkK50$D7+T86xz+_LEKMYZFBPVXX*DEcH|Ka zK1WUwHosG6z*PAs5R>o^fS?os=Gx5(_uj+$Fdp6qLUpOpam(x`Ka=jiTeVFLBKY@ zk&^GCNFbbcqf{c9shINmUDby1sUDEfdb%gmt4s85iFfa1Ub}McHs%M8jyNuGfd~k} z!sBjU`UC`O@C}btXPX@lNx9v!puI=i^HWp4YNmk3xW#y===VSv_j~YP$)EoIBz5U3 zL)eH`0P0-^90sD%ci?|78qe+7a0gxvl(d$NxkuN~Mbp8b;ZKD6ljfQtHgRl6CV0Y0 z7iDMsOGa))C{Lx-_Bkvqxy)m6d_JlqIR^!xqKB?7{Tq3naiG)Cul0*B)XRL}+(g+cfXB`xG35@^p#xjfO>x*wSuw7m zSLyufg%lj9dOV0~zZ89e0?EYV^ERH!s8AJzjNHsupCU^(SUkS&49BsZxg1JZ$i!w| zlm5BbS@CHE)=*8VwjHn^5P1XpMHin-g~$`lLAhP(80ta%ZLjCIrkkO89DL4D*2%(m zt`x1oH*L9({kN_6Cw7cFVNgh!3gG;9Sjtf~OcoM?oOqp_=l7&L{vZ5lkT; zFQtNHQVNuU9jMC0uC~ zN_T&Jzy6}ZZhmFKvBF;HXm6?8$6cQT`rlU_d<{yetXeyd_srU!C$-wnfpmEXKXt>_ z_;^2ag&jusyxU{Btk)d&&ySQ*%T>M|58dPMi|M2y+LJGX3H+)Ym$r(fTOXxQB4ekc ze|5CTSCa`s+?(c0&F=AyNesR#u3&S-0v$4_?xQ(Ue^WQnv}MS`Wk3I6&femS(zL2{ z%1bM=(e0SN#be*BI?_zg1)}f3FA3&~<#N@~WK@xE9?v z6oG7t$5f;PP`_!9R1I+IvPkciETbnRkh9QV)VM#otplJ5n>9r?$y$^pEiuUKx+tQtr>%&I zk4Xw*)g=}No?GCyw;Qf79^eGVg%@ZSe|0Dr#vq?7mpIWc2~4Z1c?yJCs0B+OM;Mzv zuLiu)IIYYFU4e&AE=TuXnepxn`KOi+tnB)(D~Oog+kx8{j9gkS47xt8)6|aWZ6AD- zYB|1+j8s%pb0K7UiT_l_X6|3OfWXbO5PrpytdNkBUmwTi!gh!;>F!x(4@|!|-M*=i zLB0bX+k>QOWDP01^?9ojjwZm5;eEnN)NOtAEA-PJh;fLRoWtimoyfvQ)9UIubN4!$ zo}FIt^=Nq4a`wL|3vOrdiWE**Pjkk@)x4d5sI2gVRZN`mbc$4W0z<2NCP6%OsrlDb zm6h^I6Nyd2n>?XU5PcM;pFD2PT8~a;xAUdeb^dZ{Utd4Z_z#8qrfA1h@^TXfgXpXuq90faQmY&;AzfGSx?!FgGGW%Mz4;BBCAnvm-5*<)co-Q>ubt zU}3&+`##t+dCl-p-~5y!Qm0W?euK(3S)7IJX+z_OIqMn8wCXJj2xgtZUhr77 z-#e=xEVU%xT^-U*X{u1jo8f3e|929Czb5GmZ$eio;Q8&}h$81~xq@dnX92?2_X|ZD zOqVD}3&D29D0n9I-)M`xiZv?&ZcuzH0c;ASU$LqPc2;R|awCK+ZU7|cw68;}Dm*xF zBDx7$k5|e8 zm^9VN`0~Lt>2q5d<2O;~{N#yOQ3tmk7a0W0wQPICqG-Cej)i!vF?6@NJQ^R?3Z)%1 zxmrH|o}6Xd8O?ZNJxA7ni0=%7DE3WW7^IW5+b)u?SYzTz+~ zu;Vf8#6SJmP-pRKalZ1;4#r3`G9+-j48iPC&QzEOVUr6-*HuWCupt&S& z%H1F|4wl9;d#}Y6{7l^fDW5YPqVzT^x{5^oboUm}6?oDkDe4rzT)4e?C~!o^IAKob zH=%%y>W^ojE8TqXv;F_Hgr|h28lG>8AaXWnVZ+l||PNLbM_;mLBiw77+(y(`qg@&m(K zzlfHAcKnikQH&&1O0b~1p|l_>)y@0qbMVOyw2l!rrm^R`+tM4z6vYn5265Pp|F}20 z9Oo!;l`^tE<;7I*UwAwBX1%N0Q|Zr!oAe2b{XK+9QQu)RTVBX+x=F$7I;o*Ij3HlS zlCwOA!-hb7%8M&}mzzl!*PjsC&2Y&KPtBe8IaA0Z0d1A^$WDIUuU%xb`6|s_IIT`U zKl$^Ue(&RsbWyhcdjj!luT@gV#VIn6b|1Kt4dU;}6^-y7^KcWnBqBsxTbEZnTfu zpoysXiMM(-cIv#B!3uYey9{mdvW4P~lYT0|EuY<1sG%;?Dyq4Y-8T`$@ojk9Vj^|9 zeB{-$guU-_{%AUcnB~i5+5VENCr-8DT+mJnhcTDx_%`8Fn$bvpt(d|LXLD%wfOij! zSnL+}%vCaqyDIh6MtNEnk6!D0JfmpBYoR&72!x>~b<<4WVz=TvM=Q;reR{Im;a;Id z2Ot!ULi)I)j%s`x?8l0Hs+BH1;0Fl&=4x${SvN8&oVCX2s`x+9erT?}mn( zv~yThe)MZa#3v6mBf;m`P#ul-d)Mvs+pLEgcoDAA+M?9}*VMbYx~0bYyH*Nz>X7CH zdSxH3AvO=q2Y)0Y4_?0E^$LGiYNq^(w>>RPj*p`1d8x_pFmXTP0 zN>vn$$g=Qj>^`=JCX&-HTdxkZjH2$UR)lui8eQU++7L2SZ{fw>Hjk#^zy{X`z@jj% ztZazU#AS?=!5^B}MMUljZ~i!dygvGnkN#QeI~5a!a*%RW&I`TQ4?($|5V?Pt-aaVY zn3|&pMbuuis*E`Y9ve7%GUD8J*acyUA|vc*JG;Bz3-gF&Ts;89kAi^5#Ry}2btDAJ z?atw$JX72rhpw}Ku#;F7m>bmqmq79!>Qchi1d?i8U;xOUJj(>}YO&P=S-e0t#JaVq zXT6y7^za@}h9VZq1mFak;q%^CnO@_k5yN_$ytl$%JwWVO8X2x#luAjUC!{uNcb^R< z?6$i{vZ3`(IpnN)b(P{4Z(ubyYS#D;uSYa((GySjng}*pMMSbcor8ixHicv?xEP7V zCRH;`myMi4=-Kd0RYbnoThLVKTkNBjB}iQDFj91mSWSEDkBTnW^?$aYq5i=tHlAmB zUZAI7y$_wn?kmd&@hJ8EvAeJ2b_4T+fia_7=a{;;=tvn5eBmv&&5uGZo z^4RvRP2>GTyyo8?M};Kyd6;oxEfOiXcOXyk>p?%9Dw5r#lr?4jdsAmdHh>sNxH%!i z-SH!!l2R@Sitcr;H4M%B6kZji7d_O2U=ud-#o0ZK&+>N=!)BRq;QwK$BPLV_bmy=O z^YPq>{9LU}F}N50*m2!vV{a?5kdJOjV|Pb=9E}QPrF=QjKlnct>i+>Um6#9+RDgxF zh^t#Ml~-Jv3@%3#p2TLUIniOG7fllD^IV0rA6+Q8zXuLtQH5FG{RxPoMQZkgK~ zFD*+bLR(u~OEr|G^HRO09RR??hP9X}C$(?uo!g%-+wBTFF;Xyj29eGz)#lFM*VJ&$ z8in;Y)N0alI<%%UjhJYNKg{Nc&r!=IRzQf{I7Ghn^{0V&`9*To0)p-ic}DGODSALm zi=F+zX=yeHjdI;CdSi~Sp?deWLygQYTP>1r9Q<*vd7z`y?qJ52@t?EQ`adfy_*kYy zd9e4{T?-Kri9{6*+4r7&)DW(`C$Audw#KY4Qk9%{+Cb)@K9-r)jVaJ^F49jx#PrPw+5Sms@GDgh6kdVL(Mi6$W|swHD?!s@fWX4}~9E?}a?gAoaUO z16M`gpa;Ab86kNeim?#TBRAKMiQ6U>}IKz(z1GMqSI zd!=1ULHe7E-^8rvy5)MP-^)`$^QRL-Iy*cti`9mmkw8HG_ty!_8)3f-mi!)vbig}H z5NK>@P+Dyd%n4Zs=iCVpik!)T**oMrj6=;UF<}U3VG?9?Ztpwx)8uV}=H_N<;GI^^ zci;MptM;a67bwjUj%2cLqSsb%_SlHV^5})NYu8^h%)6^J`YQ{P)opw;U;IhisRNA) z;V%W1S9YtEoPOO8T^8Q=HLMeJEOlab-^aTt(Te`W>0aM367XJdV|TSMmlO($EF85Rkp`gLw0kas_Y$txrQ1D=Lz)(=gn9Oa>(FwNh>k=dnGQbh{cp4avRm zAiO_I!T6h#eSY*8(yPCfsZ}p=do2?)Q)?9*0eg(r^!ZF~TV{zzMRGj9*lKI+P{Zjm zt?kC9Cu^ZSN!G=vrHh7$zD`pTC8t#k>IpK#=_-hL(Ho#o9=`PPwEl4F*@|P~+58*( zAS)K|@iinW#J^wV0T9qq>cS#u|JM)wghhY*p;@k!jtbd=if1S|Z@KC5ET99z!|2*IS`XmIj90oXIcAV@0h?n#H|Ldg* zx+UavBpFiV{s6)}3Xos#Fa$dR1XmiQG&p4>q26?I`h!^SSe9heMWgfoP|48k(1b}C zF=pC)g~Y|gc92-~8l7IC*DdS^UJcMM95RpyVTJlo|Jeut^a7Q5(bpLLKlVf_0lbds za9S!qKmXd1FbgZ{O#mD;49qA%m6JoJnd#~3wi@DbU(`#G0G^!!0CESb0T`c54EZ;C zQ7^D<)&sP=wVo(N3YpmQ0!H{R|N5?eG~jQUm>Ixc1-&(#uBeiyz|p^4ZuCW0n5}<= zycx_>D0hWq?dqFNTx*)S@>3nh%w&-wAtV0^%wOzBq-huwF0Nb(QGYOrWZgbTACUTF zc5cKnXLKzAoR@^vR51&!>slvV$VPZR327->Zvs>LXf&Wxi30K`vkaHq8x$`|G?Bl* z|L}aH>%s_=eq*2PC$KFpxQvX)-n;YrW!FKgRu~KZEKbniwxA)GGg#>1>5CyJneH5_ zh};Ec@rhP%&)H*udhDB@pEpPRlkpytDC1rp|5G={wn@TQ`)l4kftI}AhKn4;Q39<$ z*eGOBy+tlVuF)_t&lI|1e)Eu0gxKBUy)W^5^F~wJ5In0c_RHF0ii+qCK>-i2YnE#A zo~1@5mTv-HWCYK`*5zEK;6#?EEGjhc5P#CiKx3b49=`T0Qrs4p8pr;g{^ z1N2o|>+Kf4mKl&7&jOK>@q~^3d3!R0baf15^vv8~x=`Cep;pzW&VX-<6y`8eK(mZN zS$9A9K`BpGJBFs_=5Zp#D_9;yA44{+q3ZG{CO`Du@jEoPW>=I{0@e|eV~%SA@;qob zxveg-wK$%(v4}S8zT6h=nw$l_CY^&IoYm`t*>?jK9OOPE!;nzq-&VtJs?0~plDO?N z`KLf|QadGi4INY5uM;7azsJFFUk_~OYq5Fl*Kv zW22%_22=T_Oc&)&6`BZfn>!|F?+C`s} z=OuiW#3H=AN`cmS_}E9*KTG@dfH*tC)b>CYT>G&Hxj&lpi+slcK<;5qzrbUG(L0-i z{Qo>xBjBkg{puZORw%?vdY|2LB7f#!E~B*?id=sNkf~cH0R(@<(s`ZwtqpYA!LlsX z>uauAeQShh#^FS1u4irf9^(=SX}*)w^`7yyuEsl55{o{srBk8jhaZ%RY4j)#^R+gy zVBu8U^237s&$NZk5H!vRouIywvbnO+N0$eHvu6h{8eclm4h7TQY;m5ut zN?3*)+tkKmbKo-fHE-X_uw=_=?jN6_U%5uo_=;8fxj&JHXOcFqj}aTF zPdT6@?Z;sR(z?};dk)DmqeDDq+F7}kvA+dQR-3F@9>h9eN9#<`1XrwHg7-qS1qzue zOU&r|L}|miQu@$MW{(T=wQYR~}}ljEf=_250DdwNK}C0KG6P z4p{9ZgWf=qpIb*~z57sj{u=%beAXTX>)3`Zg?0BCz}xQ5h0?j~O?>mXKC)54MVEjnH4)`ns0;nFuv0)=)xArSDN)Z${n&Q2fQo zswkWv&P{4+su1qma=;(;QXOUe@FqGFCTlRlOAZp<;29v@l_L{NirDRz77lKfp<8lS zMhrRHW)EmBQ{I*PK_f_c7`!>yLHn%g*QveTBo>I3Bx*#t&TC16Hxv$#1~twe5OWmGTz2`;|3|6H{(!FobNUS1RWvLP|L zD~5mXccJB-!u^w50bHfrPmtp0CnUn5pMI%xUbDbR0mU{8Gz)sa0|I6`z|$5Ja42n_ z!d&!=_(4O)Ea5=$c|a3jk*JmGCW2d?EKrhOxROKI&xEd`Ug-%OD~o{3GMJel?yhf? z;IE*dpf2itxrYUYQmDt*^+Y635@wM6W0n7HrbsdcBL!wnWB~(|Ow|%dgXaL2@*OC2 zcEFu;vGV7Y%nwSWiY|Dccl_RW7kjY38EFC8nQAcL{FPD1XOLuB3nV}W3`q*G{jcc- zkD=b_X^o@vFAcwbk*ozf1fvk=f}MgAIPiS!8cWp*WAU0u&h}4#%k-KgK+Yl4<#28k z?DStu7ETvCO0(EZZvn%Zyr1i16af#}{7CJFaSz4o4Bk*_hlm;8EfrYA!`TW01?s00TT_ zl!IVwZFtnIhs?(-_-Oq!K3SAnyNut#DAXr?A7!HHc_^^k24daMe|YXzV$a$?M=~yj z6APui;5Jb*^=wxARh2K2mz21$)vs~F#lf+?# zKAd$b)u^JfofrV0^~{>?}`$gD3wq z5DikmTocNH^Myn*ad;aJ+^9$%MrP&@dLvv1ev7)4(lMRPT8pkG2Qu%h?JdBiJ?0G4 zgMh;*2UeVHu#Xu5<6*uD;nb#l$oy)G!(2fj92&`UlzL+$B;*jHuh%T>A4RNmA}izo z#-49)U^Y00@#Knx587uuJzYG^pz%;SE}g0+r=15CUYA=Q0k5Y>q82wv3uW@$g+!GPLx1}UB)-XQ#PnV?nE%Nw z!o+*hk7tOX_Y)`uCL^LE|X*6iaWq>y== zUQ6gMEXgDQm}hTje|AK8!Fzn@YyLj%vlnbj29`n^vNeH6(ryOMjqkK992b4YnH+lP z#67hiQ8pBIy^hv?${~c^Xb&iQT?3R?(NNJRe2(67Nk2?%G>N=pf}BSU*Re&Mx*hS~ zY-cE5^4~D(KhRE3sFKUV6{eL5no6(dOyegAc^+rW72T*g^E!Dm9kx`aj*-jD4$9PK zq0jb@9((4E{@M5JrAfnx>pc=prMt?xQD_`~o=AN+VJ9wGGgxflFw%TgWYJNeR(;%$ z>~XV4w5gDuE)(kE4DVeM&laxh>ygNru0_}FkP^@lP~a39s#^A0rSW8iU4Dk324Yw5 zM2FW+RQEWnLdfIMPrn&!8D})eDl>Z{jh-;>j8<{-d^#^yS{c`6e?)M(>cm*EiFxfl z5unoXYjDufMP4i@fGW`l}!w7`mKr*D9Smfj~T=PWm;UwnN0ER~B!_aWcm z8&8%P(RiDdJ+5eL9?@tbzK_%GkkuLwiQ*JyERz+j<%gjkG31=J*7cXY-i_zv!UA6n zrgykaq!VKI;x1LkdC(31vJviYjmeIKF*}aImeZvSKX{#XgdguN$3N*{)otQ&UPK5j zY85pNIWTESq{*f~)I~jXdiT)TKd&dTylcRskiN`RH>(iQ3kCk+H|~!z`akNk zFG;wLGdJ`L*(n@kZAxW19)*KB*V0Za7wUTAj=;dC9WW;%;qsR2XfsEstcWo2o7edu z*BMVRs4q^d5bBNCUcvYMYyYE!9^wo+cCkA?3Hy;gWOuQt&TGV!TL$|-bBpA+x^(qd zwqW#ZJDAHufVIgdrqB2B5(kSG^ETQMq&GV4QXGE*Bz^=gn*`%iFZ5tKfCejpT?#u* z;|}2Gngf)H!}hd+Zw^rB@Z(mYeO7u9o^snm(B6f}C*!}?Cm;smoKa=t@y<*GDG?v> zmx7TJ9WLxT$zM z1~+AQ&*&Gg>CJ+lM{8itX|_P@H_29KWGHldOP9WLQ}ON{_JMdDDxt8`6i#g%wQl;9Ys~jkCkLWo0lz;oCNc ziV-OIIuTY0svQXs=#e^J=)!^A`d_V(VbZgscsg2%ms zQ{;L>;`UCyNMmSR!OE|U4SKPt+<+Fxg(leRzUyR(+$7cUm`{Te;ynuF8j9Y3dcW&0 zBfQVuZJ<&~y>s#%_ozXNJPEMaTk3-M>CkG9VBxOHw7aU+QmqYxKkBF5Xr-+Jm5oTlCi^EwpQ1vAQ3DHa$`GB0Z#PT^ zS$AGk6r?}tUy7H}=`QWjvkrs_er${go6M%P&CXvV!i54|XEmgC92v8As?3f`H4KvP znophqF|q#>*!+_o3m^hs=66tGsZgG`1UEQxglYOd-I~?#E)!S$?$b4_um`h|OkBXG zc%wy9`3uiagZ35B!tzQjH~a&o7EO;q_~5uZM7PJf3kbfj_%#!GT7pT9djWEG>OA;J zHaau;h|~D3E7_|pj4rwiFbvs_a@gH1@mp7SQ&}$s65)D{RRFP5Hi$CTGl~17ztgI#tYS83)tt)%J%;=qEgit8LynM)!xG_1GL(4z zOeQneDPxv*RX!aX722Isn32Ye1|^s&jwhx|qK0J+!l+!!2ASGmkpwn-NC!8$=GN8O z;d162oeSVy+t6^+3InMW=klN+nGJv^Xbhq&C-OKAJeND$?|`GpstOJEpQ2>Z8!uHz z^ssL??{Q(lM5#5{&FjEPf1of)hvBl23M=QI1VHHQ8x9DEZjukpSD8hnU6Lz|GY*{# zwqf;N&f2`zxvK}$>!=3m!e_@q3hH3oSprJ@sr_*ld1lCjmG|aYU5|6d|R?;iL)KY zi(5EJu0L_r=J1W^_*^Zf;K86MqWJbirw(*m3*x2qs=0Hn7)8*GR>>Loyx)2ztx*(o zP8du&%!Rnvrwvqeq5I^gYc@YJE0|^h#lAJZ2qEQQH>f3??LKUrRuby>_UbmJd98DAT!JFgQwJ-~ zn-PlKNH?I?HB|EWa{5yQHy))4wWwi$z`y+0O-P*GeRiY1mefBq?ckd)n`1r~S^~3$k%|uf`)$0`Uz; zWfmP>$73W&u4Zpcr2-Y7jGos%zuKcEZSw0*%KA%Lt#ebrWgcwOKIMJ(NnSc};m3y$ zQuWm#P8(}%i9Nn@aL$;JG0Gw0(DdBKVK&Jq%Xw5j{;XjPap8~Yt#8HXz%_x>mzpGl z5Js!`H7Bmk+m5b}z47CBY6dL(_d3v0`zQ-Jq~{8~SH9LNA=;`O`q8GZu{^EgGpPkj zk=j(*XaYc>j}7T9vv4hk%YmjEH5o?n&BrK+g2{!XvAZa&!ST>iK5CmYr^vpm-2+fB_49x?K--?^@$S(P#4;eFtf?#;N`Kop*532{8kV{%yP_19Q zM;(oX#^zK&qZ_Yg^Y(l<<2f4(%>MC=mMMD5aT~fb{?qhaoN@lcN2w6QK#?3n-AA-@ zhWO3Em;PIJ*KOO4$=p42d-(g4BD;~D<9a}An04{g=cLXwUg^NobRQ8OI|0q%h^ZQ=B zYN5Gq>5G^ZvRtzM%Z6KqMIeq{ajr870_HK^5%^pRjJ&;sr#+ung>qq}&@xEZAghPg z{C|AD>=en&514{NN6_kjLx*|58!r0wv)&(+q}%uG!43%V@aK08ru@85Z{eexB_rcX zhhl5(o?Buxe|G$-@_E-U^x-I(^r&}Cf6l;RAeb0I+L=ZI zLts{2s=Vr`>{x3bz6#?EnaDSF^>Xh~)-X+CYBt&}XeszN`eN(u5^x8|;W*+5nAE=- zKaCFAM`{YM3BFjWN}i}~hutbv(B9b#hGhb)H8<^-ReqCwg1qXi#Y#1}wCR`Cpwjuk z`ci?1xWaQ7jH@i{>1O;qqZ~^n5{d+g-WE9Q_zpIWEfrmi@o&G_AaU>(_bBo(adUY$ zl<1KxzMDSp?Kq0Eb>FV_``*}nEYI!w6`bMSErYNwf-$L3ks5DDT~F(vu+&iS5Ys5K zouk36FpLowCxAyJAls&_{k1Y*oFPZYZ;WV~k7r>-$Ggp~GaTk1oFt@e1$*GbLYq*U zWa%P43p19UTsAt5Rf(>{%Zi8qd=`ClLjxQ7^5?h>Tx0Ojn!hjR{a9ls#IFfW-+yST zxZfT6xaiepeQvO$2F@-M&pCIa zET@mTs?^_GQo@N^ZqsALWclnUYn@CK{9;++sB&b#`ztI67Ei6d2}|I-;wjdCpVZ<> z_ULV0p^sa-|D9hTl_G-`|9@Ekm`96F4YgIb%mAss5#9AfRdMlC3jPy0^bqLyMt- z99!^Z94~fG*r2X|`UQH+N&z=}`7_@0GZc+hzO_M|<))Av!_3Mu54Kt-sx4v?NehFO z9P~EcwWyo>`*c5UOpyV^%BEJ$UZ>Qkrd`{XUvmDKTBB!ZZvgAy_1(?y)=C4HroUv_ zJBZql=;_1*4_A3!7XJ4uj(5A?g$j%p+u!+MZ+GHZUOxo*+l}8TzwLCERY(`5hfszJ zi$eBZmd2vg&~H+DlNjE14t~S5X~mE0UVD#gG%pl;&r6vz%SHA29R=zpH)3pw`#G1KYSdP)!LM#VM6oifK%!|LnZ}R)-=6HJIJfS%;n# z(Vj#C#5V>jYJ_ok`a`G&8U(_*E_avBZiK$M-*(+spk+gOYRsxZ}K}ioKh=~8C z_gmpmFIHwKrD(Yn6}JL`r0cJ54MP+4`^t?<#J1gjlrx?eadUG&tICB0L-1jgk^QG9 z-PC7$LB-+L*b*%S8@AkaV{5+sY1L&;#x|$8K5>unxCj-nD?YTL*m(5%#X7=KNRTn{{yFvU98+>?d7dY`&Yezt z4^5Ty5&up#*C=r`ZF6VEUtqXZILQVovwkmO!%AjWWBBzZS{a9d5nI}9;G$`N8m^Az zqp2=dRp_v5Rg~=?Uf(w2-YO-GnuvP#o~~R7X&6IFbfTv;c-IjvFB$`C!oj_w<1|$| zVeOkN6s*BiiLxAqcKEU1JiJC?c^(bl77J*=uEy!6&Z86iL$%RG=6JQ`HA5nJK2?tG zg>o9;*MaunW%bdddPK39``1eu&m2=+YqKoXGvA?P7 z6K6%O%i>1uh=v0V-;KZW*l6^&%E3j4n$n4`Cl8^uvrrk2j{Ta{o2=z#tsGT7>r+$9 zt+*tsHO_{{=8Lf6)zPM5CE!Y6!6Z4Yk0*MF&{XZhjzoM#42w`6JJKj;wzo2_HyaUu z2wG*|N}@lXoSntkN`Z)uEJ!j%TTH$2JHR5rh5x0w(|I09mK4!{m`spD=-XK>ozt`< zCZzkglF}GU|E(3(Wv$XNTayy(Ra(3-L^hl3Vg$DPQv%e(^QGWYQZFRh&*dVX=2gQi z@^IKOr?TY2kO4f_y!7vSBi+L2Uv!7R9dWw{|+MLa-uGov3jUToWtC)c-@Y;yOK9>GmZJH>yWwMU7 z^zkFLFztPhvCQ}RGg*OaE;~#nm+)BmicOG~m9$E(;js;akRAv$pj=Im5p?;Wd%M*i z7p8e*E^CO;*Ds6kK=#wPVmO}j1NJ}LJc5NFgK1e~Ge1?RNZJT3wI#+GO42(4RqtZ& z1Evbi-*{i6^xw=Fw@(x_ zc4;{t(wVyR`968o@K;HJCMqYN_PRD?OhKLL>|MA2u@g!;5>5yqAt=3;M)sn|u7HxX zGZ(PDS=wmMa8=5bqNuU`OgZkpDqt#D0z*>4+ws15EaleytFDw`hQj5cr)lL?)Jxf> zs?`>;M#b5ZUr^FB_1=XepW?@j%{-Y<)byu$f;u(7Z2A*N$=iFB7PdUwfq2glixn~lMpU3eQ|De@s_Eogfx)j zoK}HhOR!x-ovXU>OqDr*Oo!`OjV|6l_ky~>>0>n^z*t~re144}q8V!}3q<|S!{DIc zktaI7-6h%8(B5aX7DTuAXlol-jAFdnvSKtp>PP2@8u!sa*PzK$A!*r&;jp1PPNIzX z7qYwO^AAk2xl%rkv#O*CFTsi~QXnb8ESF>f`9A-W?{W64Eq@DogFyyL4fUjdfasU} zDa=QVjda529QuN)B2zv_b2+YbJ(X)DP!+${aa;T7OBR_NwJ5Yx^VehV`KPT=TPbE% zq5J+w)FqjoQ~@rsgyu~DR0wxDmYUFNOz0~e>=%(XT=H@oNUDXXK?4T zne4s4amIJX>pt&$&W|(38T*GnT+89Jo;l}z&pWRBy2i6P$0^$w_cl6Cs(u(SN-(J} zms?rj^hPV3A^6A)U4!5ah`m|rhc>^8nCD#*?;$HQWw?$h)D$43a5UZN=0^ z>*q|?eXXiK%6iia?GQ8EoXx;SCuQbAGbkPD#;|D3WiPA6D{ga9=O0*;C96YZR2D6RaaP?&4>*79y!UaQStRPe#4Bp5_ z-nzNQHDbNtr>iQZE{m z_s$}}({#?LQGY|Cu^j`KJnm`YYW);08NVma8*j_D*QlJ@DmF=Tl_xPAf@%|eN{A8w z44-TT?FWmNQLFotZ0h%W>v0ibXf8k%yD^_LuQlc1Yon9lD#rH162dfpkV^R~9Da z!l)Rd@d*<65llU#^)it*B?jqC>g#<34v4R_{icd1GQW#!;>^#Iocty# z8R67RB84+=Gg?buMv;rG zFL>Xnlyt_yN$dz=4QF_BCnIt2;dUg_v#V@JREE}k=s5dvnNH{-3MC}Glgm6uQXCiH z`IeCdXSy@b%kJ?_GWF6pivkr_7D*gY0-RQ=*0rUtlcEQY5o7Z&?WtrLh1i^T22{Py z=RbW?g<=iZ&19jGz8! zmA0;Zei;@m*Ao&C(TAr8QraUN&CKbCHO#1yuRPA6&Yifk|J!UJfo*EjkK;Z}0Px6s zKG!hPAT36rN@Lsku>;UpyHDPyB{2R-Dro`}`S;w1D^HieO*r<+i61^jeQs5BcO3{~5!Cj}i^^Vb2?od`rNBngY?<obwe z@QN-`>}E!z<>_gX^^@gAAzkg@RDoPFNIO#lD_h#9bC(w9jAh2F2K6&&xFybKH{DrY7b#fiKD5Dt3-45^KI{;b(cq&*T{4-=8}DwQW{kDdP`I*T z=W1BqvPCbs#GHyESy`TLPj{}>Od(^yQoiVSg%q#k0cr8hq{8;Ibrcd=j zs61OnB7g>J?doDzJa60ox@IH(34)LtHCfihKgc(Z603K&ZPxVlb5`7xHmw!s&&Yur zfFC*$Wy~;mtXo_;Ho4dln71F)SCO~rq}r@(R3y9nJ2-@ik}2{>+s&vcOM3~?CNaaaEtJG1c7Mb)Z=zvEl$8hOlNrtmShM;Xx~%fwkE z{d{=t+GvlcpvLKI``|XDl_Nzu4yiK>-S5WhEG(4q2a*_h+dN^sJ?Prv4MZ@%x-t*82;LkQW@S18B zm}E`*-r#UylUZ)K%c$#1f!W0-^%DL(zvz`5Hf{z5@A6DkXHmPc6SYBym}j^e4R#DI*k7D|Aan+cx>1 ztKE#2AHvVoXD9-?HL9tg0s8Z*M&7je?bM6D7r0KGuoW^hEg!Tb1;= zUWWr~ZW5gtTkzV0E+Z;IytgxK8qb1%hp!-eAMllyQb>{M{eHp%5$EHXhucrdHrCab zbSy6pS2kzQ@6DqGBstk_G60?y9l9e`bgwPxPmi^1)q`(O=NPTe=4JXFyRpWsAxusE zl&it*gF^9Cb&a#_?G7Zhx~!k8vdp4iODSWw#bR@^yc|T|25SN#nNwZmL0vcdzuvn! zt;1=j3ntFbw^H{JHgZ%>1rZ~`s^KJ`J*0oR*+MLC)Ed$PaK#qyqn`!>>Ja<}0&ZIv zB1pVdS4Dhs?UN0|-D|+DirF;1{I3f}GkQr)+g$=aicg|HQw?um_}c&vD$Ovx5Q^(f zh|@$7jk_JINx4@po=D43vr3lo4%kHf}Pf#2}%weV9{wy;+bjT%m$+`;%nsa@B!HXumb)SC!}hb1Ra-;7?vv$Yp461u%PIYo zD);P0QjkzCcwL20osc}lC8_wFC;~9eL82@#@B!UJK1#qeY#X3?jhy)+02)-v*+SvO z8^A>MD+eh}e4_mQA`n5$=Yig3Rl~i% zrrhS`DITC-Vf7A^Oa)r4q;4du$G#dXctZvuwG-=yKKwziC#>;7ck9F9%)A2X8*s16 zJt33{VN{2hzsmWeob5FRS;edguE{Lcxu$=&gVBJ@2~rrQ(g&$Yi_>aj%XBckQc^lT z{e*Y$Wvw5e2J30?-Ljai?KhJiuRZpy{GG*Q3i`^6fyn@l=A=|Ww4k5JAFv2Wnm@~? zI)obB+YZ1mi#ehy=ZHN?;d8Ad=e$tm)XIjz!7EacK;U}Pg2b=BHnoS;ET_;{S^OFR z?eBKObgd|aPhQzS9SPrXT8RAAfmf`5-qr>wj@+iVovBu~UEx{Wl&+I_FLGu!yoI`k zx;SKQ*LIwC(e9Wc=aYq7PJXxvL*hAabE*^rAg>;Gho3C1uYK5uisg+*awR)eM_!f3 z2Vg%ymG{28;i{-XLj%9F^==UcxfAwxu;t?7nrU(8;^yU*7V9O2FxJ-tgt22p6XR`y^~a1?N7J@rsQV7#Is{PD_ome13vPYrWk#U|c8 z6>K2XLZHgqo65L1C{rjZy@7GXNA6Ij$=xfZCGsXi1^WaqIUU#6mfgFtM!;cKlr zRU@hw8`pD&0|fw@F1O;2vG@aM&95TLlsGM94~IJF2M-g6z7-)og82`XBQqSP%QY+V zcx$F`ef~0|rW_rs*7YJ{n3ys&4cDrHMtdaw=a$@&Hyxge3P)sFw>mO_t>VJE^tFUX zHR3nAW{u6_hr^BEPN&6>d0H;d#7d93>b|o(_)ywPv(xw|=;zqUGl%XB= ztHOY?OBr;iYI-3fM{9$5a1(=r@+jfw^TFDI3@mWd#pI~i*(jQczEa#?i#Zk=qiuK) zHxec;W>B7)N=}9lBr}*on@sN+2?P3Ts zKHSZ+V$l*#G~Qy0TG=}L!^m{(OLglj$y`l=6EiKpW76Bg3 zZC8*?NU`?Wq#7RdY}eIScB2n*|_v@?r(iqPpWjKc{!Jsw|Ar`(xL~vw1+!Q+J!A z#ZT#@1}+_Kp{jj2;AqItfmKVMJlOfn8@aHFlSKAiOZufw zi_9nD40AkYO)4czZL;0QdL(UE2OY~{YCX)$MicYmeHBMLC%c3V0Q`#C1&!``h4doo z@k%07?fjRH`x`8yB+bpzcUn_%^=X`)xp2Y$LQ5$>v%`uihpKmSHuj&SCrNL3+ZKd_ zzHsQ=PUSjb<7JY3Y2#PNZmv}64JAWtt0l4F;nX{N4Le}!#Pzj-0i`=diGnTT6bw;C zK{7AnNudZ-5V#;-ql)B$0$ThGLiJ~ztZGT97DdNk#6q=d+P}(fUIFU~1yht!-Diu9 zus_;-zP8t$!BFpsZ|IX&9HbHGKoU8(Iw*`-uNVZ+{n&a>R+iyIyzOm=)nmfDYUvv& zNa83c=9P}3quZ8Lau^gcnCQnZXC!P)UP0Y4g}B+MCuz32+HPD)(;H8PLx`PaAoN~% zm-=)q?KDp3L<6jC`~0*`U!3eJA+5nQ{WM+F@G_Zh{bvTirlO) zfl%a^s3(Vn;kI*%V7Zb?9of=2ITxI>d_NNfm+$;s^Xsd&aSf-XBj<~}5*bY!6t_!d z9m@J#`LYP8^70(3;Al%YDPl2zOR(y!hmzMg&IkG{msIx_xbSE2R@G9-Ko)jEyCl7TZoTc1py#9 z9(lrJ7F6QjeiTHDRgD0lEIkPgx`K8v4iTRW-`fc;+cM9kcqA!T_p?Owax5xoxo=*q zhR}yvT*Ej0g!I;BUU4-=yWd=wkvl({{S)^q!&c{0##|PDBytK3-zt>UXtfg4=(?lA ze;Mrb$3{OSwKE|cVHcVQQVNB>a^)F~P0)l*l>Mb-~SQ9#6{^5ZshGf-H zd5aqoAPq!!<-FD;v8NyxLqT<6?2_Y3V2aVQD%dX{j(c5=6-%y)DDlD-jZH`R?eryepH5*aG;Usp z?8hqRZ!z_JBk(v`Wzy-8#H1DgtTIH}{+-36sH6gYS-&N1fA^}HJT~b>gF2FyJX`6( zq3F=)HW4KXYETQDeQzt=ACnv;`q2DzUKNts-fdw?uR}09X-c9r9lg5<07?S z%)$vLhryp;I>-Pm$No&7U`4b5E*MGZ?rPy~9mE!uq_tA*xPftBt@<_VteBmrIS91y zWm1&Sph~@TTJ9j`W_=E(SEmuRfE$vMayi+IzrL+7qCx}H%UNKOfpOaH{$KqIW+1luY+l14`LLn`_^>9 z)Gyus2G(JlUGna2Q$hwQudopwQx#HswwxlNK zJw-6!uu@vo#ox_JQn{jGi+x{PMpbTWYwTo7a}m>m;`dC7_2%;C^(F& z%FIK_DN95c2d{X#@io7`ETtNV#Yo+{VE?zk^-S#hz0LpDX?GiMeSDT|T0Otx-g%Ed0p8 z5_-{~O!{Zxo6j84*VBr_C3WNucN#vDjFJHeRI(~9>p$o7in5(~TG&P4V|4OA1VROv zFN!{j>cOkIUeNdv+}0~KnvAfM=d8~Cw9TI&z~1v#;wHQq<1~~0_)d-xK)=PeGx}7y zEEX{q$2RLUkH_w!Yq?wMRec;b2QW74vU7A=ooHK?JSZ^~0WxDs6Dv}XibUT3MA)QX#rdY-J}EF8`s}-T zD#M!qnkd>hEtOI6W#43xKmc$*M;SrvnCS{sK7&QsSy))yWbbG3&Ut^3Gl2UDfy(M= z;l_ER;DQHFwYls`IHO@!jr-mYDkTJ}fuTmz(Fu+)HO%(hLPQDl%(fhYn_7HTrL#e1 zv)u2pqUT7Mtzy51nqBP)cpM6g^oL_`6A>kt8ecd^7%&!EJQ;u9j75=&_33gnm2mQL zRoJP4un+d-X?XLRp0ASfHv)jdnnWqC-Z{8IZOT#o2!QFQ{g(fnws)LZ`x$S`?m1U) zt4P2KB{oTevuI3M#9*7hU9bBi6ng^|^--TmF9D(_1ufD4Zoog#kihWcSU$E;6re)*1XR90%x5~dPCOpZiyU!TZG1wf+vY0< zqomHPV?uz{+0{%4#a(xrjw)XEj_2l)sp++VxhC^kOBGK;yV_LrlJEM=1;j)y63CGED$zT>&p2@Q?o7A`u!pAQ0*=E~B7 z`KzzGPzbdgl;$&3H;+`!gP9xGj7jSB735|M8eXx|+#Yn3n$=Pos1b0x9T#GAr1QBd zYAV6(W&BZa3IS1>GQiVoY&Y$-<_LIFFeB4BKB)tx#aKxoXrRK(`Ya(h!c%;$9uxu_ zD-5%$bMuMOS>IKz_@(W%hF1Uj@Zr|ZFv4Im3a@kSL%l>wtqeB5#almVq>q zK&XQ%hD)0Gl|v%#GnZSI=j*iG4v?Bp#i>pbTS#dYbzPHoBz?oBlq;K48VM_cwx$tP zYCdCi*1GyvKgKmwUDZ5^}L~_PSosS)2ENMC}QI&js4`n%R8ZgXw_JSA3hC@`ssfs%O?~#AQ(R#u>0%h_;CK-LYr>m4;XuG z0&e_Q9j(YEre7BZ3y*nRplb3yIdO!H@6sxsi9g!5vVbM%Et3}7%NOJp$Yx}CnewFa zTxYSizeSWV;Jg-UVzEl-vM(5vw{O*!3n=Y#y0J-UZx<>0PNhe@Eb?sHgyCYjSju>p zxImz9hOuWko|EgOkSFQE-AzlzEEg_~td&4!tCX$~bIF@?^+3p$<+2q-XE9%ghB zF4xbMzU9I@guF071pB z$&`MFk<>o#JVOugg{D@XFs`(IWJYITeQDBnc+xcS5kqTH&zvtGLF@Gw<)RoqblcVj zcank8klb^RmmcRfaucWlFxr9TD<6=^Z^gf?u|FPpNl(S&c;a9F4qJ?mHjZ`q^^W;k zcICpB&gS}C+C-b~Lf$p}T3axxL~ycZ`j5FU+=}svUeTAJcqGpd-3_JFP2E3Mc?vlI zQYByLS@7aLK#;-GJ>eX0{@@mR=xHqD=W})Dx22eF^v&bO>oXIK&+J)s={K`G=bz7K zvphv@Iwn3z$z;6let5#s==_3^=hfa-s5YKG!ocNF_vl!? zj$QkiXM7^w*RITW9Ny-Dj(rS6qQ*o&!)j`Z4Gk>cCld&rLT;wp-~HwZLqan?plOzF zJno!`)UUkzyf)Yg>%v;J8;DHnJPVJUr1pG}T0h@RYv^*GRI;n&zqwWh=vjV9o^yACHpWXqq&mi6m1l&@r6dDU%{E-+7vwubrl^mE9{ zI2u<#wFXN^!;RJG62)~6WzzfgFSuUcI^2Gv4Vom4QVYVa436t1&c0};=c^N z;xZtA?+_l?CGU$)`gSOE4V$7<=ur$cB_6p0?24qMf%|z9APX4*8A;xhEkLQpu^CCF zY8p6OUT3R;Syof<=Y^X6Y+Ly#UUk0QiR4cFeOln=S)nC77B$4zx2`T)@ z4L<@1o9Nt6w-r=wat~#g2fP&@?jxZm_>)-JetixIHzW|RUdZ}r>oO!TuJ)%EXo`ds z>w;zokWhT8)g3h8!_C5D`rS&)qg91Tz|1{@89t^~UJC}oP_0ccku<$Kd9R3b|13j0i_AyJ0qYl6*E9}w z(>o<5Qd`o=QfgNdy;mKszQy0i#P>Y(bB4%|VWh`Y#p`mER{#+xf? zm69(7p4$db(9sho$X$tSSG>ga>GBBUwaed3_`>zjIM^1K=PV2Y zVkrwcaDdg$2_stit{GR**=t~srms9elCgYFiJTGv-Aai?Kf5%~l$9JbM?3na zDM0k*K{jMG@|wqH_fZ3lQ~pQpIl7jolocgiB5Rq&VE~6JIUZv4_6(yU^6P*bM-Vi8 z8Y|6ze0Fp8thX3&|A;(w{Oz;@;FH4uFN_^O$RvwFmdx`7Ow^1dy)d9N9!Sft6l|#O zc6S=y>Py@vaRw5Ybb7tY;6Idoiu3wcAH`6MyYu1635V+t(yyzLTv!uTKG6ODUVpg6uJY zNHO150K(RxF0Hcl=`Aw1BbK<4hK9Gb$CC8AwaRf!kGuqJcTo3Xoi7%9cOG#!)xT|& zvXy5io^;XU`)&CF4q+8I2E&XecK>Fje)kpAJryDXnc?rCIsb-gFo_m$zqxY$06Y5| zipJl5UM>VU2~mBbp8Uq;|K3u`-e&-YrqyY(hrc}(zk5}ouLyu-iwRDc^EacDziD3G zFM*HiJ>Q`5r!+cbB3`Gk_V)JsOJ{j;5ks^mG8G3<=F-Y%#C@VuE__NY8MC~EhTQV^ zR`jc}7~q*2-2Q{Z@m1svf-JRC3U=@hK&RQ$T!;zsXDLtvPZYi9_L-lbSHOv6W2K`D zqM)Fl15(w8-f{j<*fic@laR0|^^c=Jew+*F{ND-*!8z{EB?CN|=@5$3>)%_^pDq{? zpudixJQnb61Tq9lT^Xse5f0zplQP_@5X8E!VruM<0z$k-Fz5~>1c$)w-j7q0G|-;& z5ec=|4vHzI8=>9>6zBepCD>ty@n(6!PCZh3~PzIkIR!*}$8(y(T|NcYQ+=#E%(kFl!e&cPc4$$+|R5I z_xGP`;o|=r8~@X@NnIPa_kW-TeSdy+d4?C^GWssiN7K>9sVSEanu zw^bNS-|>#idY-D*t20YHl4K0<)0b>z|Bhr@SeMKQez-jReqERJgH{Iqf z*wE0x-3cbZ6`_re8-S+@D7VQ14xoyLJ-{wE0vK#p>d$te(tz-cv(6XgPhaWZe0Yhd zuMDuv*!Qw+G5om9nsFgGjQI;q$t|FEIRx%%dV}v_o!MC89n*jYXubXAv`k;|^u}h( zWcAsx0guxLcb-f_6pCwuLU;^5z1F4Sho;6mCLgpkcd4~ETA;>8*~ay4c_TXRzJTCb zm76OiI-=C<@_qc~4`I`b(p0vbcfQqe)Y@!^{3k&X+FoL7)~(q;O|^^OWUwF0u54oq zwgx|7m(6Iph(SyDmQw3Hx8~a~*w$~>4Q@409C(vFr_Dj#FAk=tfXzf@W|FS-rzWbF z=~%X6eM&2k&l~}PT`)5pplJ_6!DW)jlTOSB!ihbv@T@1k;8%Q=B%D$KrOnB)To6$A zzG{+RI-H3DeJ=*x)@IHgIsqVdad~sLU~~y+mwWp9`U)Ro{;6gEn-480pXMOs?(f?$ z0KhttLOPG5$@TdGu6TDgwP;}{u6Y1DsZ~Wp9>RZ30qZYt^@>zYYO`q=$=$Ijm-+-M z%zyuMX0mVH9X?X?2m~mf6#)K2F%fabu<>S{d7Q9XB>%)JCGs@gQ--#zd5E!t*GROj z>M8+O)iq31B1;eBc(yHZqwYj4Y}59mxxO(0w21ED855E_k_6E6%L9CLJ*(4CW&voO z$&vbK8uqk_*I+S5?;N17@5_d4({XU7L!VzO)*Gr1j(Ppd-{e=0RRcCq1)vPLatnGZ zMUt^JW+W8<=BWMYSEm?33h?tX^5><#{1)$WtXNqJ}sYoir zA*1&PJ1(PxA2ZxFTV^sD94N&Gxyvh#rp|S|+8rtvlD} z?DBar9vvD>i}r~tE6>7$K;~}fvL+)dYG06#cl)i=l2K2rOmQS82B7n<26?1r-v|<^ zTPU}s(dNmg`&se3<*0cd$gc#?Mlb+`$A7UxaPk9I2vDVdtqlFa=k4vypp+k`S?Y2C zQ?4|WbUWWChiEsc%B}^`18k(MK71_ZJvCj<_GSMN8?KfL;B?Jb99vF+u&O{bRaQAH zrlLS@P|1}*0T9v ztN?b_RrH}p#%$b(*J`Lf&v?ZvHbleJ#od}Ck}{B$FJ3q=KtSUMvnxeQ^Zx$X*xDpz z_1ZFJvK1Wyn38ZT%}j0N7$|8pH&C;I7Ldd^PKOhQMpZ9Pt*njd&3PqvA{5}aNJqQ( zcA8<}zV&?R5jkh1&lKq?b=}*6U zB}VrBpX`pnefB9-HbDUSUf=nrhv@6quIXCUo8u)V?buyA34n-;SinPF)_M6kfTF^5 z^RkdWc^;>@-(?h@`t01dQ)C36H?dGVk2~k(t9^UFwjG{0W)@c_v$Mb_SObYyVg~tt zg^OwTSW4W`v}FI91mw1{wS|o)$NT#x(=i>AX4rn-QJqdN6cTIpR_u%l%xT=%&Oe&5 z`Bgf1XReOKu)dPNiZ2F`HlQdRjy*hh55(1N^$3vN+G)dkD;>`u3J=ed4`Arzw=y+) zAQ7&%Taik}1`|ZP68O%g?;1?NEito5hW2-_--o`>@W@)+&myGa=*R;nYRuK{^>f4I zliB=nnbgUawj)B3$cc)Ti-hGf__DyYJOl{oj8byE0VFn>IOOdTibhyUTFWkG|HWFR zj4nqp=1Nb@Na17J@~#{}zj{wKhw|u2qLB`YnuVt^N%TR4qrE~|el&~-fTuN9EZD0d z0|=2!E%CgM|BDOYdQm>oH1NmSn8Ak5&c1at%bIXIMd7;hU9TdE6$|@@JF&Q&3+pZY z;@CJIyW==g1Me4YJu^-E)EJ}c8Z|JQ0XQSHXlAuQ_yX8`@oONmE*FQ?7w7AKI*yxT zS%7iZyvhi;;-#@tZC-pZ?135C0{~`?h;uFOb;C)wgY&Lmp=@d19>;byPp#_RLUnSa zkJtU4zEjBYrwb;bhzUY|9T!A^NFEgRl%t1q*nLmo<1dOJox8KT!!O)NyWGaJ+eul!t!{=!%Nj;mJjb+Mx9pi`JK`-TQu;itW07F}t#GE3WrJ%@&v=ovo;m13 zbr)M-as@eGt$inKVzOwsOtab>My=&%D1+OmUy87&&2Z6o(0&w+tK+g*do~miNaFUx zviacq@C)AE`kYzEN$o%)puaxI8}5}i#0no1?!&&|anyjyQvY5e?eDIJE}{fM&-(GG ze5Ah9UdvfyqK;b+q;b<&DV(`xOmVk*RL*`<+c|yKbuB$-X@GZ+ukI^C<|Efie$}nv zR=5rHz%SdY8K%7xe|#_1w!b>sZMq}ulE}d}t)@V2HR-MmPjc}%yC|ZGP~~8VkzK$= z@$tYIKC4rovaH?Qb_zbPONZMaJ!@HSy$L1ES-QUJqNQQ_(zGpO)qS!iId!x_bEmvj=)bk9=+{8{W2m%?c z+6zW-RHpb7oEVvTTH1Iu1pMvNou)?N$D|3~oGKrfptgYwS zTCOYcIR;#FHX>)Snb)~Xu7mt%X1;3) zhfTKeQF`y(_-vwyv@u#q9(HTjsRs31iX?>6?zPR5dQDp%Rc+;rIxo0*)j@ge3tHn4 zMJpGXs(!+#RyVNcz3)8hru0KafeiN9+-`O#3xVCaKc3DydrHjxI$;v0J5Fc}-;_2V zN^vcPy%g)a{*BD!p9I{+*@`+}A9UZl*?#7>>qBOFn_l{YdZPr5 ze4|tdHV)anzi{qnopgwmWoy+Qxlk(Xf3<_`Jgp8t0$d~3&fOOAU=x2t3$^CF{s}i! zAO|mOaI~GVtZi*#eyv!7D)*!kNkTl$J@aY@Cvs~t?KYJKOIj<1K_Bip`K6jCd`)`vEEcfj5_n6~4FYb*av=!~B=z3TAUvvVI>~ymyIqoD zBj2{4-?aDh39O!q*@I6tsA}We-5QD8W{t_2;*so@SQ$@Xs#)cuoRbFQ4E-;!%s=KT zrRU24_jr=+>AqbNIO}z_gB4j7?mTB3IBQk{$sR8%$e1B^98<807R9w?Z8?tNJ^VpQ zi4jXYYYsmG4&jnmBndHe*XIwkpv0ZqYvKRJLI3GF^ZDne%alW0@FTJE0vi+B&d^s0 z1h0)jF=zM^khqdOQmh`aYGmQ9cnlw-&3LW79XeXCbgJR?gWWhV-Ls5$ZIiaWW9g(O zA-U&fCEr_g%Lp~fVK|l4g`U7)QmMib6^baqa~C15oTya+~;=75@+BF5^Hi}7P_1s^jaMdxY}OV;hJ~JNww>xkInAn{5zf| z4DpgiXarkbnzg0ceG5`I_oZrJlcnW&k-#Cp1T9$I;IWZ0~mk8(5CF7$zhbl z$MaXD4N4%4Nq6Y!U|utMlPjy5+`m&l&u3Lcd`Gl1Yu4W|r<2O^l=SR4)Vk5}w1G!c z@O_^@G{59=y6{e5Tf?Zf!_MgNiyg!_WvvGpyxVE+&=oF0Lfu(ib+4@G-4f|AJ3^0@ zs^Qe@o5jPNJ2xn?_pr!rMn%r6jTv0IS!>t2<>t|u&8P)1w=lm00+8X;SUDckAIGkX zk1DS~n6~b9QleY4onjEdoo$(dE2}pu%VmjAUb6D*4l;#p25!N$(U#E%vPT8>*vqlht&09P@%v9~LSIZkJ-1?#$SwXhpWXhr_=1K^RXF@=1_dOy{~ zA!c~rx}JgArAE{Iki8X|QFH<8C}5>6DtY@~+^K*5&ca9Vh!Pwxu!sHr*3)nvkqH%Ns{~>Fjh_gI zXeJDnR`#(d`)|}F|2%SjD+-pj*aQdm8}Z#gUvS@)r$Ro;SRxp*f5ZR5BP&OP&I5y{ z|Es@#uEkd@}Sis|HaX9(|Ahw+`0RA8&xA(Yzyi{O88;yHC%-dR(xp2mBph0EftiA>Kph7tJ#>A;TNrm&8vWCr}BkVkiBAKl+g^_GS-;~sP(`-XC zp12pw!`(qz!-Mo3A4Pc;o2R}G!S{M4F)?uaj{B>hleS=09@~=Cbmp^>O9tsk@?Kql>rxJ4elB?l{OTP_$SsmZ} z{IoPzBS={a;oMs4EztvCM1_z9_?=Poq<^>HbMcnPVQ@t{rxdkMeT0yZ_p?iU zFoJ}Q9z_W-m=jQ_`oMWM&bB~8-hJ@8_>Yaqs0g^*)Cer`*JQzP z8??bFvzM}Pg4{zZS zMw`vmRO?*<&`vl<0m4F!gfH~pG!F=SgB#^$@uxNq)>m}FrK2+Mo4%cfwTy#+LGRn6 zT|X?jph*oMp)zoxxy#mTPIk|aSBjSzd};xIJ37&I+n_#6-}9gqFagD`-SwQvh)Iog zna>t?{6{}uLX8r&aTgp)AhmYpWi3wNVfM&{77(^tf8A>so)u_c)Qpa%eRDa33Ukz(xHViF=__5HFzx##%`9pQ$`M>kPkE8Jc1{2Mw9m9?T1(<)Kh zDR{C5Bw9$_<&a-@9RV4H?uaSB0*&I^FvwgZzQM|$7Am>PD9t~^4dQ4tRdrXZxXBT# z59BY_*x}#Cuu@0V@i-8z17-W~@*JSe(FE#2G&;svFMTU-&bZnx!kb_rjIh*VuSU=; z>fU0C6MtX7Fxj+Io9e#kj|Y3m1B$&+P(~vtLvI(>9*I2c1l3+f&YnT9)0YL!m8Dr< zc=WY11;d=Dy`Jf0p15Is72Grex~Yt%L7`gX!+O4F{Jt z%-4(P-ZpJ_H+xI7))ig2R#3;u@cW(9LC_c#pTDvpLA;cNYh%eg1^PL=t>E8v(UOn6T4^92ElOXQw-ETE%>p-+`ABFmzvz7S})Hfdx^{fr39~ z*`($Di$OD*v_1MxJDkq+-1#5pARMT>g74`+klkM_jTYa|B1JNYh@zmQ|IrE#n$zoLRw5t506v1xs44@ataJ<00hg ztFp{`W02ugi6>MAUU`PYJ%U%n^=3Yrs@x1R`fmEFOvOn+AZ@U{rH|IfJ8rNVaHa#T z*HL`_i{;qi1<LG`lB7vCL@ra)aqaRv?4dn44W>!?AuDjPd~hjF>bZP- zlzG>=r8(q9d&5S~o$7jseu}MjJq9 zZZiBHVkdXtD*I~D&nkpI-v-pi6Tt7i%Dyr?@7aI%0z0ys#QkRxymIZ^YaXN*T!eCb z2c5g2y4U)hFX-jcOFIORk5h$a@X|*rZb3h3^i{~_tD2%gP*a?9X<}ZM#k<731V#0b zk;ajnk@LqdfBpjRCBQZCYH4aEXCYGXc1qh}SacV3jU(&K;~s_k`kngB^xodGn@Ooe54D9JF|J8QY@bmiKhw%>? z%Z8Ml^oIBytxa2)^*%|MoDQ$38A|5R|=%dFydl4-!b} zSTh_gG79;uH;W2RS$WI*Oh*IZa6G#=p;S#8Ls#N^Pp4;F12jOp&eHkV{B-oi<>*0W z!*(J2X#d@cvD-pV*|C@Mg~*Q4m6v4x)fa?@cMvAwon%ItUVwj#L5ZO+jkt zAfY2QN)ZL6gc6FtLx<3NOJL7DYpq?6{jPWa{J!J;tw3_$bIdu%xUO-IP=STp=gP;s zK)b<$s^syzXmr+2a45YQPfTn;vmigx`)DQ{A+5Zky=lE&vs68IdH+Ea?DT!r=zws* z3ZrRv>6wi6qcL4;0kLO245+P8;V%t~Y5bLcY?e-zcw-Z@_*@)Pg$xP|9-1h4ecrJ# zSMqAj!RzD^S zuVz;N7KbL48~SULOR8-?x|BRq_{mN45NMMPV3A09FBM%atGSipl~_K!=VXdQ*|u&} z8)TN;3&d&IX?P{zo@`?8{?M5hrZ-4UTu&}H@IObc=r)YRsk+!BsN)&H=sUy-(x{nE~Xoi9E+u}Oui<030HB0`M_Eu*1P^> z0p8-WKV{gWYi+ciC|B}pY(vJSBU$y=gJc~K{;onnm(l&|l+d;2g7bpf`lA(Lr_=Bl z!v?KQ@0Q)Nb$NKTb8B3o_Tt1(+&jz1jh>W z`w%BudUB7(9VTkm>eX4&x#ZOK)oLWJb-SeaO6#EH=}+Gs194{l1^1oGe>5(`4NkX!-#tMfHa81s8krZ~iI^5_H=aqSJ(6_wpWIXv&i zS1Pt*gwo(clS@jM%0P5i=6_R)W-iS1wihsJ+-jqMO3uQy;DG|oTWuR)qjC>8otU1T zux*muF#K^CENLOCwhM&$RhQnDGRh<%!atCOVGNZl(L-AI`=qN+H#|{H*6!P6Rt!lZ zgkwjWZ#*ws0$De1?+m2P2&u4+N3c|=Zusz>bjR5kuxouu{C?o{WhU%Nd(hE4{hM`@ z9?<&lj1D2`2+9y3%lw>tGZ(FNU>7VI1uuuoa-sEtzAxsYAJi$$O+%d&u4U!_@J{ie_v^~^I~aTN@#}2q;W_O5#F+d1N!;h;a)BZdjn~1a8LV|W zC>p^Z2iSRSa_{pja350gD6Yym{w~}wpYUc{sA+B1$iy<#)(?1UJZOZVx}W@y+9BU} zdjb~tVXo81y`H&~raccmv<&@e9~}GY>3TyQKG%LKfL;w|wA;@#R^j0-`jm52>{>w1 z78RRJH(hr4%f2gj!O%xLS!eiYHkR?lsll0HW+|%T{eByTu*5Ye^J2K9acecuk(z{o zwZWr!*?7YD8)yYyt7tcqyy%{-{OWYQ9`@-$i6v|WVXiB!RW#v5E*^1zh{qRnJJvUS zB51RJ#vPC{FV0`4JJF{)B?oL)cCkFc4S&UkYj9_XMsCtP5}0vesaq&(HBnWr{!?w+ zT7FbQtzmtk<#l4v8rQT5FksA0p6JeU%N=Vp3q7fwB0_-+GGrF^NDWy02R5)zunWc$2=i*0Q6#7MqjcGi85;fkCC|R&uv-zTUf}avxGV-N> zP$?iOi1H37Ew38Vd(=5GNE$b^m?Noc;!+v4Ja;F8x6GxZ?(prRy!4BM7%<})zP)@G<_*k1+6pUjXHT!*79KG7QQa440C`? zeGt2D$UG1WMWyZ@DGx}zY_Li3e_M7?xwNHrU@&P4x2^Rvn~oDD@`i;tUIA>fgfGR-IjRuQL(_P?`^WFs@)DWtH<(6_`P5qgNI zhe9%!A@=6ApM`C1>hi+3&+`$&3n$b#z)+e_fjZw}P)&%bei*r1KtV=1RKyzUUZ1^} z*Q*alms~72kE`f{#OF@BX4HifbHeY`JKP+A`{sm=5NeD~)xDCBqL_nsS`aakeDsN7 z8Cn_IM!^L;ZYZ}E2r{xU#N3~|g(Tx-l{U#3^5(C%1I9KPF{@pzC52QXJSQ>N_!()4 z>_jOjS}znsm$EVjd&5v?`^X88*|CN* zU}W&I5=zPhZSSs?r1(sn=`Pp;-HRb(8g92rzO40XS|czo-h=bz-k|RrK`3U~6AYaX zc$B$eya9h&Uv&WPxi`qJt*5nbL$90*ry)xntMLaDo_kzY$hkd{P{j}aZHI4TM1cC* zbIYsS-Me6h6u#ANkwUgRzirnuZHNDoenCTV6>OL`?^s3m7+NNV!=B}yN*-Gt2?RG< zgzw!Xr)_L;84g5-`O3}ncG>W{%nMQ+yRLP)4X^OHae{xIH~iIdZO94~1) zx$~uD0(!vD;pe#P`JeWJ~E3x(+l@^IFb8VV77wM z^|e&f|py2>U`qowISrT>RI7;CKG`}%1NGInMNxxRYQn@t|w(v<8J))ZYud1fg-?|DQzq#4M$ zUo4lP!0m`-O)ME}Ljo5Ao8m{<*-q+!tt!D|$xP2XVR%$i{HINI@8~NfMPq812|tY@ zkzSgE>B~YZF56MdaOSX21#N|LD({)IRn7cv#Ty2`SgJq0hWeglLb&led%gjY#8bqo z7${Febk1dS$|a8tC)GGKTJ{izV;|k7jj79RiwxqkX0#S?Y3w(~1HY%8yC7cwTSo!+ zAcIirau+&Rm1PiPY-y>ngzd=`_FH+R-S&;04TNJ;iZIPU<;e~wtsxiPmU~(2Ssk*o z%3XZrkMoSr^l2v4dnF&yE}D_tk!Wdad~=9lDJ|3Ciy|{Re3aI&D~v8WuKqX$r{Pa` zS?v4DssL&>+iI`e;Li(KXifC|z_;^Lj7H!*q|o{X>jH3|bn}P_ohV$}?}j;&dIx0f zT<8f;3d22dNushPwNJjHNL{VU6mRSEMbVrP8l7S;UbPvq_0`FK5ggO0TJ2SqP0hbM z${L+{d-pKs z0&k;bctDD_)NS;jIehy(_xR-6gFpS~pX^0Os$@O-Ms))smP*Zaq2EZ-gCgz8Wq_87{E=!49h-}TEJgHZ?Ac`lUYw;%)dW3rL z#PMU-a3|H^0mXb}?DifQp7Hye z)4y)2Zm*2NOn}=N24RU8!++p=>aROv5sG7H1N&W!vY(8QYhr=h00WII)it7$po$=6 zvXW+#yHDOqdUVV}t<(Z``f8W7L4O2?9&@pGe5uI&+gRP(bi#PP!Nc%7!+5cTh{Z17 z*$D1jWS^k+$#DO>(y}VeyvHL3u6nniTl~-JqG6=(t+#v^Ofr)T}CT)?v3^z6%pKx)LFGX;BhjRv&!jgp1py)d6dcz^uTx)IOy{ z!^~pp?LE5pszN)vDongOT`R3rvPaqUh-|h8oyON>Knm$HWV&G-xK>Ib)tZHZYeH0} zE7$oLPsViyLw`%my&u>bJ!w2YSw+VF&OZFOM)2sUO^7CBF}C6VqMBqHhqdyZ3fJ^2 z#&0f{2J4Dqv;y-SY&D6F6Uq_7EW0VMIeB)bThg30w~gcaRv!t&UgX6OiV7ldlik-2 z)((0{8dVL@#xC@a`>j{;xX(bRGxjnJVonYf{DIoj*6rE>b~Prg8L} zDAj|xRhhoV9E)~ILS*Oocs2Z!own{;A@*sgYbPCtWmY*;Hf z%G@EN9ZD&5Lr(oZVZAF5?qIfhiQa_up2~om@6iTn50gr_B7P@Xat!)dvNm9Oj%qj= zzmw4oW$4c_gPTU@`~&&O$&o3ca0GgrwaOHr7UQ|eLxO(@-9o({c{Bl zaT1U9z+Z*$z=jW5yyoL?#;z6B(bV2p;Sknxh4>dC8Y50%kx67xCobbVxss^e8Y{l6 z-c^*>DdI(lMGG>~F<&yf!<0s<^s}zDyBtc_H%uu|lxIcFcX#9z?)Io z&ap{~|Ee=wSw>$@DFb&`3-y)Jp)9_s#rzN6?emxdo*a8WmGr0-hPU5yrkYTXB9GhC zE_u#moq=m#>)hO;E{eXe0y2`Q4uvpHvaR>`d*6i!)@g;p>DmNBIF{X+3Pg&Zv4{hW zGdDhwv#9cVj76@MXrx|z9H+eGOvsH-t$VfCziS}^O_cH$VJ+upr|bPi9D2leE*S<6 z+07XrX%&Cult=HD?84aWEG$s2JQ}=ZZ|qGDx2zc{-T;vHHSv`2omF-Z(Q>cu(j46` zhLhz+%cVd@LoxHpkTWO8R(5c+4Dfhf_7>|aDnBZD2m*&_4+*urZ7;(I$ZWFO8h%O&EY&_+Lmo( z-0vB~oFs?xYAk25d0|HtLPZWnwV$5kJuY0r!i&-&2PKb9L}NFN&a_v%Xy~9Vu)0IF zi-}*<`&<;5tN3gth88v8&hQIG&K7XF8XLyF>Bju7imSQts$8!|!g=Z2?uC+cp6F_^ z`NC{?yH;_|gfp4V9EHhoKA@*UZIsnP&o2r@$IQe!3J-+IZq;0m(R%@rPkO>#s>EXa zz=uI=`sHC6%JQN4Qt3vyWOHcV0OimkLR@584epzAkRb~9@$0zm|DlXZwak2Kqxi7( z((rULy(C#&LZ)5tHSu?<5A{w{I_?pg2h7pYw;$e1BbJ)#p9_^dcG1hsjs1nAex2ow zEnc<~f~13CjsFbE=N59pvxI2u_+q^2aGc-W=udh&nov-^S%1MU4jt-9#3* zV3R6wLk@eP?b~ zmyM*s4K#qaTQ<<$JM0qg)=TKBz*$1^tD2%Y$a{PH(H*{ga;ECug%h%P&AdG74o*f9 zwki|~(gWWGA-3oBk{5|};EzzpU)R3K>sl|Yo*ZYv97scY41;4=U&Hr~6xxkX6%=*J zq(SPp*Q#0$8viV5D!TgDX$3PzJ1|fBDT3pe`Hx2o-Y6fGq-c3T z9eEF1AL9=|w%vR;=zq^OC8OFY^F91;rG2flocz!v4g?a4oX4$UCU+#?spai4 zB3O}?_ti_}j2TeGYjCgMOH=$4)Xhy3PcPWD>OI*_I0EUXE$|})Q_p8ShfFf~BI0-0 zqbte}bhIG(OT%i>{vchx1+OYua)sA;#_z39JY03-`K$lOP`4T`TQUY+580!RoAUa{ zeOPZ&_P2T)qpM*6{gAN8PtHVl&?VDOee9~IYnGU;0|_nKK(6lJANn1TykuL$A*X)( z)<|f(2TQeQ{5?zR^W^BkB!;LOD{SzAGlymUE^O{&t(@l4KGyFu)u!X#{YkZT))J)? zh3#S62LJTeav)<&@%KG2UBye*r(gJ4p0z*jwL8jw{C0NybBt}@;Ocbaan75+Lk)s@ zTI$0W@X4uetY(y^O&py|oL5r&WMn|j>}p3h5Iv*8a{W;<`hu*)AcTXOOG~c6;Y6R| z_>h!oAW_vwy<{kaS?)}1Pyo~|tnK}0bTD5Q(;WW%8`9}=p^)$zMkzU<{QADxpVSVz={oiq{rl6DeYf@xthu^+|uNrwm5rPA@pm zaTU3ng;~zm-ZyHS6o?WfsGjvVwFR}_HL~)#Ny{DJt%%}ig?fp9`d%~**jp#4BZl7? zc(kr`uG6}r6j4H}=4q#}fBYM>;qF_JFXHc+&nl(VRF?bSJkd3^Zp-!Zt>5t+%^B*A z1)Evhlbm*&&3Q({{dR3vmOMYWR&0*Ef<9&$zj0*&b8_={5ZJv_<_bSNkBnhXQ|TeB zeB8+J-z%Rp`H^9-Vd4ApZB?l-Bab0>b(Pf@hP`uG`%U(hri|Z%j#QU!uaC#HKS2%^ zAbC?GYc}(WC&D>ita8MPI(d2@=sFe_PlzrwU8?0;zPRSBpzrMPVq&Yk)Ol%P45oH| zZI?D6Qa0-?mJx{d@aCT+>MGqKDmY_^QUR|pJ!rKKKRC%oE7G42KlD05xLXt+yk*~X zh<=eD{}EiW9!z-rrk|y91dXR?)&htX%gC6c4;ndqS}PoVfp@cZcfL|(=c0^Gy<~38 z3cLsbiPT8*f17)--Ba6GSg`c?q+igHLy%9N{Pa(Fb#ztv8JiDvSSsO!;X5k2kj2GG z$I`CN*ff>qP`*+(_L2!29nh}2$C;ydWSF9D^nEe*-gVgB1w|ft@}k|Lyva@tod6>UDQ!N#0b&fVoS~XqLY!h7HT=S9MsEu(>sU>SWqW;J z?VuY9?3UVuEXqNZsVOwia;Yf;fnS`kT#&STaaU`m(B1i)i~Q!$SKF@o8?2#rd@C1; z%sCwI9!PP6Vt_=X$t2(5cgHV1GAg_K9??qJuLdl%9!6$LFFs%#M9cGf^Z8Ql|MKwa zarf{aL&1NCcd|LYqEwMERar5*BaqhR$sbU-Kc`KcS25sAu#GSpN;nmzK?4tBz%Y2!_UN}!`)LIlk%yqBr9C9`*QS8K`$nPYFF_NrZ(qWrq$BYB=%tLmBKg3Z`!H^OVzs7sQ6tA;Z&G-)!y4=`@EgyaEOeRVC&L zr&Bht=zDa(X>;(olY@R^OHmT1LaR}XBe9Bwn`*&YuSBN?VGlQ5dH(Bz$bc~ds8Tk2 zbS*ngcTH4GCwxrrO-2^}iv{pSkX>F9TZH%nIt_2DnMeO3RFV-Z{+zkwOI91<7kFtu zmQWkO+T1A^N9r*-8~@tEncvSsC}lo-X7>&vA#I1{Ej`I~oL{!u31Y_lByA*_R$k6- zn`(CjO|Dfp&(g9aBRN!`flpW$F$m_*ef}+|U`-6GcrG|4O3+W1on1# z4B1m*HL_DufFzVTf4v`YSw_&L7SH@{8`9TV7iBgZ8xmtd~UutH&f>7S7CW4clJ z_yczwFnX;IeYID(R$zZ*y!@|_2$J)gEJfW7+2jD*dPK61&)JN?A?;60;ttf0vudZC zzsdCFM=tMXFsQl{=leP1Lh)p3BdF(nm|T0{@Ugd-f49E(xI$u26Uf1YjDk`sY5drg zn@I12TzMn5L}`jT%}fcwZPs7&`yf8ZsWb>;_VhBoeq*KboqUe%LyOkI&&^Nr1coT4 z%bHp`^YqoUOtOqW>OtPBFBk9F@mzR7q^Ea_O<)r{m4SUR?vlgj^>p&@s1V#CM1=~2 z`OrG*{7f)abLZBumH}qF(6(>9HL<`MNtL)KTlbevvi(G+*WxWuu<08sNe6m=hV01l z9r>yoqU`Uxua%rSkoV-ZQsxW$aiU9gZO#oR)Ryn3Su1k9IA;4)FgR`)1+s^u&7voW zFw(e$?kJUUk*FgV6XQn{5U#}*Z`e5yUhOrm99V3x-%fc@1AIfU;7|SuJ4-eD{$mA+ z7mAl_;eMe4f5*rn)<|tgq=g)Qpd_L(D6Bgqo6%j{Y7Xu;uTV)$I4FpV;`T?`v=CfkZv&fqBuc7DtMSZXr@E*LK4mF8b)te5`G* z*I&saAWygFyQc*+R3(-2Z67dlkFC%v!9BH;5hx3#YkegeWkDX70xIT*Et`JAiZmVu z(D1@-Z!m2hT5#>div$gt!i!aNJx5N8CqM<#(jU0K+V8K)&q&ruHA8DJz8-;2Y`wkL z4rJEFq3hOcYj=kZ%82ut7j8$By7enudDyA1u<331ojc7o*A_o!lNG(#gt`8Q-m%w* ztZ10*nZk-45NczM_J?CT0{z>IJ&yXUmy(a6o~T|2f;Nm&6!0ms{F7y`!z_AMF*l+P z5&vUre;lgQgb=ei4$G-t&&4J-oabQD*trg=%a`$dwg>dYC6H$n2Z`cAw8HGN^x!Lgq z>UwD*8la>TY;9G6a9W1XiH7UurAsbBEl~w7yTN6soIoXi4^S$QIF1T^HFCiTQ*_gw zgKC$KUuV1_W84!>8rSBlhPk$x6OZyEN*MpK_w6+|Q9?m#%Ct%U!vnV3@iWdEQk&R{ z`1G|;RsO3RC=HHh(CK#~nl$%5T;Q#7O}S_Jcd`7}>e5Vggc{@du=^6oz1?HCTCH1< zosayy{7E}s=13TIh3zY!<8pg~GyU5<{cKnBP9ishEyuJDYN=8ppMp%B|6A6}-<5G( za{Ui2@qz+OWAL7{&rOg>HE?6P>+;h+OMAg=`VMGdMefQi8u}NB>r&V!8cvi!a2`MI z_8lCqeDbeg_mboWWg#+R6_Zt%27?z$yMW1XE@1=4TbLHlEJSUzrY)4%Yyt9sg??9@j$w)ZadM0!PT z7{Sla@-du5E#w=A?&71~1(hErc3!}2e z^TG01X9AsM+xNy?;vL47pQt@VR~o94 zR@{9}bI6V>O!oe#idhDz$W8IF*w5*nmHFa+ta+oqCV0nREqdfLu3+Jn`{MxbKcLOMy|_F_T#MphCS)XP=x51Z z`Ea#p=akA|y~#Wo2vN)j&>Ng9dtumWPd53>h>KJj$cJ5R#7L{2W8)j*%}$IjzIF~eD+&Mc8gjj`lyDq zb8E`(y?ukl&lVbY78(`vrfyC#0iPx7W4q0A!Q z6$$5JK=9h5p#6yqofO7k{Z<36uB59tNifFu2aU51x0d*-u2)&2>SK1nHE~FFjQ0F4 zbW6&Ya80CPC$q^@z81{*yG=3Z-bHx17I2b0AySIA6Gq|Od`oCrX<9l&2p))La|+PU z|8DRdF}bCcT1{oUT&D_*vZUXTGlW^!ltjJJ<%1<}yFQUw`{z?;U3xV?wsOjHa9p_y z(@lv=JKlUDjAAXqV_6N=iVDyiSLxqcG|#_(b}Pd*GD{JRuM49+9L&I(M^o~BP*(?M zWJrc>WFn(^($qdzvfN4hfn8D(P%Px5glIW!(`Pv=btIBw53I>lNYn`onjTSw=lQD1 zz@}D`&xW3ziwf}`9?SCyWtQ7!27B+6;7WX- zAue(H9gl20oyar{%-h{bdELev>Mke~tV)-t%i$<`Kc}W{6YLC<(v+I78PCj7jPz4| zP400KIfh~n#_RaOZ)8j=o*?c1PKFh1iLc20)z5iuh*yLM3jozxlH28DMbY`9 zpz!$Xg6-YrGp3Qs!2>m3=N4bsqriXx6^2_u1}MDmJdQ#BY6ofU8K}r5T}mKt`M$BWW0|n9<373<-T@hlDB;QfR2te zG`f7oudS^>D}00koX@pS)QW-%!u4$mzj%`KWowca2+NROFJdHWc}IsPkK+u(0lipc zUS#Nc_T{(DE3?3J7-4X(B=o65fujY$d1SXU^WxDp_irDF%CfbCVrf{H?9aXD8FHGLlxQAEW7s9wpFAhaxin^mvoOrH!v}sc zkN&>lK?knCj;`W{Z8Oa$IOZUyI7iZ7_dbMrI56_T-Rq@bn-9+&CZ&lvC-3TjS{9?a z@@#C-5RI3MiuLUXA1$7Yd`pdFuk_sCP4n^sFSi|tep+?0FBneV|D>ur(UvW35M3s2 zfI2AjOI^O5QR=`GLY(K(24OCJO)0b2|Hj18Nxxs+N@JG$G`A@6f=2Y3_jJ*A33RJ) zwZ^Jcuc7|P;P^q*Im1s3@Nrrd+Fih}@F)u9WR2Cl{m1%MPp9nU5Eu-7pVpY!N#$dH z=T{k-|squ{Da*{pz`V$k8iuN0#dE_B}|d^13Q0YX?-BYJD}z zd_rKKPC4T31AvxUzoTk4P1U0qIl&qta6licy$)ggX0e~*0WaB-L9PPXQCu;U9bQ%A zG2(Z5-@MeO@A>9Ebp6Ihu@`i&jyP;4TH3x$9Be&soCd=U&{8G~Lt9qaTPMIw9f>tz zTz`}UzUFE5yZ6I&1@T#zd4ZGHb)aiyCVWL!ID!%9K@SVeo7?%9P&u25=>uK{lgq}p zao6is%mR{_2TKoYJHBf+WuUEY)?1LqtvxI~nLzI8=S>NFXjW?=e%jvoREx2Y@=OmZ zH|LD~od{XZoW#D9d>N-mFZT;A9EXv$yQFT982a&=Q9+oOGuhQIp4ur%V<^nIXBMK= z_wo^nYzWlM*_gzVGWoJKuG#Z_DXr9k;rWY*pi1^9-_EP#Clbl6n4G^@iaz!leOC_t zrvcU~R%Q`)Pj8Gb_#?>Kp><2A6?X2;`AJtT4zefK%y=5_=(%c@)bRD`u>j{X#RS)todJ_a zpCEQ}NvG$<>0T+)#ahYXuY*H7Qy6V`xajzMGCQuMVILiKrSptc9mq{ezsX{@yABV# zW{YejSMm}iBWs4YN{Su06sxjN%|XY>2GEAp1&miDn2|PnYO$5OR~CS;KT5+$ie3|3yde098dV?YS|PED{-o`K?4ipXq3uC^Xj(fD%tl(aH9Jm8{9gW7 zI2-jyBIyf_SWE8-f~qUdd)_!x^VHsRuqeu(M=xg6XsX|DCn*$nBUj&y5pq-1Q!GUzl%@o zT5rO%WSE^X-kAp(0&t(e))5`i;+>@}u#?r~JE~OM|61sBX(UheQMR;u|)_{}2 zESgncNmVf2*}3WSAZbl7ebo`N^2wyUb=O^W-+}=H155?$+u%Cr5B)m=Sd?8)6n&Sp zfBEa(BbTW&o*RGd@A!T8xRjjQIZb7Mzn|xTV4G_KNA|3_f4X0Dv7FZ`{@&Xi z1;1zBpukjX(t)-aa9H|nwL5?1;6euWoyV_i+{^8D0X(tSdZJMv=jrlA%l(;&`yYM} z3Gqb-9TfLkesc-Wi$)i@pNZAkT?t$E6+1XclLtBSW#vnZV$F7DEWdA*=j`QS>xrY7 zjrSfnDjTb^y?xKuK61A_sD&&uatgV&yT1@3shNDNahpq9Jpb|7VYj!;waT&;^A}1j z7hcGTNw7+HYbMN{YO!ucX(ObI?#pWih{fgs*EN`bVc##O z6d2?fe0z$xTRfoDMgjZOPtzb