From 21d84615d5da5fa81e29b3b570dbc82672ca3c49 Mon Sep 17 00:00:00 2001 From: GGBond8488 <857631483@qq.com> Date: Wed, 29 Nov 2023 07:11:38 +0000 Subject: [PATCH 01/10] add cinn diff --- padiff/cinn_diff/README.md | 143 +++++++++++++++ padiff/cinn_diff/__init__.py | 4 + padiff/cinn_diff/analyze.py | 107 +++++++++++ padiff/cinn_diff/compare_utils.py | 77 ++++++++ padiff/cinn_diff/env.py | 100 +++++++++++ padiff/cinn_diff/graph.py | 286 ++++++++++++++++++++++++++++++ padiff/cinn_diff/img/run_ret.png | Bin 0 -> 64863 bytes padiff/cinn_diff/logs.py | 0 padiff/cinn_diff/read_file.py | 253 ++++++++++++++++++++++++++ padiff/cinn_diff/requirments.txt | 2 + padiff/cinn_diff/run.py | 16 ++ padiff/cinn_diff/utils.py | 63 +++++++ 12 files changed, 1051 insertions(+) create mode 100644 padiff/cinn_diff/README.md create mode 100644 padiff/cinn_diff/__init__.py create mode 100644 padiff/cinn_diff/analyze.py create mode 100644 padiff/cinn_diff/compare_utils.py create mode 100644 padiff/cinn_diff/env.py create mode 100644 padiff/cinn_diff/graph.py create mode 100644 padiff/cinn_diff/img/run_ret.png create mode 100644 padiff/cinn_diff/logs.py create mode 100644 padiff/cinn_diff/read_file.py create mode 100644 padiff/cinn_diff/requirments.txt create mode 100644 padiff/cinn_diff/run.py create mode 100644 padiff/cinn_diff/utils.py diff --git a/padiff/cinn_diff/README.md b/padiff/cinn_diff/README.md new file mode 100644 index 0000000..93edd40 --- /dev/null +++ b/padiff/cinn_diff/README.md @@ -0,0 +1,143 @@ +## 快速开始 + +### 安装依赖 + +`python -m pip install -r requirments.txt` + +### 一键运行模式 + +**example** +```python +import os +from analyze import auto_diff +from env import Env + + +def run(run_script, base_env, cinn_env): + run_env = Env(run_script, base_env, cinn_env) + run_env.run_base_model() #可以注释掉选择不运行base model + run_env.run_cinn_model() #也可以注释掉选择不运行cinn model + auto_diff(run_env.base_path, run_env.cinn_path, rtol=1e-3, atol=1e-3) + + +if __name__ == '__main__': + run_script = "/root/dev/PaddleNLP/model_zoo/bert/run_bert.sh" + run_script = "/root/dev/PaddleClas/run_resnet.sh" + run(run_script, None, None) +``` +**run_script** +模型运行脚本，使用时提供脚本路径，需在模型内部实现好动转静 + +**base_env** +模型基线运行的环境变量 +初始配置为 +```python +{ + "CUDA_VISIBLE_DEVICES" : "0", + "NVIDIA_TF32_OVERRIDE" : "1", + "CUDA_LAUNCH_BLOCKING" : "1", + "FLAGS_save_static_runtime_data" : "1", + "FLAGS_static_runtime_data_save_path" : "./", + "FLAGS_cudnn_deterministc" : "1", + "FLAGS_cinn_cudnn_deterministc" : "1", + "FLAGS_prim_all" : "true" +} +``` + +**cinn_env** +模型接入编译器运行的环境变量 +初始配置为 +```python +{ + "FLAGS_use_cinn" : "1", + "FLAGS_deny_cinn_ops" :"", + "FLAGS_use_reduce_split_pass" : "1", + "FLAGS_nvrtc_compile_to_cubin" : "0", + "FLAGS_cinn_use_op_fusion" : "1", + "FLAGS_cinn_parallel_compile_size" : "8", + "FLAGS_cinn_pass_visualize_dir": "", +} +``` + +### 手动运行模式 + +step1: 准备模型运行脚本，跑通动转静+组合算子+编译器 + +step2: 手动运行动转静+组合算子的基线模型 + +基线模型运行是需要配置如下环境变量 +``` +"FLAGS_save_static_runtime_data" : "1", + +"FLAGS_static_runtime_data_save_path" : "./base", +``` +step3: 手动运行动转静+组合算子+编译器的模型 + +接入编译器的模型运行需要配置如下环境变量 +``` +"FLAGS_save_static_runtime_data" : "1", + +"FLAGS_static_runtime_data_save_path" : "./cinn", + +"FLAGS_cinn_pass_visualize_dir": "./cinn/cinn_pass", +``` +step4: 运行模型精度对齐脚本 + +```python +from analyze import auto_diff + +base_path = "/root/dev/PaddleClas/base" +compare_path = "/root/dev/PaddleClas/cinn" +auto_diff(base_path, compare_path, atol=0, rtol=0) +``` + +模型运行脚本环境变量配置例子 +``` shell +#!/bin/bash +export CUDA_VISIBLE_DEVICES=5 +export NVIDIA_TF32_OVERRIDE=1 +export CUDA_LAUNCH_BLOCKING=1 +export FLAGS_save_static_runtime_data=true +export FLAGS_cudnn_deterministc=1 +export FLAGS_cinn_cudnn_deterministc=1 +# export FLAGS_check_nan_inf=1 +rm -rf ./cinn/* +export FLAGS_static_runtime_data_save_path="./cinn/" + +# 跑 动转静 + 组合算子时打开下面这1行 +export FLAGS_prim_all=true +# 跑 动转静 + 组合算子 + CINN时打开下面这6行 +export FLAGS_use_cinn=1 +export FLAGS_deny_cinn_ops="reduce_sum" +export FLAGS_use_reduce_split_pass=1 +export FLAGS_nvrtc_compile_to_cubin=0 +export FLAGS_cinn_use_op_fusion=1 +export FLAGS_cinn_parallel_compile_size=8 + + +# # before and after cinn program and graph pass(including group opfusion pass) in each sub-graph +rm -rf ./cinn_pass/* +export FLAGS_cinn_pass_visualize_dir="./cinn/cinn_pass/" + +task_name_or_path="llama_output" +python run_pretrain.py \ + --model_type "llama" \ + ... +``` + +## 运行结果 +![运行结果图](./img/run_ret.png) + + +## 功能扩展 +1. 【开发中】参数式启动 +2. 【开发中】中间变量读取展示接口 + +更多功能正在研发中... + + + + + + + diff --git a/padiff/cinn_diff/__init__.py b/padiff/cinn_diff/__init__.py new file mode 100644 index 0000000..5ea3f0f --- /dev/null +++ b/padiff/cinn_diff/__init__.py @@ -0,0 +1,4 @@ +from .graph import * +from .compare_utils import * +from .utils import * +from .env import * \ No newline at end of file diff --git a/padiff/cinn_diff/analyze.py b/padiff/cinn_diff/analyze.py new file mode 100644 index 0000000..584d72d --- /dev/null +++ b/padiff/cinn_diff/analyze.py @@ -0,0 +1,107 @@ +from logging import warning +from read_file import read_all +from compare_utils import Comparator + +# 需要优化 +def back_track_group(base, compare, cluster, cmp, graph, node): + inputs = graph.inputs() + all_inputs_equal = True + paddle_output_name = "" + cur_cluster_cinn2paddle = { v : k for k, v in cluster.varmaps.items()} + for input in inputs: + tmp = input + paddle_name = cur_cluster_cinn2paddle.get(tmp.name, "") + if not paddle_name: + print(f"can't find {node.name}'s paddle name") + diff_ret = { + "cluster": cluster.idx, + "group" : cluster.cinn_group, + "output" : paddle_output_name, + "output_cinn_var" : node.name, + "subgraph_id": node.graph_id if node else None + } + return diff_ret + paddle_output_name = paddle_name + base_var_path = base.all_vars_paths[paddle_name] + compare_var_path = compare.all_vars_paths[paddle_name] + ret = cmp.allclose(base_var_path, compare_var_path) + if not ret: + all_inputs_equal = False + group = cluster.cinn_group + for graph in group.subgraphs: + node = graph.find(tmp.name) + if node and not graph.is_input(node): + return back_track_group(base, compare, cluster, cmp, graph, node) + if all_inputs_equal: + diff_ret = { + "cluster": cluster.idx, + "group" : cluster.cinn_group, + "output" : paddle_output_name, + "output_cinn_var" : node.name, + "subgraph_id": node.graph_id if node else None + } + return diff_ret + + +def auto_diff(base_path, compare_path, rtol=1e-6, atol=1e-6): + base = read_all(base_path) + compare = read_all(compare_path) + cmp = Comparator(rtol=rtol, atol=atol) + + # step1: 确认cluster的输入输出是否对齐 + for cluster in compare.all_clusters: + #print(cluster.idx) + input_equals_flag = True + output_equals_flag = True + for input in cluster.inputs: + base_var_path = base.all_vars_paths[input] + compare_var_path = compare.all_vars_paths[input] + ret = cmp.allclose(base_var_path, compare_var_path) + if not ret: + input_equals_flag = False + cmp.record_input_diff(cluster.idx, input) + continue + + if input_equals_flag: + # step2: 找到cluster内部对不齐的点 + for output in cluster.outputs: + base_var_path = base.all_vars_paths[output] + compare_var_path = compare.all_vars_paths[output] + ret = cmp.allclose(base_var_path, compare_var_path) + if not ret: + output_equals_flag = False + # step3: 找到对不齐变量对应的group + output_cinn_var = cluster.varmaps.get(output, "") + if not output_cinn_var: + print("can't find var " + output + " corresponding cinn var name") + else: + find_diff_group_flag = False + # step4 : 从对不齐的输出出发，找到第一次出现输出对不齐的group（输入能对齐，输出无法对齐） + group = cluster.cinn_group + for graph in group.subgraphs: + node = graph.find(output_cinn_var) + if node and not graph.is_input(node): + # 找到对不齐的第一个输出，开始回溯 + diff_ret = back_track_group(base, compare, cluster, cmp, graph, node) + if diff_ret: # 输入能对齐，输出无法对齐 + diff_ret["output"] = output + cmp.record_group_output_diff(diff_ret) + find_diff_group_flag = True + break + + if not find_diff_group_flag: + cmp.record_output_diff(cluster.idx, output, cluster.varmaps.get(output, "")) + print("can't find diff group in cluster_" + cluster.idx + " but diff exsits") + + if output_equals_flag: + print("cluster_" + cluster.idx + " has no diff") + + for diff in cmp.record: + print(diff) + return cmp.record + + +if __name__ == "__main__": + base_path = "/root/dev/PaddleClas/base" + compare_path = "/root/dev/PaddleClas/cinn" + auto_diff(base_path, compare_path, atol=0, rtol=0) \ No newline at end of file diff --git a/padiff/cinn_diff/compare_utils.py b/padiff/cinn_diff/compare_utils.py new file mode 100644 index 0000000..80b2934 --- /dev/null +++ b/padiff/cinn_diff/compare_utils.py @@ -0,0 +1,77 @@ +from typing_extensions import Self +import paddle +import numpy as np +from colorama import Fore,Back,Style + + +class Comparator: + + def __init__(self, rtol=0, atol=0) -> None: + self.cluster_ret = {} + self.graph_ret = {} + self.record = [] + self.rtol = rtol + self.atol = atol + + @classmethod + def load_var(self, path): + return paddle.Tensor(paddle.core.load_dense_tensor(path)) + + def allclose(self, base_path, compare_path): + base_var = self.load_var(base_path) + compare_var = self.load_var(compare_path) + ret = np.allclose(base_var, compare_var, rtol=self.rtol, atol=self.atol) + return ret + + def assert_allclose(self, base_path, compare_path): + base_var = self.load_var(base_path) + compare_var = self.load_var(compare_path) + ret = np.testing.assert_allclose(base_var, compare_var, rtol=self.rtol, atol=self.atol) + return ret + + def record_diff(self, diff, type): + diff = { + "event": diff, + "type" : type, + } + self.record.append(diff) + + def record_input_diff(self, cluster_idx, input): + self.record_diff( + { + "cluster_idx": cluster_idx, + "cluster_input_diff_name": input + }, + "cluster_input_diff" + ) + + def record_output_diff(self, cluster_idx, output, output_cinn_name): + self.record_diff( + { + "cluster_idx": cluster_idx, + "cluster_output_diff_paddle_name": output, + "cluster_output_diff_cinn_name" : output_cinn_name + }, + "cluster_output_diff" + ) + + def record_group_output_diff(self, diff_ret): + self.record_diff( + { + "cluster_idx":diff_ret["cluster"], + "cluster_output_diff_paddle_name" : diff_ret["output"], + "group_idx": diff_ret["group"].group_id, + "group_output_diff_cinn_name": diff_ret["output_cinn_var"], + "group_graphviz_path": diff_ret["group"].dot_path, + "group_test_py_code_path": diff_ret["group"].txt_path, + "group_diff_subgraph_id": diff_ret["subgraph_id"], + }, + "group_output_diff" + ) + + +if __name__ == "__main__": + cmp = Comparator() + base = cmp.load_var("/root/dev/PaddleClas/base/saved_tensors/batch_norm_grad-input-batch_norm_0.tmp_3@GRAD") + cinn = cmp.load_var("/root/dev/PaddleClas/cinn/saved_tensors/batch_norm_grad-input-batch_norm_0.tmp_3@GRAD") + np.testing.assert_allclose(base, cinn, rtol=0, atol=0) \ No newline at end of file diff --git a/padiff/cinn_diff/env.py b/padiff/cinn_diff/env.py new file mode 100644 index 0000000..f5af006 --- /dev/null +++ b/padiff/cinn_diff/env.py @@ -0,0 +1,100 @@ +import os +import subprocess +from sys import stderr + + +class Env: + + base_dir_name = "base" + cinn_dir_name = "cinn" + cinn_pass_dir = "cinn_pass" + cinn_graph_dir = "cinn_graph" + + def __init__(self, script=None, base_env=None, cinn_env=None, ): + self._base_env = { + "CUDA_VISIBLE_DEVICES" : "7", + "NVIDIA_TF32_OVERRIDE" : "1", + "CUDA_LAUNCH_BLOCKING" : "1", + "FLAGS_save_static_runtime_data" : "1", + "FLAGS_static_runtime_data_save_path" : "./", + "FLAGS_cudnn_deterministc" : "1", + "FLAGS_cinn_cudnn_deterministc" : "1", + "FLAGS_prim_all" : "true" + } + self._cinn_env = { + "FLAGS_use_cinn" : "1", + "FLAGS_deny_cinn_ops" :"reduce_sum", + "FLAGS_use_reduce_split_pass" : "1", + "FLAGS_nvrtc_compile_to_cubin" : "0", + "FLAGS_cinn_use_op_fusion" : "1", + "FLAGS_cinn_parallel_compile_size" : "8", + "FLAGS_cinn_pass_visualize_dir": "", + } + self.base_env = base_env if base_env else self._base_env + self.cinn_env = cinn_env if cinn_env else self._cinn_env + self.base_path = os.path.join(os.path.dirname(script), self.base_dir_name) + self.cinn_path = os.path.join(os.path.dirname(script), self.cinn_dir_name) + self.script = script + self.script_path = os.path.dirname(script) + self.script_name = os.path.basename(script) + self.os_env = dict(os.environ) + + def init_base_env(self): + if os.path.exists(self.base_path): + print("base path exists, remove it") + os.system("rm -rf " + self.base_path) + self.base_env['FLAGS_static_runtime_data_save_path'] = self.base_path + self.base_env['FLAGS_save_static_runtime_data'] = "1" + + def set_base_env(self, env): + self.base_env = env + + def init_cinn_env(self): + self.base_env['FLAGS_static_runtime_data_save_path'] = self.cinn_path + if os.path.exists(self.cinn_path): + print("cinn path exists, remove it") + os.system("rm -rf " + self.cinn_path) + self.cinn_env['FLAGS_cinn_pass_visualize_dir'] = os.path.join(self.cinn_path, self.cinn_pass_dir) + self.cinn_env['FLAGS_cinn_subgraph_graphviz_dir'] = os.path.join(self.cinn_path, self.cinn_graph_dir) + + def set_cinn_env(self, env): + self.cinn_env = env + + + def set_script(self, name): + self.script = name + + def run_model(self, run_env, log): + print(self.script) + ret = subprocess.run(["sh", self.script_name], env=run_env, stdout=log, stderr=log) + print(ret) + + + def run_base_model(self): + self.init_base_env() + os.chdir(self.script_path) + run_env = self.base_env.copy() + print(run_env) + run_env.update(self.os_env) + base_log = open("base.log", "w") + self.run_model(run_env, base_log) + base_log.close() + + def run_cinn_model(self): + self.init_cinn_env() + os.chdir(self.script_path) + run_env = self.cinn_env.copy() + run_env.update(self.base_env) + print(run_env) + run_env.update(self.os_env) + cinn_log = open("cinn.log", "w") + self.run_model(run_env, cinn_log) + cinn_log.close() + + + + + + + + diff --git a/padiff/cinn_diff/graph.py b/padiff/cinn_diff/graph.py new file mode 100644 index 0000000..6cbe39a --- /dev/null +++ b/padiff/cinn_diff/graph.py @@ -0,0 +1,286 @@ + +from lib2to3.pytree import Node +import graphviz +import pygraphviz as pgv +from utils import retry, suppress_stdout + +@retry(max_times=1) +def get_graph(dot_path): + return pgv.AGraph(dot_path) + +def construct_graph_by_dot(dot_path, sep="\\n"): + #print("dot_path:" + dot_path) + graph_source = get_graph(dot_path) + # ['color', 'label', 'style'] + all_nodes = [] + idx2nodes = {} + ret_subgraphs = [] + subgraphs = graph_source.subgraphs() + if not subgraphs: + subgraphs = [graph_source] + for subgraph in subgraphs: + subgraph_id = subgraph.get_name().split("_")[-1] + tmp_nodes = [] + for node in subgraph.nodes(): + name = node.attr['label'].split(sep)[0] + idx = node.get_name() + cls_node = Node(name = name, idx = idx, type="unknown", graph_id = subgraph_id) + idx2nodes[idx] = cls_node + all_nodes.append(cls_node) + tmp_nodes.append(cls_node) + ret_subgraphs.append(Graph(nodes=tmp_nodes, name = f"group_{subgraph_id}")) + + for edge in graph_source.edges(): + start, end = edge + if start not in idx2nodes.keys() or end not in idx2nodes.keys(): + continue + # 输出边 + idx2nodes[start].add_output(idx2nodes[end]) + # 输入边 + idx2nodes[end].add_input(idx2nodes[start]) + + return all_nodes, ret_subgraphs + +class Graph: + + def __init__(self, nodes, name) -> None: + self.nodes = nodes + self.name = str(name) + + def add_node(self, node): + if isinstance(node, Node): + if node in self.nodes: + return + else: + self.nodes.append(node) + else: + raise ValueError(" param type must be Node") + + # just for cinn graph + def graph_inputs(self): + inputs = set() + for node in self.nodes: + if node.name == "feed": + inputs.add(node.outputs[0].name) + + return inputs + + def inputs(self): + inputs = [] + for node in self.nodes: + if node.name == "feed": + inputs.append(node.outputs[0]) + if not node.inputs: + inputs.append(node) + # 输入在另一个子图中，也算作当前子图的输入 + for node in node.inputs: + if node not in self.nodes: + inputs.append(node) + return inputs + + + # just for cinn graph + def graph_outputs(self): + outputs = set() + for node in self.nodes: + if node.name == "fetch": + outputs.add(node.inputs[0].name) + if not node.outputs: + outputs.add(node.name) + return outputs + + def outputs(self): + outputs = [] + for node in self.nodes: + if node.name == "fetch": + outputs.append(node.inputs[0]) + if not node.outputs: + outputs.append(node) + # 输出在另一个子图中，也算作当前子图的输出 + for node in node.outputs: + if node not in self.nodes: + outputs.append(node) + return outputs + + def find(self, cinn_var_name): + for node in self.nodes: + if node.name == cinn_var_name: + return node + return None + + def is_input(self, node): + return node in self.inputs() + + def is_output(self, node): + return node in self.outputs() + + def export_dot(self): + dot = graphviz.Digraph(comment = self.name) + for item in self.nodes: + dot.node(item.idx, item.idx + "\n" + item.name + ":" + item.node_type) + for next in item.outputs: + dot.edge(item.idx, next.idx) + return dot + + def __str__(self) -> str: + return "graph_" + str(self.name) + + def __repr__(self) -> str: + return "graph_" + str(self.name) + + +class Pass: + + def __init__(self, id, pass_name=None, before_txt=None, after_txt=None, before_dot=None, after_dot=None) -> None: + self.pass_id = id + self.pass_name = pass_name + self.before_txt = before_txt + self.after_txt = after_txt + self.before_dot = before_dot + self.after_dot = after_dot + + def set_pass_name(self, pass_name): + self.pass_name = pass_name + + def set_before_txt(self, before_txt): + self.before_txt = before_txt + + def set_after_txt(self, after_txt): + self.after_txt = after_txt + + def set_before_dot(self, before_dot): + self.before_dot = before_dot + + def set_after_dot(self, after_dot): + self.after_dot = after_dot + + def __str__(self) -> str: + return "pass_" + str(self.pass_id) + "_" + self.pass_name + + def __repr__(self) -> str: + return "pass_" + str(self.pass_id) + "_" + self.pass_name + + +class Group: + + def __init__(self, group_id, all_passes, last_pass_id) -> None: + self.group_id = group_id + self.passes = all_passes + self.dot_path = all_passes[last_pass_id].after_dot + self.txt_path = all_passes[last_pass_id].after_txt + self.all_nodes, self.subgraphs = construct_graph_by_dot(self.dot_path) + self.fetch = None + self.feed = None + + + def export_graph(self): + self.graph = Graph(self.all_nodes, self.__str__) + dot = self.graph.export_dot() + dot.render(self.__str__(), format='png', cleanup=True) + + def export_dot(self): + dot = graphviz.Source(self.dot_path) + dot.render(self.__str__(), format='png', cleanup=True) + + def __str__(self) -> str: + return "fusion_group_" + str(self.group_id) + + def __repr__(self) -> str: + return "fusion_group_" + str(self.group_id) + + +class Node: + def __init__(self, name, type, idx, graph_id = None) -> None: + self.name = name #var name, like arg_1 + self.node_type = type if type else "unknown" + self.idx = idx # node name like node1 + self.inputs = [] + self.outputs = [] + self.cinn_name = "" + self.graph_id = graph_id + + def is_op(self): + return self.node_type == "op" + + def is_var(self): + return self.node_type == "var" + + def is_leaf(self): + return self.outputs == [] or self.outputs[0].name == ["fetch"] + + def is_root(self): + return self.inputs == [] or self.inputs[0].name == ["feed"] + + def set_outputs(self, outputs): + self.outputs = outputs + + def set_inputs(self, inputs): + self.inputs = inputs + + def add_input(self, node): + if isinstance(node, Node): + if node in self.inputs: + return + else: + self.inputs.append(node) + else: + raise ValueError("Node input must be Node") + + def add_output(self, node): + if isinstance(node, Node): + if node in self.outputs: + return + else: + self.outputs.append(node) + else: + raise ValueError("Node output must be Node") + + def __str__(self) -> str: + return self.name + "_" + self.idx + " : " + self.node_type + + def __repr__(self) -> str: + return self.name + "_" + self.idx + " : " + self.node_type + + +class Cluster: + + def __init__(self, idx, graph, ops, inputs, outputs, graph_key, varmaps=None) -> None: + self.idx = idx + self.graph = graph + self.ops = ops + self.inputs = inputs + self.outputs = outputs + self.graph_key = graph_key + self.varmaps = varmaps + self.cinn_group = None + + def set_varmaps(self, varmaps: dict): + self.varmaps = varmaps + + def set_associate_groups(self, group): + if isinstance(group, [list, set, tuple]): + self.associate_groups.extend(list(group)) + elif isinstance(group, str): + self.associate_groups.append(group) + else: + raise ValueError(f"group must be str or sequence type, but got {type(group)}") + + + def __str__(self) -> str: + return "Cluster_" + str(self.idx) + + def __repr__(self) -> str: + return "Cluster_" + str(self.idx) + + def print_varmaps(self): + for paddle_name, cinn_name in self.varmaps.items(): + print({ + 'graph_key': self.graph_key, + 'paddle_name': paddle_name, + 'cinn_name': cinn_name + }) + + + + + diff --git a/padiff/cinn_diff/img/run_ret.png b/padiff/cinn_diff/img/run_ret.png new file mode 100644 index 0000000000000000000000000000000000000000..e224fd88dd7c70bfc6482a2425d441d949290e5a GIT binary patch literal 64863 zcma&N1ymi~madIUfZ*-~hv4q60fGdATX1)Gf;$9v2_(3?ySr`N-Q92Tb)O#p?SFKi zn=y8(fZ9@ftu^O6-}$W2ukw;eAMieafq@}OONl9gfx+Ma=PhtB!2e+i`J!N8h?Zue zqF<#&MM=Ke+ZdZ!8i9dHh5k~5{jNNSm8lsQ6Egt;Ck5a7NeNO4J_kBR2MV7Qo;(2u z-9N0Ap0)u?Osq1um@LeyV6l=e+~6!68q-L7rZ;FPEylO(Y3Oe3ZnV{BEd4>Dk+ty- zEc?NaE_ORd18h4SAs>FDGlJRd0C9@d7YuP9jFM^R@&}r#mKK{Yi|bo+OEY+Bj+T?E z?Tf=}d)P-^k}YuGhybp5#;qMuPXaJcLItwVV7^5Yn({O;)aL{tP6bRtA;XwlnxQV# zw!hJci7KRJ^h7;z!Pse&=tg0Lwuoj}Ke#Q%NN%zAlo_;1A^QGC?*Zo{O+v0Db1r;- zmj0jxUrAV#B!svHJ~DE~RsGhN#oaT$V~nhCqyw3YMHVN0*)aV&KJ{!fI3ADuSm75y zn;=0>eHqN&@{5;7#_tb`OE{~c0cLH`eLzTg2`?D6ICZp?Dk!gy4;Oz9d0+H+XfH@i z*b>n$nTwH9qhNdl?WY|{P&0G}E$uqFuqD_9nIAoyqEWRp4~_5+Lg{#1l{)PUcbQ{H zdDlFon>CAU@0$emdeJ>|B$>VvZ4X9B=?8HdW)usdW3pkn2&KgcTC^%d7_0nCDDh`i z6e_r$jZPWw5++%^#Ox}@!6c;Ja4WPhE@s-6cA;d9oDyW`uztc2;$44jtPu6V06vJt zFs*ZpO-~f)%XOQF4%&;D zdW>@@bFsJcv7Z63TcZJ@%x`Bm0U#SZtQoOlwoBn?#8F{c18U|#9AU&ype+eku+w14 zZVMCwYcBLV+>xy`{tcR(jPnrNj0xceenmEgN6~~lVc2bC;}8hslQJsE zW{U~UE+M|g7|Cm7p8y3_Md=iaQj4S`lN~KTJ#)X!^-3z~-_qiXT@Ww`ZG=#WQOSFo zJ+=-`4zNNOWs0v}t&zm8!6c8RYmQewupMvTeSF?>Y?=lP!H%zj`r8lE#-iYT>v+KS zQeExMa+HGU7@+PMbKn(HNtvJ%J3j7i!L;}i!$IVbp!N8Z$G}i_?EkdQlI_G`f*gVF z&BmaIE$+~$K(z$t?Xa#OX7lY{hvI@{@E>01X+UsMpmK$G@8F#XCV_*`A$9CQVHPbx zmFz*%Cj*zFMvEjTF_VIciwr^KW`RtJP9QTH(D=cs6mTq>8iPLImQUCi$tTtn!#O~O zA^N9?!Q3B|{pl;^kVxJHxg%d{q`XA<1oXaA4O*#$OU}(c{56?7;fauLZYxg5471k= zfkR(14t{v=R3-hFu+c7=iVr1t#oehJ4sBshdMgW(=^@j0FsFzvwosW7SLcKG^q%l- zSZjXsTZQMv_o6MZZ!+1a5%5t^x)8xkK{0a4BGi;BD0QeG(d4_UXQW*v$%?qa zv8F?~dk%UidTx67d$xLkIk>Ez=ruA@dhWM{f{^3``BFjUrzv)d0O&RX# z?rD#S5_yXg{-y^t5FjNDq!aQJ@RI--?QZdayw3ULQpEzray#*@UzF<88u10(!qrmM zB3Hh<6r-N)U+`jbhCpvOuNM_Yd~n_6zpqW{anIbLVrS z50>^FuCK0>4$kJ=bKcC72bt!WIxBj+4}-LEZ&&pigk$_VvU~KRJDma)f)KmEc58KG z1|bFAMcRIX&2!D8k-(3T$9F=WCp_g$=WO|L|GME4p=b!Z1+{h1rx~Fc=^^(;u1k)b z(&CGJs$<$=8cfP=a$*`0`w2^48gz2#$k1q1N~_*J^Y;io35qR@y#D0{zAi26w#Amw zmhrARvPFZ;UZU`)L=4&AMu|p^M&U-$zfxvRi*Z$JRb5I3=e}6v+?XF#&z)IWS}9p! zf!G%K>e(xH7O)oQK&#bxi_jpsy5hQKV}%+M&>pC~zWvyj$CU?;@Gk)acW#tU6j}dG z)QLP46){z%a_zL7$zRst#)-zchn(;>*_vU#B$y<%PbCuh(chUjk3-7KGRxe`G_}v7 zW^i%2?Ny6-D>SGZa(x5FUcN$ z+ne37+R4~x9ebiwPQvZi>f+lIJ%_(dIdNFn%PUW^@jLTg6MRX3v43%X=bm0>bJ zD(Zj5dewRxeI*2s@Qv`511I#`^;_;Z?2r^r6n@UO5+474TS73$FjuTMuV;4NxqfG< zd;8_~0)?KOFP}XBvkDa(x1-g508@CyaRsO%({}v)V59#N9I7HJ9htTL7cM?|q(Z7G zUVG#1Kh)+b1~Z@{$;pqCQ~SG<`}+fvbdqb~l~JhuY(4(p<&TFJp6u%E;O0%V8uC!Vhl%H7O9D&Phyqg?u zCSjIa)90w?gUK%={_)#!os<;{L=089W^v0& zjme|*kn~9IGY3P5`n}t|g58rrwc+sr8?>Vkhu%WU6!}!1KiZqrG>ML1ZZlfU?W5Ub zn&g58`iAik95*DrikIbIQvubz_)LU{IM_-KiH#ap}_VgdT`hof03pcR@`Q!TfrgIBGbQ!mzL zV%KMd`rnrPPY72@sa6t8GWA}I46Cb^pY;3x;GLzWX7uqU&F0o#g37eFoV8a8tC}mv z8m+I0Uib?L=G(S6ebT?`{w8@HMb9h4-L!7{R+($q+E3Bntneetb(y7d=X8*nsyri1aMqdU z@Vb#z-$XvsD1+a5>}`4Ds%$>Ut+To)6Ip_G=$2zy#S>4OF8P~-5|?iC4oFx@NWZSKMn8i}B>lvH7X_7n{TucF*P)#^`np_bqqgmSLayySD9s zjbK9J6ykAjnV0Jln<0~_y&=Y1-PCpouh#jjwKc{KnMTMg2pj@aRV*-@I!GZ|NZ*f7 z97ErY7`aG7s(c8A)&=#A+PO${z-Btn?(VJcogC18TQ&8cpdZ-v%kd;3Nd{rFPftZf zDXQpQI4Q6(Z)pNyUtVnV$0ao2JCb%@Uc6>sUUcg^^z_eMwMveKF+~FTyD7(#MxMW% z+lVb@c_(c}yJUxSKt8*^VbA!)+(Eq!I|1c~rO|h3V>vl6THqWG3r8C2Oa|8 z`2Te-0Zt7D`Oohmz`%mdz@Yx~8hPOO{x1f2yg%nZju6=p|GEMOCmZr#=U^@GmqPa7 zx&jAyTPbx1Ffc5N_XoJN68Sk8m=Kt>n6Qc~_;Cg-28Q@lzvlLc_o4onI%;*U0F!}G zXCSv9wS6X)fhr%X#8>97Khz<)1|rNEe##>_DAk#`=;_>9i_yAS?AK#s$79zcyMOPT z3kwZz+C6x_x!indSD=dbw{G0Qa+p&$Swt~Ow+lJ|)mns2u(q65bbs9KMiyji^|<4B zdAv#6Kw&wlnU>#ey<27F_qb&XCUQ3&m`!$?*PaIvI*c$I*RMDuEHk#<8?c(sP(Qtn zu#Q9BN+V_Yyf!YH7nhdYk%|T-M{=%maqQ+Ai|O)bV_i0eb|#rV$dET3#T{me%Qgj zdORvmgC%k$32C!gYZLbFxy*1`Wo|xgD)u52@N7`GD6K8Q)nRPCRbP+)Q#hvNh9ZPIr<2OZ zrsYDS!$Aet>+zt__^7nLj=*<3lqxehCe74lM(yx3#to8|>&DOBREx68)4gmlEhYP)1KXJ8~%Z_*>KblVZLPl;~^BqG;JVh z>)@n*txj$-sA`U*8~_`H<+_5L(Z>5O!bg);RSWM}coq zuhkQ{zuf?20j6-qwgxB~*Rqz&Axw0Jm;XSIv|?}9PTlm%aml?7 zgnv!wK-d`LIVv$kag|G95pQu9{8K2Rcwp7A5#;A9{CuE{`V|TTTwLgXI%vT9%|O6l zS?5qBX-x?n-PrUKy08In-pG&Haq794)BgH+ly~S^9nH0HOJuVQg zwN16oGEA?`;faN8=O6izkSF<56jxWe_u~PFyF7P@??&B{HS0M4U3#tHc8q8hc^1#H z2*PI-mJtQP7nSufnwZW2*fe-_N{oeIUGJBhd5B{3Ic&}iwg(4jL^~>REG=7M{8mk> zmFuZw3jV-Q#hEhgW@?LC<^T1DF#NEDNosxldhJXz-FE+Ygq*fLSv#z{?cu82d0zBl zC$3Rtc}|@-A6E^J3{quT&gpoZj6B%73}iV@NgHj&|9qm3-oZ-G>)JL7J{KS0|HbUM z|J5QA^m{O#DjfUC^Q3m(-v#UD?d7xPe{g%SL{mJIY8Ze;iH60KLP{#92z>Ec{{W&Ys)VQi& z^R@^W0R}Tx*d)vAR?4GR<=@=uoDj*%;fdpyL>+gtV)wppSxVAH#+h!$_+B&qwsDS= zwd2w8*wK8KA&ob4nkN|__b&uD0@06i?c-c!9_+BEfp>gT(iv_lP`72kSC>Iv9wfQG znb!@Ue7@g`o*SWo;T}Zc;(>8I5(TD()-gO%zsG4`KamGL3PVTt81I=@LJ%BI7{O~r zk%U*MY;$^Rx_k$*c|$JLx8-iI9lI{6KyACOu=2LYq2C2Q|Jz~%{x*>cS0kNf1%-%$ zaIF=1EL-)|n4T@-{G+)Lid>GtM_%T9RD9B~KZ-rgvMLlQh$kaj;E|doEX6pLrecq5 zTWOo&x{3Eq_U9i-&O0{$1Xj;Fo3&M2PO+KsZ&}^B9NrDdjaWSt10srD(5FA7WXz6a zr(CP=HG$9(ux{t1dnz?3+Ohtd3OfcC@e60q#rmUS_5U0k{<#-6Mu2-(qsudag?RRe<*J^FZp-kuAzlTU}mz`j44Dv;GvlMA4O3!`Oqy&|pm+&Tbi&{#BXKTI zkoEYN?(bm_2s~_~ck@GG>v~px7Z=z$4LECmTg|QuTG=ZyfG;*w2V%aJt2|5tHg(>q z=|Y8mtexR|PFO++@AikEiv4NqyvA6!U{Glng7RagNHIt@dJ7Iy`suW}+-X;28-cN< zV&JGJ6n%6fnArMCWBWTrIKFj5Ckcw49f+U%a!%7G1soCv3j0%e)evovy)|206U5pl zi3n;g{eYNlV5&uoDiPgKa9pMnG%Tpg8L~09PMd{zIAaRl#LPVin@h1BFJW8|ef!N!A1HPn*Fkd08<*n=$!)X2y zU`5z$>21twKNGSVN5uAS5;`w*YPla2aNs#U>pD(~&qe6@|FmtK;7tFBJrjk`IaaZb zj7r=Lf~OJ_kWtJM@H!Ts*oLo`o!4@R{6h zx$1yKUO2NHh#|dQTCa)J?0>vDGC`5p7Vep?x6#GNR5>NQDBOpa6YbfjgAq5^ExKe(wo1Mz_%iouT8j%JF^K?!}41BsjC8O)qGOdrL;ZW`b0V<3Nu`gsPT7u zr+u9?{8BpJ?g1%Dc#13cd2nZSXA`lxdVlqgav_OKBYME2@45_wjPpaE12A2r*!(&< zeYASy1jwUj%>Ee8Gz$m7;0B_&O=?(_8Wqj^K!Z!|$QU3+?10hfHEp8HwrU6h0B z|I;gwU_!7m@JeBR;h%@_qX(5#1(u5Iu*?1XH}`v#5c)+*nM*Z%@0(6fRLNRKRq zuSj|SM!5YGn=weUA%e)HW<>rc;q!lsq9M|3+PERo{L_Dvr2bPaehu{ffD=Zdi16=k z<6V1E2MaCP6U8;Wi%{d^h05JuYC6osKF^g8yJ- zTfW2aIPUAs>H55g7?Z7fc8iOBn^!d(>Kv8Q;I(TI)Pp8~r9d}PG@EW9B z!;1#;>^iqMwUiOsd?&9ej9-W*nY7rZR-PU%IV|9e6IlJy4y&fQH@gX)u-dVUni~hs`u*=k&OG{1%G|k&tJmuM!BcDq3b~# zm$gso&AtO1Hu?5sDQPsUe>Htul%MVNJdJMUS`^s-$)q2Chll~^Zg*-P^ZYPmmB*3{1H@Y>Ru zNP!rBg%?hp{@P_7cdnoMF7-V6+$u{{{9uW}cjPzTU}vx5_5h3(|NB^F3;O|+TVT3a zt(;nGh5;PnpNiCZ$*OexUlplvQQwtP`HfM*FI5(x29^M$-aX$GUH=Xnr22=^3CwA@ zlF8{%h1F~-U+ls7YnciWxRvgZ;mn8tUO}rhhCub)uXqIk?)^3On(uBqzpN7I4HC%K zYzsCmDe5IGU^V|(rqYyaj2{-Vt+{R?y;sNGeV9q^*ZUi7SSeoJnBoatxhYTW9%rrd z1U}Cf(557&>zr+Pltuq28~iQ|5ybS3CTd! z14Ge4NOnW72Zv(?F1ps{WyO#LNFJ;KtCQ-1C`$Rf-bq=;)*nQ!>)=N%Kuqa(Q}%!b zzAOT!nIgFD5Fya(#wh1nO*5yyANV~m1W6(IYqp_eHmC}qk9AVQ^IU0ebJaB`vXRv~ zYJKNj1!svI_BY^FbhJk zEXU<+6^0bzZ=Irk$fLpT7VzpVqy0c=JaA`Z5`4Yif<6NzzG=R1=gswe`|E65+uf>1 z&;ybJ(<1QoBAJmce5H*R$~>`tn3)MCo8W=VCk&qH_N1ux-2m#EY6`xilIl8nX$$#Q z)QQJUB<*1szVi-9O^&nLfqTFiF~0Ae1U?867ln2@0z{GUd!q0oK$Th&eBKj$gICf8 z91I8tZeSJLU;J(Uj}T?Hl=L_%eZ&0(EG^62qF(*UWbHGbx2LsXmQDOJ%bw4Ne%dc0 zrcyEJ9+s4wVi4hqMe-jIa0H%?Sv7U+t9+-u>FcVCiM^iSqQ7_S<#)}h>3){RfcquS zf1@%DI6fVb%E(1iufY48fojaq7n)&~6|+7E;>~&8F7{uB#rUzIS_VpU2BvY;DI_Oco5N3G9FJ(_df)*h%q^sIeg6{0%A3 z$y;C8Ub1Po!czZ%>cLm8S#K>)pB)e_`;@3*{wEH>Na~c7GDVS!Deku<0bplTJOEZt zX+A+~oK!ova|n4>RNSypH*bZm_k+=rH2Q0rP1}ZkXkI|iPF#_K{Yh0S?pL@`)^R>3 zk9pto{i}QiCX`Jo9xTo!OV(nGNvby6WT1i^@q9X|uPYnO9=#EL1Wblvb-+#;L6fE5 z;8ADo7ddM_6X^C$HPlS#1}uXgHwHno?AzaTr)Dhlll|~{rHY2P(O;Yt_-_*Ub;509 zpMUXQ(%b#=uzyYzwn?5?8vGcl)MKX`b-|(+qy{sgPe8%9whz?(2C$8aCvdM5ggiz= z82IF8HWiBP?Mv3pqnn-CEwj=~Nt<|I&rT2srdM;b;x3?0@C>1&T$*|OR?;78uGB$C zJ}*bQyX|i;j3aB~db+L9tenC+>s7eVq^I+v zIO%z+!x;C@tgR$6Hb25cQ2e=BR@;s|&+&D>!tZti`%YKbS(Im)6*}SXS@r{HQ59|5 z({1^3vO|Q&aZ!5|ku^IV@;ZHQp7RQP7lAn|dJ(<;cvRjgg)M+J6Z91IJwGzfe0}rx zO3@ZJC$i!${Ykij7b?+IqAtjt^5DE=&NWkeKbY9NVG)RQR+kZt@i8_H{V>RQwNk$0hL zClKlmw_+ot@>fY|OeUQc99gFIf^nMF@_d!CJT8Irg8m%wRWmg>@ZgiL&)xJ4+=09}2ZoaYVb@-BLd;f@#wHV{xBjCMHj36uBOD4d+jUePz zEkVeXTm2$40L{zN}z9W{FbxQRh&o!Zhok*wH+r0_1-Ofl=h)0AcSHZ8G|;6JI_ zpreQw9UBNjF>*x#O;p{YdaNG#2ysPEkZwOjvkR2j)I-C7GSXt&|9lzKwF$_m>YdeU zlW~8)G%?}UpMZ6mOa=~#eb96#Acf;bR+(U^xL{PuC~%|Ux{*sCNU8-RA?gJfy`k}5 zao8{hdYQkH1Jm7hQ^Y522d@BoQt<~P*6~OqP|j}8p05Zq1o>?_LiXGI)Mbc6kPWE> zR_co-3YO_52Gh*plf%`SA=UC&wfP&*n2LdN^%9^B={s6w?=u;n`GTw~KYecX}p!PI5Ok)18{bzCWiC%yQq| zZPeIuQ%7;0mN&RXN+Lh4AYSw48Fo2Z8Vhz)xEu?jBXxNJ=e=WNIkU9J0EZMo0pXv{ zZ#zPy@pSk6=7y3QF~mjxq$l(eivkhSGSKQG%wUSt|M`gF+J%tU88KZQS*DrPd@sZ0 zn{Z<9;vsTP+dIM1@MfA>cGa^6>AzN505%@y3IHlqhs0tPnM%pL$}0Rjm^Xi-w1fF&|X&vubv1Gl=P-}H~9G56xb}Y>1!a~ zA!WVlNAurSP(g(TDAOgdiV2GHqkK4yagMrXHh2vS-y4(+!F=os&P{}JA{}vAh?P)pQlZn*B1#=S zo^FGKF*^#kc`zz9>9S$(yNXk76}Z}eoAR#r7Y8}(bQQQYIcU$I!>`zOBl&^(3B)i% z`-_qWexJ~Ny-b4GfbfTX#N7RxPP-R~-Lvu`rWx1YPF6<@pL+WPe$OdTqa&Tl{Sd{W z!b(P%N}O6N#AKRC@<&J`TeO@r&%Ywb*dzaQ8V;;n)uVeh-3II!aosIK+q-YHS+^}SDBUtlf=h3$<`ov z=kAJz2&lzgOuiKQ9bvtV%~-Jx670boW1*4rt|+{=-iCNq&*B8=co(P-=|m#g9jD}j z$c5}fgymL&XUKe2=b;sec@rr8(eYmsuj6{MlCs#|E5Qi&yc))I*FYQvTeBScp40ZX z*ISjj&-Kd&{V+8Sc7JEX9Eh#r3tF8GkEaWxx%aaJAn5Kz)(7EX6=hMl+;J{9S>>jt zBnqv{rMnA#^bEsLB$vcHd{6h4w(-kU(xq3X5_sbaf!CY*Ft~ERs3@258`&`Dr}G$Q z?o(ob4Odb<+inXeiOno*Eq}}H`6&@nf03Ck4KF_;IctDMw$fpkiBW6n+UHz@?k`N1 zz)5(xWa{IlrnmZG%CH46eXQMYOvgZP51#c5AI(5+oLWlNbb2vJk?CMD+NG8aLJPin zzHU@Kk8nqr5%JDI)kQ*cDiYI%F3Todfr1=5b_N(vpgX>|WL=gd?69diUleVrQ&oel zOgNmL&!h3plyGMsz^0dQbI7_wh1e`LJBg$+%2&4W?os^9^bcNtI&&g;a8)@5o zj^YPV4s$J<5GV~l$Y4NN<c@xka9T2~$16(b^K()WIOP0K@s2U) zAww8y#MGz9kABI%!J+WKV*>a1W!=$;!{f_r!9CPMNHZ$#ZEj82j}X4Iq+r!b?MFLe zcS2jhP)TD;QG8ids3&R&KZyI-h3Zlty70tzRWxtiL~s4mZpRu*{$$2~!Hg6F3hRTm z$oU{tHb&)rmr$y^=qiekw!k#AyZl^JndgbwVRAqP3lWWH3#f}-`IA>|rDsB;Gc0aTw0yEx!^9x|QVZ^HF^` zIb8bYk*@VZkf^|X0}peMOm6O&kf)T5Mox%nmj?{?E@EYeRN;UbI2<^n z`G`0hEpmRTk z6Tf*$sJsv9f5rPnAl@&z@MZlb2&7`z!7|86<&L!Ihdzse!!Zw){S>7OwIm}jJ(I*Y zG+vn5_)y+|(K4+2DS5?)ZP4Xtcd$OgWsBP6^}Uj0v;x&Iqf>sV@Q2tPc<)s^SuEAr z(u5%Nkh+}sdywbkMGNpOd=2x3_|}&G17*{i*K{|<1gm(R#svh8HI1a$e5js|fZk$j zV>}bYH^Vwe19|5dP~3+JE;`e|m>%^+^b2;rReh9aoQ7eM4`dNq8~46&p>{w5#r*PR zR@QmSjK|+{UoaNzAJD=diyk+M2(X9k(xa4-+~Ro$_YAu2uYtS*4YXM?4}@^@^9sL1 zA!0Nwm#qL5pFeeVJ{9Wcnh6nu2+yVrul!(I4ON+p1Wo1c!jwxa2O|Cku;_3tF< zT0`q;eW6HFuH5o>Uo6$EUtSo`(?MX}P!==L9WHb%sybpEk!^N9=~5LxL*!eWjR0YU zG<0ESV?qy^Dn_0$ehL@14UDyyAunZ^CW)hDGJR6g3IWo-6tD2d?mcD4-HcDlk))4e=idjqO)Sm`O};T0b^>>mW=hpwnGLQ0%Q zpJJOE!a$@+fI+E5qNjH{8s&2*06Xu4#_EsYnwL4ei~|mFXrBj#SoUaiXKym)uRHe_ z(4_(nUGP-o(gNq*sc0CSs_=XqDZ+c)z0kC4Pz>t*q`xd(j67koF1-w$rT4fQ@P0AN za`)5{PW7o#-aLk|*ZS%M4Ow`uJrO#`$~2qW@Tm0m|wFzW9`vcIcsp_G$FBEuf7j^gXt5CjW;Y?iiYw!!%*PXG=5yewU!!gK z!VzqbiL8q6lN>njPY6fc4z@xU359{4NeVH)W)RF-uS@uUpj7ih6hRF{(1IEM1C`{` z)p*2{OQpN%0ff0yRPQbjeUHMj@@ah09a6a6^AqTZuMnq^{=|u^PNuGzXB<9HJMGRq zI^H6Fc7@@d(s$>wc=Ojy#$#oLMAOTMlmfocE-kSv`X_uDktUBF1xdM zGjX9BFB*m$2{YXv^B=i|Rt#K4A#PbJKPpgH*Io6Th@L`6TD#)580>w(Ht#JQ*5q~| zI&o@b!&~8(Yn*lowe^$rU*W}uf<YyuF4=t#~%u6>U+y zZ`4D4Z60>i#}DoJbVHOJ+6&C~ykb#E!8dk@0=A+B+emmSu{XdK*!V;Ag!|-2!jsdQ z{>Fc=ae)U9m;f9+JieEb6Kc!H? z30x*POYU*^shGF2kK0uxXy);=BH-+1dYp#sxZTXEA#Y->?E8s}1f+Wk$;Em6VRnb? zPLOop;EHc+Zbsb1`W4^ztA|+W4QQB<-P=7^;HeBgLcEk~ITeOkbpwXxm`iqx7EqB!AoB zXj+F#t}yncfX2neAA{XF0)6W7J=+ysO0kUDAGz{J>v|58en=vBuI(v^?PJtS$B=(ZpQF-S<$Mt5j^RO?Mwfip@3d~gzXfQ!Q=FyBzdi2drY~0 z(HQ@Tyo=FU2!S-&7bl)o!UPf1X^1E%YB}C^$*O))dj?H1kqVDS>?@P!{W%FOLn+qb ze0kQQQId9;x2APt!A;k_e4i^C6BU0PWp>a5U;iNUVv)zu2n5Vvj)zO5KLg;zTin;-Pp!7;W~)Xh5r459aX47N<+l0G!{A%$HLT+s z7>tS($Pzq=-5~L5$)LIukscyBg=>o*r^jRvp{ZVr*|YSu-orXcb_XG$+lVW`VAC51 zL!;+^LV%!m9rpc8?SFtJeI#3YwEi^&8h(f@ilJ#gcU=>o%T}*K zD0o8ne8&+I{ZN(MneGBf=oiipPQ2fMm67xvSZ76OafGX1{+0uf2np+#{1pVSZl{&{ z3}m!q>zG?|tGAE3x#&svIA?Hn%&oGu+tWLF3EZ+C+O(_24ze1jO1s-WJt3&jRGf{& zgZzHG2HTn++cuVKdeeWT&r+X++i=5SnPo-#%Yr<=b@{}(tA;wt_8w?@&z6}KT*mYu zb@J14JTw+EGKIGmZ%S+e7RZKIR@+OBjl1-c+uA)m75-=DEN75er)N+z!h$JWDZsoK z17lii+^GjNXgkxBYH_$bM_LYNloe|4V9^-904?42T7y&H6em2=T_u2@t7-Y|ZqErR z*21E&3t>X-`A}2)ycKQN{59bNg^H)c9jSxs?vvNkEz&MO+4tD<((@U>Yo&Rro9|`* zHYc&ieQ{*Xa5>dikP+Qzm%y^eEzbZnA)0GPhtjovr->8F(GZy~Q^}?Na5}c5kIJHN z%7u}X;t_C5q6%R4K4reBfYN=w97)Z@SJQBK=UaN)=nV{=Lp|#O=Bn0Q%Aw#0A=?%l z58-tNS+Q2cuxp2tj0C0g8$pQ2ab@rZn!~1^|Ec^`e*`YNme9|Nu)NI)Kp|UwuL!~c zV?IY)8JL@T5Dn%C;DjhEv%CaN@z2YZux z-&!x`I%4OiJRmK*-XXE#(BlJtu*C1fZ?W zp)pQRc{voALY)32dNz3`us+qy_Z%zv}{z=#fHT-3R9fhMh!*w%M7%hU1t2AOgnEM)4l0biDAyAyp z()Wk=PvSnxbmRN!tXGV{Gr-T*kH<=4Wu41y_m%3;h#%CDN2!R=2iTfw2g}VCplLp; zB>NdN&{n|=D$xgr>lo+M^38{$So13Mc1v!-oFKzJmZsY@>4hTcirb! zg3~pCgPP*k(EO33yH4Os3h<45T+f#r_pBNn40wHcS{e&Ir-x|(8f(0?P7g``Ow za1opRRQ!!vm2zoQF^TqPtF6uYiVgt(!+eUGNljD@YHXqmEymDQpSJH-%WESsGrq`_ z{)uuEFAU#feHY0Al1ewo>hb%Veoivb8@5%LUZTGa#q7EnOpB@B(zlA#%?n9%2Zss4 zgzFkh1j+3(53k>qP}g=pb7Y&k;g0S+{&-6C#(+H!EFTQlcR(5OA^!2FwaqycV8Lp= z;)03N_XYmMUZ`gJbbD+aOUT+_M3IEN`C&e~R&)Q;z+cwPuWZ^haJuJ8dkujA}R`=P$0VPChwjY4r2EGF5 zHnjpy)@8|x6F-tpYXKlVON5ppi5*Js5w8?aR|;Dx<=0+2clzt29)cXE<;~?aPvM61 z<>LWk7{}!6T7fR&Ugx_TFTNu<$Q5quR>P=xAi4MuwdXYqPPEMIn}3#=)G(}kUq;Cg z_j_0^+2umnW}N(Hq~0s`@_il>=SP!1@C#w48l7R)cmIw=k$ z_o7~g1%U`aRegNs*sPOfQVgWbcb zK{r2J927Cz7md?K8MvlXzWK*=cmq;Kf}u)26=vhBAGpslpKjnHV&(svi|R*4eul5Z07ICd1}!(otQ-xPp-^il-6--0OXJt0~=!5~_0;ao5p zh|)lz83-}Do%U~6e|J{@Qj|Za0#~UDXUBpbs>gVSmjS+^&U9g8oN@|NO>5G6DA}T{ z@%T5a^Ky{FW4(;}RofhXtg`k2#M-lO^v54FPXC5%eubq6f=|ALAURy-+TKz#VAp| zButkxA~g@SXb%{>esfJZs4CBD^)6M`=*SpVEc)s|D=5pj%j9BTQnTDc1%OvbMFv0R zJ#U3eX{Jihzz07Z2LP;P5Q*vO|Cb&sB8A||EGM{ME`XwF45`i>@u>I^ibI z@tn%<$nSObtB*&H_q&-+*G0V&ci>g#ya$5|({^t}-~SV9M-36_CzRHwr`-5&c=i8e z+Wg!_grE^`RPctv{|#aN56V120=G|yh@|v&$k>pC$@|&$9eKrIYCim1oZ5N|D(;Zi z#Uu6udTfJVGWi-wJ^_v0%0nTWlvV;=x&*>W3LF--3;XUtcAobM>x&S(G0%(W1G zS0X|q0~ETgCcxd*vH~ELL)X{YyW4lH8Sp>LNj3mjqM*b#NAXfErjpWATKVJqjvYmH z#+LIg3TVVNpSRYB&2YwJJNkSzS8|fCEzWtN8l~9FhUnZh}$(2*s(u9R5xIDKf4-E@hzeq7UDO4Y)slx7$UN z(SBkdf*v?;O{>~p2TS!fdQ?FGfZV?f4AZ5?W#eWT9q5*JOgYuw6+ng{g*%!12@FSD z-EUZ4p6;T<1uA?1T4WR&TT73~=h@D}1-K{;_=(B3oKqA}EAY;2`nUQgktAaZVl8t& z@q!1juKK~AV$A++r7naZ}cWV+*GG>aem0W=_vG!{?|q!8LaqwIacc%!lb7l zbfWmEalFzuI(4DLqJPbh|Dx%!{6?s7-rYK(hJWCB090C(XaTkz(e{K14U;heaCb`W zoJO(H)mhpFpgJXCjDp#^@S#xDbiHcTf5eJI_$U|3v&U1)eOQ&nH3s}drLjND3vRT{ z3P6wIRA?cFJPs)9=03=ZbqEn1s&~YYE0kpcP}H&|pw{Nh0m54qGvTOgcUMI3U&LGt z&B%rvte+tvPL21jCS0VyN#1!%t)plEPTR)R<(3sviu}%g< zN=l&2)Ydj6YtBkprlx@i6Y?VS&5mn^2fzg8fNYp=V#osb+3%$P3E699Kas^x#4i43 zD%lJ?R{&o3_h9B&^K^h$y`pkqg3IBMmPR#Ifx`N(2R>bEG4K1L-yJkt>r1Mqf)pl{ z@hQf6`H+|44rvb|>(-}jz>Mp^(X-hCr~~AB_v1dV=-Yo$oh^u6*ZsiT&EL7KM^WD= zxUQN$Jd5Oa6e``e3m(Qvs=Vbg>;XWr{5@1uHLcjY8KdL{U#>g;WCmtiEsdFazt0+9 zN%sF5LX!Xb1e!8H%a5#ZMBexMR{-~~xps_*<$he*Ka%b+MpLf3 z$A`_oC%4rf%~>S75!@92zH92RVh8s(z)4NC+=49hEvkXdAo$K&60zGhjXQ~~CrkC5 z3IY%LDSU2K!fQN_MvV!Khfskw4b((&3r5n>!wp&Phcv7JkJO!yMli_OUKHL)?7mM5 zz|L#Q02DvVchQ5s3)ug4e=ELU1AwW*D8Tia>G-@j1$cWnTKA(iPrUPu0k7IB%Ow?E zAASQom)$DsQr^7I`wl3x`(cScPg=QcGfJ|*w7wSdkpz0DL!iiLo}xwVK_c_%xsYEW zWy2ux&&`lRK>%6(9}??JG!g$YNn=_gCp0)@j}b_k(Y*bnZpktk*bTQTa8u3#H^DH$ zAX;u|9P{o?d95R);H!HbKLj+8D+RHY+-k}RGSn4cD8J?;M1C#E&>{s{Pi*po2Q4np$d zmNv^nw1SxWq1v4h=578i<4*teM8aZoifBa8_)`$SWD*JM-PT@?+~7k*mtmIUS&M$h zzX8k3@Z27@_0PKCQ78XY1$Y78{~_?C$*uz-xAgAN|6 zEe5XZSspJ6{qb`IO8R7bMJ(E2wPao=jCs;OHsG?&NPA;vhm!Mhy+XSr=C(<5AZ0g^ zS?tQ0^=%$`3Kp4Sm9LJ#V5|-|qG6i6_GU&pR>7eCwI(BYVoQ8-bqq}Oi4Cxp;AJHV zemu*_{e~aB8Y&CZVV<$*Z_J0V+M3s#Vlp@4#0LM-IQ&Ml9qT@ryMhb%*y`za7K0W7 znlW7Nftmwu;l{c+xL$XJvriJrA}nzL}cS-6?ixQ}KZGE9( zLKbjs6mGi)lZ+^3ykT>DSn!AhfqqhX%-DjGsWN>=>E5ioyQRK!R)df!LS5&UCpdoe zuVo!q+X&kc0uX~T?Sa6dRYGc288P>AU_hI06e}3f7OlQ<>0>*Em-$FkatHIF!|o)T zdZhszW)i3%6hw*Zf=3`)`hx@Fj;q}rzqCFzRLu$Z4Dcj~NfRMpFjJs{*UQ?mESx-6 zrcndS=+7H#1BKSDk$9`>sm>=vSm(ckB3L2709#V^q2{?@df^)6=y#aR{8VUn>mJP6 zI0NTMqV_FUD^A-kC^W?}pfc*adrs1-y4zk=6hdAi3=5A%-il1q79Nz^e?^I5#bcR;!p`p3L zf@Z?e(-8b-g~R2U1p+a}CBkb;Y)BvQ4+i*u_WM1`+Y`&@Hx_m^E>|9@=8#N*;|BG#OtM^>ggWq zPUu$lkHVn}Wp@EsF;fe)##?#^%j$a;^Tx5%|K8IIQ8EcwONE&I@*iQdlG(One@MRh zJd`HPB}!x3plAO8k^lqBDbxnNL7xz_?W?HDh5iyOGh7eO6fe6vQ zMSS81N>VnEpBE`jQ+I;MrA&3)I7DYD1}kEE<*Ywkq$$6CG^ zg_Oq|msTIk!L(A?b~Hzh^7D`|GU5OV3Cwlu*17~tVGLL@m6~`&x*<2Pvzaz z7smVY#Vxw1^*V<(TF#9Q)8W%YC*Y3NI7q6*VJAS5AXbv{GU4~Xcke7}T(MgK-o-s< zwvTbE4GPyy&s`J*|0{kYRPbyldmc_-B;*PD=8!1nOZ8E_c?FLsdpL#GXt#2vVSLEq zf=;CaOZ(@QIMn_~wej<8Z6;^Jtp@k`fAlN}rV8GkAG!Or|&EC8q7snFl% zsr4u=kI+WTK?g4H^KN};LDN#DP6*b(=(8aCP(*qJ>)ewiBe{a)&4}s^Aa>MTv_g$L ztnD?vf1c5wz%)^VHuDC%*eSNE#ANN^Y>FurUC)?VFI&yszh>5^!dn{vwh{UHvn<6u zdXH)pWd%ZFRmQdab=#RvKGq%_iQxk-X_7JYooY1{NA2bI)Jlh9kAlRi2g|X@3HQIR zavaZKsXb*<1PDTay7n%h3yei>_{_3)!@oI)8Mh~=8gBxELzum@KDlaZ_Ddo{c0zm5 z;)(-;dei)|K9+87Oxu>dcn*(Ldx+1gtkD3>`|IH4-{7QzybK6LtCEyt*?&qC$kqoU z0p3zf_0X;(Lmv)5PI#pKCjGDbKt5ErnW{iBSO2011chfOD@yy;V!71azfQ(NSqA#H zs7Br71#@5L9%h75y%Q@r1^!_ILXJ`iag6q@P;Nf>LC#mD6{#HuCxe0uDWgzOW;4_8 z{(4!p)AV*%m01HpC`r~A>qTu`$LD;w<^2fmF*n=RBA5py?d<@5yhWYSQpjzAiWh0O z463x#QPK@@E{isY9%@fTqjLFOCJW&=3*JrKEmEv_8CYt5{4`;`u88p`BaSKAGdvZH z%Dv%%;3icMi-(DSc0K+hj%D^1VOzJk*@vMWcwEgv9r7Y!7S>YS+9!p)Do2PlA2Bp= zg(VwX{ZMHQ!-sQN;1S@zxqzkQ=`yxK8;A&-?ahj#L$qH?fJ1f|Ik7`3h0jGd_K~j0Tjz~;03Q=xNlEoKo2MPnvmFX66 zNYyA=WXjx&q*AyoRPoa4lQbtq2nDf!mo!~;e8VAN`2RH1WIgvQCXCA8GUY5QaxaHT zUnm|++%BW5Un#oDVk+y(x}Za8+&+`cZ1Cs1=}d+i)ryh(k})amlY+mUord~QMjO6( zi<|zH$G70AwAw)q{lvY{m-RkeTvOp^vvns3rw_6&#W;&$x?g6(-S%^g$cTS@I~7EK zVI+praJ65o93-0Bfs#hI4$hR7ybWvnxElIxPNP1qv35>8fm&F0cgn5xAkj}^-}YGz zrJd#!D5W?*@bY!32wb506}mxu-OI1@BcxE@Z*yY09E|}Xkx73Mg7gQQB`!AVZ65AN z+h7+O6x!RIR8I?a=P^fAj*L9jJ1dUYSynt{9XhN@n>d``W?7oU>rnn-*l-;D>q)u= z87KJWWxpNZ@XtAW=8OnuVf5}~ZalDKw7UmY*6GBbH0Wcy&7S61;hkPVZ2h{{-|~oQ z#Mp;g;w3aVn}y;r;cmP2-@0F`C~611Y(<1vh7BV!!w2xa+K0rxed)Rp@s_Ex#fJyD zh=L~Je$&=wDE-h`H0Y_A701Gt{Yi2F%qPO%5M313aAF{JrEB})if?XywyMs^j+T5Z z@1}eCQu~q9{;=>F$=Nli#VYfmSkHxL8GW?KZ1e2#x<61_mltTwWHDsko2u>II|7GP zHL3(^A1c|5niS`pY4;K*q-1T3Ag?zB9Y}rLJU$1HCy_oXs*+Q@j)U0Lj z<`>~&aHrw0kH0;V>mmq}?}hkq3W_7QkoBI}x4#L6SLsWUDcfd%@prKA1ZHJ-yEA*6jF zeKyoHwp_^%ODc5R3i-nMI{PtxijFrm<|A}m!~{A18HZRknu3B?t5IJN5zr;KrY%c2 zGcxw%yso2w?Bo%NRFo)qVKf4>A8_l8QkC5#xiMm8mZ_tsv)!qYCu9`h>^39ddHP2}bxoc7W8$^O z&Na?D|NdNqDblWXBW;K~y#bu)xZOM7oWoY7OWQqG>8ZMfm*-W>QI%1VS{SE)miXSy zM|MsW-n-v4nRJMrWc}@RN<-mz1AkOt?-0p;Au#lK{s7BXD3{sIKLtsEfTR;rSH z0dhtBD1#YhoI%NnYTCBqhf&ExW2@qgm2~Y)FF^acz(*IE-)}$<$UnY0Zd$#&Fr)8aTpt|l|5wm{p9kG92pL0~lBy4F3Sfk87TInL3 zqAj!OeZ6lamUz9@skV0=`Mhaa!cVlmH;Bu21=a;aLc3TxXSd|ZY#CW&?01reY94JJ zYY*&x=tcU%3LLmP1_iNRaw88_eCkjB8{i_J^Ve$QPYbE`SAS$`&`;|qoWAe?eyU^9 z7@2o7Xkk%Kh7qx~rl4S>IX9BaRVID`P``Bi|PVgl$3HtL%)wbC@+0i%sT;1h>83a|hsk`oD7 zmmmtd+no^WqY7)HzTbh&ufj<c27|WV{8uVqD+29IsLl!N1t9j%4efe=sNhdx60KUlbrT^Y%9@ z;b{3I@96+o2a+;M1Hu`Nh^E0BdCLbjsCD2Mh??-(_yYI%$$Tw00slb1?B<54PuPp+ z1h-e|)$c}k0<}`OZLkkuXd+mIvAA}fcMv5 zM$f%zx1T|wRd{DR6N807H;>Srje2qBUd!I+1rBAx6p6OTrhgF^>MB7vFXN@zLSus^ zemwRoYG=G}1ry7VK~TE(>ms~gSj?vuChE#am7a#wn&BmST+g%;g-n4+XJOiRE{c{a zrg{o+d@hv_kylopN(ODG^^9OhxdL_kB5542Z|l+Lnog+g%16ooQY&hDV=;dc$JL*B zS$V0Uiht?Jurwwwgajzc9kC;b)_T%6j296o0^qVQxn#GDJdFbt2W={3aIH^&7LXct zLg_6cb1SUxLcaZs=_~?QTs)>^8Z_ko_JsF`2xa4<7tha0`Ltt*5a6m;Vts&<^h4Vv zm@)eq+TJaoQ^ksifz6UJNJd{PRvI1Vbn8Fo(S{v|d6nIjAqsjTKF`IW&EhT0w}Fj| zx#bQg%v*K$?K&xuD7?Ej0KdSp*%8)(;necDO29?0bLUt*(IBPN(%z2E5IYZetjaWc z6!)z&4YYM?3QZafYperS46!4WSeY}#vFXXhXwjTV{0RSKw2ltTI{PDE?e`p=fF3x$ zv+a(MwL8kq?Y6!k*dcTJC#3F9C0052$PSIF*q{b zVmv5BS9`82z;>m`E^z{l_VbnDFrT2)Dt$h)p|s!UBQ<2&+9S)7la&irMVIy`3GMac znwG@d!Ie9+9YPxNugeIZOdx}9LVFiGS#4)^En*|JVTr3x68vf}}Er(d>F zr{T}mhkUt(+(ePFaOtE0{gwiXT-CJ{u7k~yjb$}Q54$qA*&zSQqxG$a5cUDq{UFrk z{jq{L{OpdbHh!GVwd}h{>9HwWLC4)mC>7#)ZiFpiACJG^qv`|{2PwN{?}O~ot?a*| z6Nx~oZPxE%eIW_GebawO+CL95&Z8XL{xsEn4>pSg&}S3Guk#b-B=DGmF`8ly(iOJR zUSx7_NeVJu5Jo?qs3Gg=*!i__G^gq-7iX%+PH(w6$o=x$qudxq%ZG`cN%Q&t6rEr_ zTUUuRYUStt!bZYHQv_qJE`_JO!A~U#$&ObM4dmI;{91XfS_b&3qv&G=Y{&9vM~xQw?*UKrf4Y9TdI&FG=g!q#b+fF)0>| zSwc$I;sWG}+YrYdF1#L0Qd zoD@7bIfU}jer5yysUpcz=DHT}PFuK`mq9pG$>LDkhjFCHJEywY2v zwfae;8$Sq)%%)e=9|Wu+0!*Us`Bk7a(04Y{cU1Dk7JM3u$Mj zuZCT9rjC&pU^`-Y&?vVfL|Erj!&#p&GI?|$?!M-id=FuBn)T5rn(y%5suqGfS9Mld zcJCDU#|r-EWnqA)#Wwpbog%&dj{LtKnEyCp{ly;nC$lVV8KpAM`AOp6*1^B8ivRg1 zpy<8OH~U31hW_7c=YM|c-`6=tf+~)e2pA|Ivpeppp8$YYJUF0Dx`Rfh37{OA_~WbQ zx?^zo2Z##oNVnl9y!zi;^HMc(TXZ*jj@zT%%#D}Jy0f8VJj0!TK%rIv%3Ze( zg%|Qa-7$Mi^Au$c#u=Qm2NMG15z3!^&z|ca9*~P?ZE*)B=Q_AWr7}*NLoJ0sR5oT` zgoV<1Tz~sWhkZ0tra|V0VzN=Q4rt?W?}g{n{=;-dl;3E3k?F* zpVD*(E;bE8z_+V^LUb==2}11Sl>m+x*TJ%zu5dVdjRUFebc3bVjIaNn*bQIIR2a{M z7{*yTU;gqKgB9o};YIIY%L3d1=&g6UUoL$eUm&q8GyseC%aIbjrb(M(@Sj;sI}DsC!_%Tc#8G5|!Ok@ert!qGeZd{}-wkwPVMZzG`BBG1uDi3BT zT>s2;U)LDaG{66@(M*M=3ZvU>jTNY!S=dBnwbLwucCh7VPNUyg6+lUSt%30nM_$D8 zUZXHE?*-K%_$}iVe!eB98-ko|{%P}g$=U>(`0GFzXVyG)18UvVCcvxv0p#y;s*pw? z(EAAeJpB>cYC3KQI|lhyoA^HEVuk&6s9itj1T%5fCun3u{!j3}a&W-=B4dWFrjq+J zV^Dp8f}h(DppQ70Ki)Xx18S8b`0lJXyA^SEs;|Y~P+P@<~t{oMedD+5Ltr2cyQP~brd z0kojmCyo2sU4UE~l`AFS(H0X@=@Vvdn@7b&QbjX#ib9oZ)Q11L4;IX=zg!MpZ$YN4 zBz+FvHnh(`;?tz|eN+Z)$0V{NUH{9|0DFX)Fv9TpLE{SHg3U|y=VcnY`re2#6U91@ zo%TId;6xvNC9xBs0#uu7k8VaZ&LFyff+#`;0n(4N`wQK)I*OTxdExQ5b@y%i+{O+>iSHvy<;Vcpe6r7%X?Ven6Yx#G8xdV%otALNTf67R~px`WG(*V;RO*L#e>ceep#~*aU2??YE+RHIS~Zwa-Q}Yac?8M9X<_Q1$GH8 zaI>m30OYh#z@FejUho)%a7gwO7GKRAh8=+8;FQfM;Wg-<>_#;KKmIfnK6ndBA$4`y zm=hS0=Y;Wq2SQG1t57uO7!6`fFC5D%Mn#oP;e~np;&-~UsKZhIFcn*DJ3RjLNyBw;MXT<`Lj88a-}K3T z9@YXhPxzdFQyoHopbZCob>-@1u7F9eRWxS^RjR-YsoCsHw9JaAs~3#^tf1M(c9;o&U! zZbe_W=RJPUBW4P(n`SRDqXqW{3v3knnoH&dZ4PxHuo5=*td`h^^v*i|h z3T^*UhJD8tOuLA(aGS9|;|y>PyWDZMJ-Bjw2A2XvH>K)*-Kw1{B~ACg$V;fP!hdqC z97DN>Cnt8GR6Bh1R#&z0S7A|e+~pvi$GA;Hyb=5*UArDc^vEx)q9;za0sQl`?z9Bw1}21l3?xlm z71sOU11!iSM8df(_!x3o)4!g8FIpCbV_OIOu>IgDZA~8x&K>wWKJ=bQ09(5UWgW|^ z+`>NhL4&Y@UR25ycs!oIp6|uho{dL=LHS^(v={0&Fo+5<^Wl;lN<)Uh6S<+bee;Ec zp=1F06lSG!{Yx}uG6vYnC5+&m;yTE0_)Af3v*3K+jh@fJS??fh!-!g!Z&czl_4Z}{1nQR}V7BFnqXjV!y33;dd z2jSnQh}39$U>cw=&tn25k~>G;jgDW0InBQtU%QNZ(gHpJdY!i?2F-r%R-(!qpS5sm zCFn7|!|YBc$~Ud1-4JH!L`#tBsN2bp^ObK)PiHyORIM50yh2C1LKb}+e+Y2M0Oi`& z8%u4rVlmQ$@0}|!v)i;6#X|1^V z@m$6xp8sSu}}&?>*s&%*j$6i?-1idi=E2$$f4^OI~q7)`GaoTYG32qV8RMB3k$ z?|1@1zq)Q`%D<}{o|*tqQ@&i25STM6@tl3}vR@c6*pRfl4*1m?^k{nuGzxYvgd{-y zcd-#m#*0JDAe@vU^jk6PZ3q!|^y7sISdBr5Z4GKbCWN3TgBb{f9iPo9eQ;!xzcdVi zEzdNxk~1g8HD7&)9T@={QWW!UCJ6I=%AOeKObg$2zeQ!L%VEO#k5h9eKw@z81SAI5 zVMiO+@OzAmrZhpkCk4z$Fc1UZlo^$eTPrNc;aJ0;BLT!2#5OUZp$&#HQ24`>0>x-TYfpXBS{|7FH*_A&t0ZW^6Fu^{TlRMb$Tq7er-V| zy7-lMr3OeHW|j0-D0+DJoF{6$4#~Z)1D<^nP%eP28SYXAWo^oz@8t_z$<;&>5YU-l zpp-rO5kM4AM@_IsjTG_oZD_f~9xYiE-l%&i6NO}v3h)oS$IO%lrDcsQbMCQk{A7<&$j>@$i`an38&$eSryX}@r$lAlYHLS=eS=lN zq;n|AerXimDkOxG_-8Lts@+lfl+1qSG8<;X8KVb6p=&yA3I9d`Wd}bu7qCK z9ky9B1p1G&vSAZM9@d*m$}JnRjp9S}51z2i275f0VHX6^`pGjoTuj+2D2$ytE`bdTuLU+M_vnBH=LV0d5l>3*)Y2XYd-HRTJlWhOQU(U)oM7* zcN2!}(MG$*w98`Uy8C6#&j7FXj@L>wo~Q=x&!DKa%Wn^}q;3CWqPUsB#KS?;l*i~A zJuK*kwYGMXO@q8EAG!2xNwhmJN6BT8y2jdU-s}^REy)x_%n1H0~CozzS@zkyu~xSX=pZi_oZlmkX+*2z!-u2PGe(gx8$M8W)vA6!@6Sf77f3Z{6!% zAMr7GL6jCBrplmyfDl^Z1XFEhm4x}%S7C5EbC#t{&$MMUZ{MVNntcHA{6ilUfW9&g z-okH$k46YAUe2^s)_;3&wHEa6s+6nIy{W7-AF+J?04tx3~MymFpFAu_qii zqx$(y77yyc)T5_a2X&=$$(*0~h7iZR)+`jEM4g-L6Nr{BN}ETrAWC}`{_SScFuV$0 zfBW&9VDV3Eo*J{l9%Oq?qqIW`N1VRN)wt}_t$fp{SMzzKW8kIPG32jFZFGsW`#rpW zovm}Q6we?!X1%ezonUNotvBloz|HMH&J(!!x%Sa1_$FoIS<{Ua_dQgW6%bJBgJH@& z{)W&bmB=Tj-DpP&EV0BE`N0+VR2K|+BSt~Op64q@6HO(=>5l}d5dvTU3Kd>A(2OAk zQ*CDDZlQ7)tBle7|i>3#iKpBir`(e?g ztW=Vx=+CiP)7;{E`Q*!JT)BUS1)98PL6s}o%^5yJ9DDP9Qxn5*58~%P*Lol+@R283 zXfjvYzRdIp|?mwj+mMw7$IMO7S5}8- zZ|aJHf4GIwLBX2cP|wiR|1k%4Vo6`V$a~ zNO@Em*L>Ivl6KAU({D5~21`-|uoWJU`mUj8-}r+*vgfGSAHnm@?=gie`!tNfsQVq> zHc=N12-YzAwC*3;!AwTn)Y31)vRqI_iCfWjm9XdYrGltCkZb=tucVl~IM?APt?{(rEz>ty)Zr_b!?-^GVLmfhxL3uJYW}3_C333c>-Ux6KR zPeIp%gl-1bl-aAh&yr{+mk zx5SGeEnQIj++s@B2*ug zf8jzTy3DYPz!;go$R(0l6Dubv#CB!F&Q?Qa;}u_ArjiG^Z90~g6kyyzgwCW7x!-Qz z6I{X{qSiVXH+iRZKhzN141fYs*Q$zm0Ae8Z4ZQw_$+LCM;sN}wr=_0vN8w(RBkvI5 z?40w)F1|nBnO8uk6qpUj+RJP2$oX5Gf8KqaTz&@C@c1LW#nj*K#J1=7Kkh@ExO%$O zEFqn3P3y?qBV0~Za!|n+vqBAt7hJxuh=#s`ykj5CbG*9xpa+=+rfyPa+nL7ZW6Z}n zlhj;Sg}=7*y(U%n(^pa+477S}g{SZ5a=1occ{Q#CRYJi|nqxAm7tgV_KO+dWvPpDl z0yh3=<7XS7BBxmUxCdLnjbF5acqUDTl#zZ(tq@(F>2i(cpk}O|HU)qwJYUZ3 zoo(;G=#B^n&7cV+Q`<`zl=0g)9Vkp-3F3`x0KLvh)?NErkZ>w=0my!aZ^*%X!@(ik z=~>E}%55;`0IDcwc8%01A(Fe|yvz$MK;-SF59sY5;9d!tWfMn?*j*H5umN5PM1Sv| z=Z}&=%ieIXC#(%o_}73(lnnob4WM8YldQ}+m5ED*44pPO!gw4`}Bi;qSb+( zGf)EBd-y=t@tniEl5MyXi6U_}ZBMLmPpoK0*qm?}ygbu`fT5%(5ak)BF+G=Q&XF$& z30xD=4n(v;NQJP;3L#QDjWNW$6>3Cu5BAS$-*)59V!=sFNah*to;M$7VAcpM>t^rue8SSjFhpf1=2w$cXa{9KHD z4i>Md%oLTxuQs=5>YfQI&%3i+Y~RecQe}E^^M@S zciiAXQ?pZqe~{tYsys=Pp!67pp;#jno)WR9HK*`$h4CG~vy@1=Ip4GF=TnICn|C@U zlt7^{a$<~XIpk*qJ%27(IKLzD;CcN!Ji<>ElLr8!9HD`_vN&$LO^a5t??GVwe6-C~ zL1*i$)5urF6Q+h@F5`UJ3~-MOAOI!I(e4(#UCZ}%q43?5 z887$MncwP15Y~2`_H-~_pC5fCA!&TZ_{siNq+k!c8#STnR7B?oadl^JFSyci6n#JM^y36##_k= zmV)4(%3*;tt_UyKAj;%8AH5In$9uI-&4Udo4e~wuetM%|ymMX@*|;M&yT&+WF#Fw@ zg6b&K<(E4M4k#r;U+hin83G7vd35TJa8`k0Emih~pITk5M{E>E=;WM@oh-mwqS6V- zwu+VA^kjMz2XFQn#mZDyd@AqUs3*D?EMf$?M{Nf~?7(w@&YU4*-h>P)lz)WeW7sHO z#3ZcL;_cTJ@T7dVm5oP^6YMi^TChXuqTQ;ytIDL6Tgpm=zDtDY8O(RcHSkA`R#*&D zHs=zo5vp;q-w7lylSNJboGAiv0dF-rA|TzC2-9@?K|U=r+(a`f^?`$(2|ZgeMSJ}S zO`}DQS`go;Qqov4i}if%fsi9ZMrAxB* z!mB{ILzhKK^9hu;FG$xLU!%<<=I@^+^j`b|SF3o2{m6@Mw(Qgmi^jrp>c(8i@6(?s@hnR))mdO+4Uw;BuIn*Vq3(@8D zAnUXIbk*<`Q?hHlB?Z_B=_wXm9lAC5HNf%ZDCZND*;xKyC5U-#s1It?t;OV5tdzTTbyoLC5|4pNA5s+^O=xBAq=b+{fZ$> zBKdTmRzk83djI`?g8d~Rz67Ky?qQ`hs^t$ztVvzN&B}xQp%w9R%n=^1T;b)kuwK{* ztnvkW%1FS7V-3fLO3BbsNX)={EA1(<2>WN^DP-GFK1vQt6;JcplJ!a2m!FEg$Sjjl z(NN{1F0#t{3%{(LaQuLDJuTbUoDoa+@~hb?K4Awf30zf?DT3xv-I%Q$|HN;RdCf9U z0vdVBPnsBi*3zJ2D~eWHFTYwmE8gNngEys4Cd*P%{plEmx@UYNzjAl3mPK`vZ^A2J675}KTy*IgbN7Xf?rU+Mv*rn`!2~_-a)2F z8N(b)MLXkinZKwWu{(s(%p^qrw$(WK- zKKXYhev|?ZpLMhmJ^Jzg7>NGU!16C2%MzB^hz%K2+yDK{|KHhz;3M*%5tN(dPiW+k z{%({0fB(t5Q&fn>#k&5LrDWLG|1z3PyhH-R{~Ua7h&T-lDwP1s6c<)cTV#XYeuU=> zE0>Y?uctnhg5Hb?ILepU+r7v4(p{Ti*pvQwIHerJENt>NjLf`g38M2W9!wi~BaCvU z_ykUmZ>6>2Un6BChu59{F;GV8*}uxoN8$@ ztAWqe=FjjxOfV|3VkTV_xdM>vCsf9(JjYdR8Q8xG%{?2!fe-lCR20~%MerLAnM{8g z_#~@s+}mP8xjTIiYk;w&>(Wen2gY*)lM!e5fM&c7N^vf5*eJeJAdrM{8Y<>vLEhyT z0V~IoR7qAf!86}xV0g0vaZUaauplL(5+OY$F@oa(HLFT9D|rydp2I)ByRJKong$yg zSY5ISBI|eV-zkzZ^H|n0E;@`ihtj#WVR83jFsks1cO``VBk$%i`-luXaiilX2|I`M1m4!7M3_OFe4~6Ae=*8^OV!pxk^OD&$H zywh^%Kb$mzEc=n3HM$Ws`e}fp2x!x!CSZ~q4F1N*FipYliU(XwfC9fFc{0#d4&sW_ zR5IIK^Ufg_c6{;5mSE$m6+~{cLzQTG1xORAi*? zN7zcH>yA~!JP3|SlTY>oSJJ;yyaKdaPL$-bgqLBfVnF{Iqu&O=TRbd6_d}> z=zJZ(g!bj@SChe1a6)Xz>RB#+0WY49_WKU&I=IFl3^Cx8Ev;Nz+ zqSM~Eq)0zX1E&kd(g0e(q7PakKqCFP+DKiPb=gQhzgUfuON^3z(~VBXT`G?C=i>|b z(4a57RChKKB{G zuv+1Ru;4d~);f50-_B-GSU26BQ*9*mVWINBhy=4YIqP^a#w1_pqv$L8(#Xf5lo|?2 zV&@{km*e_Zf-a3x=yOtsJ%pjEr6?_n|b=#II1@pgRos(>b%i?O3(I*>4D;@!$`i}AR(tyxgz zvGs=;Ufj!vZ(9!wjWq2K(!tNg0;5G57jS|$XQYOU-|L+fRK`Wp{>cullnOo6_{GgUuT%9&V!~(B^}ZxH@9)>g zz}nLY|Fiy(x!Q4H?FTD}F2tU)7o$Dkmz&pa@{%+d8k zQ{$#_s~HX2ncUf^z4P)om6Bh@rja&r&YG+GFSNg#R?g@>Tu&gDM~iCBB_l>ZEpTI8;i_Ae zfe72%5rW6YyS&o!9j`-Pp6qhC3J*Be){z&og=on%)icf+EXy4e zO^1E3MSU-Ktl@Ol5f#(1a5I1LWR-t?@6s9qM3*8v4vy5kRnvEFC2j4?Z|5UQZiyaX z_nKw?tk@|=?X|$J;n(wLY)hR)@h*r1(x-bY8f~>Mv8D^xH1|1AAg!N9s(L+Z2Yj>U zaOO#5bNW3>u4&G2n_QFHx!u4J-HzJlj(c`P72h4mAWJWAHKvO)35VtALKbnR=V|$O zs(P>BkBPnoSIrtuQ(je$61MtB8aKx9V5VO*9C+Ai>AVK6QN@-oz6bRmN*qbSrEg06 zU^Ea~MOL_EKh3!Bi;qs0Zwq=Pue0yYYnmD^mkFCDFDYiAtbF z`X<##*~p9^bYn;c4^>9KXNlpth#?;^$URw}czhz6aAdFVBla5i)w_xsgXp9njyL4vF@Lxvgz^y^eKk7c-2S3XD-+_|`uRAv0R|}Pf2rZlUcSwgX9Z7AbgGK3Wv6J5 z+}DcvpEl!GDO`NJ>jCk2rixF73zW85H>EHl9dmDG6!bU-8=)NC!c)t|E29^aoZ=}v z0^ao-L)MMrpqu+Gh6N{F+v5uN!9i{A(;s*B>ALn$9U~;N!%rImz;kJsZwBfI2y4gb zSczxV$4~L4WO$r0dODtpqjA5Si>%r1j(>4{4HY~#GnZh4st*^m;k?%4Blg|?EI)gx z-o4YKC@o;PmXp;`SMMjlzSJ?9#Fi-yM6yvoagXEq;qXka-%!Jq(ASU+Quph+Ow*_I zE@Ns7*CO$Ge!-o&TaDmpTx}V)MzsWELVCt>8sFVTv)W~S$_h%3*KBE?oPC8sFU#Z= z>nQD-erME*0!iOXE>*SIPJMrNMQG_!7VT@?h))}e_01~oR75S>xnUjOV|q_H+7wb8 zyP=QhTUcq@>ed-TJeEnLB@@31o8%h|NQ1eK%i5;$s*X#Y+i#Evz|b;zp0eg|n|VTo z%O3M5DQckeW}C3M@?Aug+~NbRDLEUrn1blD2+<4mBgto(whD9(&kJHDgJVKJw8aba zByWIhQzbq}lFDaO2D*SDT^o$RU@nzpc#wRhP-0;>+ktVYI2M5fw8 zycLinCAZlrfkvo_qFIKmj5OhB&Ad zQ5G07q{JReQONFQCe^Ls&yZ_X{Yn5V8SHGe;`aSkl@My+93zIRE zzUSp$vU8!cW}D=^Xi4mG6o<)Rip~on=WNWLM{e??%AO8ak_vj;%kP&=_TqLbzeWgJ z^5Qg#*ap3q8(!P?rC(uCfUpV}EIpNS=`^im>r{HUzi4u(Tx+0BWjkiiZjkG#&v{(m z>qFxMhLm70-GT7YODEz@wp208EeR=gORsM5V6SZ}=^efvv+*|7+s@FF)h*l`Cn*l| zO1*BKjjzNmMENa3(y3AY<2!W5F=--eZk3=dHEV9^gL2`&SOA*W-ER&Ur;EZ$wiWjDo3|NUo(^pef^n1-ua`j6s$sU zNbl$pygwRF)$bobn7X63esAn;wFAoVVkuYLwL5Y?x30ZI4GJe}*;P|utoCsGrrFPcrVNunM>l?f} zc`0~QQqp)V{*}1Bfio2*S(`71^Sx%w-@e?cA*=j7sC0t{Xp9TU7`!4HRVSQ%T z#c4@?mjXkw8bUPfVmLElsJ&7Y2}ZD0Oi)el-ah`9TLfFZ;h%5cvl?(!S#}w5l1mfgm-W`TLWK#B<>ywgJx~-GF2X8*KZ`O5J@0ikdlS zVoF^K%R6srT$YAe(P=#ykFA(+<%5I+tY5%xnGbmPbQiF-Yo@5ZixWXky6Fo?;hO$a ziNw1!^Yn=Fp&DWJ5pbPgfXARyA+ z-2wsw4BbkXNQX2MA|>73pn!mM!+>ssrK&-Z(t zU>O|-jI|_Fqsw{pi&U*jF?i_^ehrr-viw!==FoKTE802|#)M-P7X}kwotSG;Wy>${ zo2n{1dXPl`8x6oJ-mZqdZ?=ZN6XlO`H*0GxGuT?jVmEcma~qxUk;$4m0Zp&r z9oa2*Ebf&s?qYr%MD!*$6EEHe#z`ip=4PFZEX8KZWzwdff#lH04@qlZr>=_=aL1JvLlVX0g`SkK*LUYJZiOrtT`QNt-h2FR`@Jw7 z7s8u`-0)iDTYbyfUdA2GBdv>U=9Prmo_c~&j|XJ=$zp#u5^nh1UBOe10OkIgNWX@f z1--6@o9jtpK7+O_^`p)B5rW(GX}6tt&diC>T(!oaN zoB?Waz4YGexfpYk)hBu*fB^pq{6o%dq^nhP8Z)?3;$GVHw+9E#sjCs6mDmdNifRd1 z_vAy}-!VNINM?;BZHI|sVK#%@(7tn0Kb0+SS3Ny6+v(>8c{2;K(Mb5jSNbH zaC$t}vD%(1Cs=>s{n4sBNtxNA6{s;i8dMU4FH^${-z^75CIz~}GARf_xufxwt|3CClH1q>3Grj70|(k{GaU z$u=7J9!*3#@M}k9h@|+Q=8yX5t%LX^viTg|b> zu3@-Uz(ynF)%lr~ODDQM$VzRK?xi|wcGogyunk+kpEPj(aXkA`4Cmm9vjF+7QeQUJ zL{MZky>@0#QdVMUfsdRI6cO(I-E}$o(!9#?z16*zpLG28 zSk1WL2}ja1C`;(XzwYd~PS6A~MOoxE!3|*>QCm10pvSlJ`_{ zL>+@ze!y)8Qdh0-Q5k|SEU_IKv~~P+D9gb}{WRWuw$wrN@2MLJaVTpWJ(s>L+m1|| zim@V+t0dX}@;RD?-GMU}w|rM-#;)o|1Q=yn7FA|%U~N;E!I&(X||NCNAxur10W!^Y^m9Zg}a@!YXb@>wg@dwXP|(N-3D5 zA;lTMhn!TNYD?N&&-NQP?s^!xA6E*!|K=LHx*W}o0po2Q@#y7|yD-A+8dLxAubZjf z-@)M+yPCplt5sx@-CB|M4}DhpfJ&W|MrV(~OzUNlvhUZZfKyOROiuaj74oQqM+$NF zUF94v=v_%VgUT0Ay3SK|oZf!rV19~q{Xy2k-Y8EyMb(E&&$s}SW3qC$(U@`! z%Bwu-m~$poND8A8!l^qZj_VUYkC!~2Ir9<6=BR*Ej@>;EF`OXI$b?fGTmT){Y&OV0 zthN@Pms{J{(RlQpW0|z;;Y>Q#n7rn{f+S|fht&=l7aC-vaR0*i@D%^;u`{lPeW!H! zy61LL;iqPLXX3e7Xb|^Gd2sGd-^C2Ceg(L{O8h1%SI^>UT+s><1>@@nz=)R(Y4~%@ z$Xb+k_ZF%z`xXr(JI^OILk0wRYQS-lM2uIe7h;596V z*n3CELTGMk;2{6~*NrP)6L-s~R@?=PkvvTADHh%;%2&ff+IAt z_=kN8qN?`u3%|L=d>IIlGDD2+W6zafVj`b3YKtA9U^ccC`S<-RPxv4}vGfpW)9hKO2Mb>}t5ybZ~Y4V|HEydPzpV`uL$VJYI#7<-$zSjqeP>`G7$ns|u z?_bCu8dvf}f8W*Y6fTVjW*L4b{(TCvJ9hVuxgL$&`rO|;Z2xG}}@ z(~+UF=x*=H2mWQ#t`S}vv@~?AQ}9&Q&_9D{eKoNVsy?CpdPy|q16PEi7eXwQV_SF1 zqaY^Ga5;DAl@^2Za=GZKJLn-yF2>CRF5vtZTB~uWU=!)zXV8_s+|bWAYbN!Tm7HHm zH~9-Q-Y-m_1T?CQEfNDK9Y5j9`(pG+4M7RQIebsjmRvFht!_)DnH^{%eK6o7aga`( zxh6to+j_w}3u-^0+MQ_OSWrvIicUAg#)}UhgNz=axn||IUM}nQW$_vq8Yn`$hO=rY zc#r0WL|(KIcx(%AyA`;f_HM*Kmsl?x4h8k2R-b*<0Q7Jh3{T4B;`~+xR0FfYx*b1H zLnBx3pL?~#Z#wa9!0;-fy5 zY_z4jlm)DLEme9AK+$bG&)RI(9zB!!3N@Zvtgi$4bh>&Tx?-tuHedZJFI%;AjW-&2 zXD?T)g^SPSH5QR?l#-<);XR8$AS`2VQTH{);-f=$Ktv#uYTO?S}3P zHJBss{h4J1hr4^+7`S=@A9?(IX7;hpxL8a5Fh&qZl)q9AZ|-vW5U4*0$wj-xr;COt z)wL_fDD&Has#vokXn>B(GH?wDn6z8gUe5}%>Yv@qwj&b*DpAHud#~T_B0Eo&J0AF% zubZ?v{e;t;eUOY*0-mqOzkq_Nw1uw0DqpxZg648O-d^^Jb$B=Nhz5MBrWtDLn`b=W z{;s2uTw)P7qy#qlbQn%k-5Rud-wl}=r?L&6A$wbc_wiNxcLh7wLljs(&x@{x(K2|N zYvSDoGT^5WA)X8_Dby z8(Xxas^_V3`D32mMW{J zD9|3jpUy#jj}j_r7e+mw{VkBZC@L4X;xs8I8=cnbewjd}|J72zrCB~h*49TK+F)s~ zQs*3G2veMN6vP?24HdgtIsS+1;m3JfK_KqOdg#ZDA+%+EnQ}SG`s?>@!KdFe9N)iX z^IA}E@0(>;wF2YRLJM2E)lLHfw^bc_rVV+zwUjB?PF{1BkuGVJT4-a>SJ>FAF8M17 zAh+zwbLLA4>V>!fqfA0TOImo21ffOHJa26I66rNbpINjHc6FH<>NG>dZmrbO&>7fD zpu&8nKn*r#6#0Q~vA&uxj5Y(6=Hd3ub{K^ew@InDgVRY5k0t`_0qiN*-Xshy@VW8R z1;v3tO&q(}PVpfj+!_vCNMuj9+{cHhk49|tv{Uc=1`W%xERNqH_+~Eanj};ky zNx&l$+g?z9TI~5(XvKj`UhM-q$H_}JM2Ekngn_fUQq4cqwKwxFR-m|cK4oyxXysu~ zJG@kpUYxt8pb6SI6E@Bcz%FzAR2cK7r}{HP>)gEgSk-EH$@=c&8L21(bLobpEb{p; z%nVsiltB%7VUeE7Ui^VYd!T^ z`PjNQw56A|rZyFRf`t*aMXWQ_@w5!5ivTJvL?RP2AZvgAIsr=|f_wRKn$hfrqv31s zmU66d+`0nw4)6S51kl;)@R-IC`y+s1e(FI^N@D|-v-mhNP|VM3NnIw-%Z`o81SuJ) zUQC+=j>$53cx0<`$EL@fF8~zm$=HW@CEokFRGSd%9}HV%z{eeWjN<#|IOIJx;>ihV zy?y;kNU$inzTKY}mRiZ~@l^8y(_!b#_R06+706YSLCbUbnD%$?`&3q~YB^s+VMSoD&JcH zSzghQRZ@LF3%XZEg-mU)H}s9_`RjgtOtQW@-_DCyVp-eJKdh~5`H>zyWTpNJBI)~`llqCdYiEWuK{9K zf6-9S9pQnNe?2>XFSDg zi_qPXH;nc0TqtxOSB5 zr9bYzw=KV|FAdpRlsv@-4b<>#s1#IbISN|Q{jSoep^Dl&{PTlZpyqNtFzGfIqz$_kQ4MaG&65^tr+$9paar4_#y2q#U*NP=wL=$+S`2E0)kA$cPk=!_8JWx zi-${lmb6B@*&)8GGq`4PX}{ zaDW*=%6qbRQin~%j<6F0yFh=#TI0a{2%Ei4wa13!*y`XpM?QoqGP8_2x zKb4=_O6Q>du&!A$GfFyM=Xwv|Ec6{i2r|wSma;W&f)8o^%x95N0slZ1e0JoxYMqhc zJN1sdLaT3A^fQ01y?$ciJqTTkhR4`aPY|2A0mE~jUR)Z)#7)m4Zf=Y7p|IGYX=n7n zH`P3iGbs-$VuK*6A=j8&n0!;P^pax{$Z=a~)*LD7ApU|1yS`%A0~MaOm`fvw`?BK! z@-hv5wPMEtVOP&CFO5i-Ox!Y@FtHLXE1-s#DQ;wuRtjx@Lvq(?-Gu-Jx~*3!38xt; z6D!BJhcwx^EnDvUR~7m>93!*cc$=7)*f`s(q(O5Z{Z2o#%p~9oY34aYlMG<8x4&J> zwcR{X9fe(WM_k5kyLp1i$q6`~es*faMW!|%@ZAiLpUvr~wo?Yd?isPeI2qD$TZ^4y*9urEWiQ%JpOgm#6%#H4JxV1#MX&WS<%uqUjnpE;|t*B5kCJ6eP#wgA4Q9 z$l{c-a1>1+{#+zg|1ehn6{87Um{sJ3$gA z^I~Tm#5S?$%83ljh9G&ju_M!`K~sK4sZ=K!wDJp3$-{r|Y8;lHsf5_fJS)AywbBm6 zu1j?y&UmI@Q6lEXQZWx+H;^N@u&-g?{Q2yHFbaX}y0Kl0ymmH-KL~NT_7Ot0W7i+v z=G0j5#fo&~XYpg^LEqE*VF98C-xh|I8bRkeQoVIp++^N~)cp=duN2u{Kb7pX39+~T zO`+fG{Ool@7Vl%+3+Ach@UZnO?!W&eCXR(>sZWdu8{-+jVIHIrCBEZYciZ`>$&w{m zWWuRu&%zask%l?p6wdf?DCJwlrO07BiY39cy*@DA3xyRTjT(#`m$AjdN)64160B;; zo%vTavQrwLnet?Q)>hBkWfS;_VJiy>mHqh7CWpnRsP_6b?)@7@j-R_v0{{I13wGJT ztMOGMw^)0EGEyQ}dMfHRSyl$YCAWop{Z_6>HDacuc>J{ zbJ6pDTYX^K5tw`U0CYd+A$hPrZidv~c=E-m*HRz$k;{X(jCtITZt;Kq!oPm(#dt4l%&C;0>PbrU zpfbQzNAB4s?f?AJ|NYti^G38m?bVM5ZHM3fzux@6{f1<~OVB2f6<@deZy)5pz4q^g zFiF588I;VF#%Ql?&IBnqYGHmkNUVC~jjzUDlxMVDgFdk)l7)&NOXD(seFTmx`#5j6 ze)M{D7sK)U2fF!4q$KEjcbq(jE?eC^EO2NvCLbwgNnT0;H?6^= zht-Jdd4mvW(GU9k07cIpASDudj)3uSvmjhfGO~EU?J0arq9t(Yf)l#45BeG$cc+D7 zgbp5EWSCcXT2BcP0xXE?q^>U^Lwhgm2Xv3@Fq1d2#*#iFQ>p5P1Bx`>tVzcf=LBDi z2eXg+PvrM5yPGM(0#;hD_gRsx_*GyN#CFmOjB^O(*(zSKQh`FB3v!qPQQP&xft>${ z@4j;+0QDNFcgdD6dndmoB^Rc{2Gd^#Wjz>HeHqq}p#t|Vuugma z9P{BJven9xx&8*g0_Ur;H~MA3Ro=h}K$?kdAV)n2PFO(`yX;WIFb{X0=rRBkO?znzaBBbCcS4ET(J8i7*HkcS|+!m25$kft^S6vAr@dNft$$y z9^p1F_d6>N)$8D7L69e2Sr1q#h*W`2JrB69$9#h;SIlVAsL3%`p0hTEiMPqX{*X-$ zy!Mz!Q}hBPORHd@i}O)ZU3304$hM^O-%f1s2tT*($IPW_elh(YHhgY>@C@ z3H5$J6$trFVj~ehasW)`*o5RD09HHO`R%P$S_;BVOJSEu(CBr4)h9qm`%?9M$oQ@@ z)>W|Kz-Ln^vI|G3s|GMd;ov7dMxZ1X?sD{7&g5YqVnO2z2gCTIQS9m zUnZ(l2dtRxX8w3k$Lv`B89I^{yHaiV9hCqk^_<)>PWKV0O(r{a=Hnc{xa;*LAN}QX zZ_O}u4FSHS4pRRHHxsH82E13}+xdJ#Y>+rPICVYZt6%gMz4&}HyB&#qY%>2L5snh_ ze<+h1P`yG*=!7GjgdU~lyQR?4j`P3`HRuWc^6`r2i1(s;GXQ_siHtl}VE50F#k3{g zqex@6lpwPj3pU@Y@vJhi1-~%6?*`_dF^@L@a5nb>yO=`bYgn_kLVpfr(Ghj5e2ZI4y zd)9-@6z@Af0|=+96Cmu8TEt=YwgomHf2~y9B_bTeA#K1W?#!3Xwoa~aAqO( z13dcr)au*_+IOq337VOn#voGvugwtq{`+ztIICslF45^shiI5E&a)p*eLbOVP84V^ zBF5Bb$-!T;K)ASGaWIttpVBf$A`fRM^-i1RR4Fw*Ga**`^*0=k?so~dY4oiy{YoAt zzy<)5gM0}mxY_r@81Q~nEM9-0m=?@O@389SX8@U;{*j`02ako?y{)c9%{OwVAx6lZ%lcxwEh zmx9>%?wFt2KJv1cuT}MC>Cdh4QCxB}oqhMCihLHarK_=+4>|6X?sttG@1m@x!jzR9 zQZ|$4=cDzIYwDf-F|yz@Z4s|1_uUWF&+DfZz22NP;cdJ&&;vzM7E~10P;=QdVO`&| zUu2)XFPfGNucI+cecuFntd^iZA9d}XYzJ$n#N$`7k!GgkJr002s^2?cQs8f{vPXMR zJLu4g*nsi1gTupU_Q0yI2T4~_R?Z~38T$Fm^Yi@UtPN0P<%YBeQXPPo=3-`&qZh;+ z1szs?@rG=XI@3o~3KIdSZ`8h#4Ze_NSlk3Yr2)VGxH8nrklFP!t`Gy9F~B}#SFVV* zmg4rT+@Jk6T!3ir;|V`l#{P+0ExVSGVxt_pD*$+?_WrbBhv(894V#4XAXGFjg zd=0!xSymGM>J-86wMGo%6^wG%n^P{W@8~)P?+$zqyiw26?DxkDl_5kRwW^GMo{Iad zgf~5p3|4c<2#axX5g5zd00XL6JhgCPqdWxq<@9Ru4f^o7JW?;IMVif`&3*|B34Krb z8yXlVVP@PpzftLuzO63?v)JMOvCtd#C5^N1V^GO;ldnD@%NB^%z^%@FM1Mn zz%4~OMN))L<%=lX$lgh`rB&1i6)+-7nyyw!#~cz}Bu-N0{6ebT2W)qF^ZwYR8pk2R zM?Ljy>luW7b|QmtJES`En%xRameYe&#^Pb=JDt z&^}!S31H$gq*PopY4Fjs2XUde9CtlvVUYLQ#amHvP3BkbrWbk7dLXgGd(((U?YuvT z>>Mctg*}N5hwYyY(~V&(DtCUv&m;c1Xn-nd-Ljwf@StXQ%r&5E!g^Ej6v>CGeQNak zvU7Dqa3S)^V`jf_Nw;f@-{+w8CY;=5T)e=#v-=YhiP?^;LWBN#+y?D>tL*SpVofxk z22Ss4(Kuz_bph-Uv8cB#Z;g2GnY_ehOWHTdW83s7F|Yzry=;zXR$J@@_XP#C?Q$hA`{aeezhswqsPvokB0G40KJNoy8oU)`S{%yi?1PWe!^?vY$vyXT;6fW0Lay*?he^rrBi4 zHOV_=pdOvU(R?V8@oSq>EQLLIyZ2RMp8#L$%@*gruNX2|DqS(i^vPm}B~#pfCp2k9 z6dT{Zomo`-zIVAXb2g>nS0+|MR;9qLf=15aPob9`WQQuH#wv+XWng@S#l>R>+}z#&XxkvG8ayA*>*4 z9qxCg?H>@IHbo9Rf0d!#AbTtDP}$DBdO5=&Wi29gKQQ-7HXIAZq}0yyawW8wPM2O) zn(1vyUO8F)ni!pk)?VPfZjMJ=I`P_5B_yoX1`$P5E<~!>G~{qR4y|{p4mw2~a$Fe& z^dfGRNR5Nwr|nF)pg7D=;1FeThq9e{Nj!Ljt4H%ysJ}Asfz&Vi2u^$eLB*4 z{VR9e?+)bzESFBz1TOwU;g$E_N|am&o(_Kit;Inq;VN1bv#D6@-?4?2U&GzKEcFyx zlr;BSVd2gC=-!dfHf@TLOl6`A7%XK})NB_q{tV6h&~C7=zP|8k@oJJ-9+BTJxZY%@ z2r00KX>_z;A}jMiG=v!-)4;GPZTiaezc7yFg^B99&m5|-?9xVJ&(2n6{_ zSg5ZGPADrUZHvjPCzJndOx{tc`Q@1*paoyDa@KH!o_!`e2L9qqJj!B03q7I}4gwBf zN8FCwSU^kWm}3`OQRU=>#!-w3w$1y4vz0yLw|s*+)l@mIKa}J`;EcZ=iChhRJaCSh z&k*r;D>9*G@BPZQO!8kiVfA|5-W6zLSAQ0~sR)7DL;pe~qZT(a$&b}W=a`gM4AiiC z>DCv={?Pdi-NdaA%^FvR#Jep@ed>kU1teRBW&9?vxc844peYHKok;`Q zg1*4)Re$g6iOM$>HO`Jf3p$Ed43(+%h336^P7v>RDGaNO1i2>E-+}B(W7anL@NfXr zLr}pQEm_A5!&7tA^j(TYLPXyQpy*p+0dXB~E1Lqd$oGjV3y_zK2&k@ z6R}Mt6=h_ks*U1Jg&N58;A{U|sL%LkqsaTw_u809zOa&;Ek>4{c>$pYyC^uMlYfpU zQM-BdL-C#BYqh*U)l~N5-mvTWRT0^QHu{O={#yb077p5-BSxFmN0upgl@`G2lvyz~ zoJ{-q+JxdZQwAn+JaYW)u~KGD`U)!d&QEEo(wqJdyy-!>EqVw^Q|4|Wc~ws8xiqE% zqe2#UJKxK1n zKGh%ApH=;Ob-%5(-em@wN^RcyPp3xeqT)xg%CHJ54J*uApQARhzsTqH#M(c|?9Wvg zQQtFYsIVDlI`8+WzQv6?-=ZqlElnpN6#19G%Ej#ELL}ZhTA;{%-r2lvM%?Wwp z(98dcf6jIyFX-@>AF%-_-;Ew}@qJ1X?3_5_a?ItSkB==;X|Ikf+3jT;1frfhivdbl zKTUp|__GB2FABZKKgU@+x|JhXi{nx`$MS^4XUPekvZzwmqp5y=5Nk*vSeD|QAmoxl z^Dm(BsW|Dx;7cNRts_a-f&xx_I<%zzn$!8?K8qHiFt{C-oA1jKpSzh6KdnXQ$xtR@ zxljRrFIEx!yCPXu;;=N?motwYfj&`tphB^kSOjpK<;^B##nC?F#aMH}WM>lS!MJ%t z%G;>Lhu2cv;)(*_T2$j}jCS}_CX%m)mV*=FO!M8Z2;>;-VUdQkAcrW4&TX8E7n`tD zY`kq6M7`;tvdXEq00qvqYdiP|-yOl7T0+yXG^8Q#He}vb;A4OWz8tVSIBgOi63<9> zs-J?#ta{HCw_-Qmgq$@!8HY9{1u}2@$_Rw9W&4+S(o;3wZHrneP^yN=Q~7|cF7wbZ z*+X?>r?mlms+-XM4!~XrcN<6!I;brDh;fp}Ci?tTON%njpWrpk1Vq5WUYZ7QL{n(V z;??>BRG*`(gcz13v0wRFSBA*F-&w3I2fn!hI1i>y#Y ztbgU-)-oP?z?Mk~q#lEi4a*=rI*yzcQ*2S5yE?D?)bI;N|JdNAmw z;(9gf&yCgAd6bp#=jQD;LHz>z;wYvIBOx6AfAqdUEkz67rp!xG!`h*8Pv}F*7|{aM zlfRTp%;v?{H%tjS^UW&DcnKOLUyz?O1-;9MTJzy_hFs;FykrQoq1*k~Xi+*PNU8PV z#UtU=Q>z$VX{Ha;MK3dc+3VlHtP2@R*}C3v4=R0^ID2MWZ*+9-MKx$#eyP@~thygZ zMb$v(Ct0pHJT<_hMQ1iRk}p0e&IegLEE%FX_uf%AeyHo$bz>NAQx_N)IC({-hf^YI zPJFNX8A&Uy#O~X7ocTqH-v_*h&X97_zzRnbFc$DEqEoNj^%^TS5VQ*;bBL64;EbxS zw+q;$?lP3-Wv3ly&)A93AMu2Hc7jsObEdT0gp)}(g6@~NBck=#xeMIly`B!I*Sg&W z@C-ITbL`)lT^p41>7Lrlx2LL#*a)gt>f+Cyj7 zBDFTEW_oJ9i#n=+mU6I^xBh85rsPnBg5BHMVufh*ghYIsbwo zzU!T9pZ;9Sl%J*a`wiK)x<$Uktc2iQewsTVmo2yAM&hLK!eIk7886LO6iZ?E-&?>b z;n^`h%e>{s9Or{8g1O%$NWZ{Cx=VLn%CsaMx@vDTpWPt=OwYT}ozVBVO?hSYtC$zR zf~>SLdgMkyHNTad---G&`Cu5i9uSGht3&ahT`mP8kpfYu)}7C)5}eVp;kuvRwpA)K zc+e9=y3ue*oVC1*oCAI_S=fqQc@u*8_YP2TCQwbD@o@?@wva0yr*qh&G zCw=XNtNNSrlV0pA5^xg*-y?X@fUz~=(RR~Jg7S*|N1<-^bHQCX)D>mO*cj_qxbIv| z7R+cCWea!pdOtF9y=us9C>LwYyg8WE62vG29FeG7kh)lam^z)B|J@A!a-~EK`ovUP z=qj8lNhxQK>;qk$t{Tr+#3+;_u5#DpeIe!!tBKb!PM4e~#3u^`Y!XNzz#xF^ZQFi* zQjWZ-uD;Mg4b1<;eA#F47Y9Cmss^?NdHe!NkoMSS*>x=fBPT}1CuR<@;r=%!4r2KM z;{QlibhLkk9B1gAx%` z?G+^s*Nx|^tZw-TSaQ^5Km8GzS7L^UBFu=u1=<{#5C}EuT@m=N3t$dfdhCqnI<1BS zM083;CMLWL4D2&U%%Pt;0}qr?sQlpMMWADK?E$&f*{RXQ>768Y=zS|3lC+fFnl^Uu z#ng0ANVW~(Yg*`MzLXfe$v1KiAR^LG11M z=Nw#HvcfqYdRL`f2y9SAoNZcObFMiR^IhG8_aPsAS}~e4OO^WPD;r|P3j~r>3ulvQ z$g=+ndWJ2PSHA=OY(*FxyB|J%vCd_mtGGxPCdd>4RDJcQkoYL5$-+I?CEd)L+6wJW zg}!p!o&uk3tytSX_7|d$G~5{RaNnfA`kAwqxInN!D8#j0i%%JjCRju z8U;FFj}qwNv)YSK55LlE{es&XEPS!bx|%4IJLj(h)oM2zi(h)>WVE|S_s8GDUZaN^ zM=sb=C=g)JDlUDthKCkheJw6y(JqE9qQuJ`vd^`wjgBRFr zqV^CKrHPcfW_1YExUvU%!e~G*)JP|LFeUlM3_eB3N0B!$)Yo; zRN~Fp=|m{%CP&>D++0MjTp_GU?br2d(YA`cdgc1G_MSy^mKm^|GIl>jMW&wMPA}n%7rt$;^loS>KK`Nqoi(5NeNOu^hV#lXgkW4v| z1(og$^)g(wP&vT{31# z?^=HWhnLA@wrwx=vra{sZB=z|bd18a+R0=*gXjK+wfY3bW38O=E+7IT=7t)Yf8c$N z9ltMmnMN=@-Z8sa3M%HmD#k|;0gb05tV8eT$BGEel2_up zY+>|TAF67aKa_znImc(9?z^~AT7Gc)o<@PQv3q$eId6@{&11+)0XjrveyvVQUDPZn=!7iQq^=mj3dCgX#3J^2P#%Zx}KQ1d28*FVI)8Mzf zE8xK_Xi2_Uy4m1ktz+OZpxM%l{n7BVn8aK0^^W6bM^ML@tE1fBJ^x)iky zR-!ncl-2wp-H$^KN`rpaJ;v%@vm$wb)@)*O&rrTNbi2^K{>2^fzEWOK>9;!F}vsuAy< z3&T0$%^U8pW-$5kSaiVa+7yq4PQM=Bu=TcdN=x9&P>rnsC>1cbrWsC+6majuhjGjv z{y2`Oc+t}7ipcCyskBuY=H8ONnmjjoROtC_B3Z@Y;SAKWP~=VAY0fHgyiAB`r!SK~ z{(Usz|G1mqs7jE$1q|zMA)#CU957TajD*O z=@9|SDXiK%Yh-DX_4>{Z_anWi5om&FR)F4Qg3VTs}+Hrd_4 z)548&3p?MPfzuv2e-xv6NS6EUaa{^4Lzo3BiQGK_&kTN5XxJ8c=9IpJ2nS(!Ktv+P zRlD)`QF5U2Pi(6tTO}s5zC^*L!6_M|W5wj!>Z0kEU>f_Dk84U<^aBc$@dC2Ot#o9^ zQruOlkIU|&Bn+RPuKu@@tGbc-wyS!&{k~e?Vb@!7c4L0Y_oeNM-RrDp-J0)y8cA`9 z`vSUCis6;%BR-lZBcUXB!tvH;8~Wu<92b)cb`hWl^-*dUo^( zu`P)>*MCqdPZ44k} zpUmUlmF#u=hmhs&cVk^M@6AAV&j$ppm}*s-KqObwQdRp`L*goY>fM^n@q<1|$83zR zbU3S?tIrWA$6wrSyN%0C$JqW|MTVIrE8DAYVPtOLt-pjX8SPhW=^j>}>YJZn;Lw0z z*fDDv?oO)hEzFF(fe>>M}rUB~8uuL#D_}4Jr7z#eYOCW^vLc|X!Xc3_| z=&PvS_<%ka8zB`FJ1}t!9Rr|#H~o=Lc?tjOO9*gkA#PQH_PnWy9;Wro#<9sg!H5Q^ zDuqV&XaJMc!K3%cb!%zN9P%d*gI-LpI2dNXu~6Xz=~l*NNkwT^cn=G5gCxkmI8 zl&JMln5Xj0SPuSnqmll&q$hLtt7fg)Fd=QN*F)XtBa>c=(<+^JtW^HrOJqPMqf_fJ z)YRx9FoZ&ur>1;3#DUJ9#y>p(L~?wTS(_vvGdO@V<{7!`?zbE-oKxx|M|UP0)mNTn zasPrF5ihc!xf;-TY&))bWNmxzM1H8bX}pU@9FK26@<3tJ9#eoMZsdiZ?~-oY0o7$Y z8Ux$9j(yE9pLp!!zKakCVqaD+``1gMJna_mBiIyT!%EL)SBD3#1kAmg-CW3T8tUjc+W1I&R^XC&4A)#1wt%8v63AfU?`+!$9F7 z6^j_!&ymVgvC;f7YdR1q?oMgj)#yzb&yV+zXlB$wDM5G#6$^P z-vqdjw_ZGFaf4WsA@*Ml17<)!W4mgQ6-06BSBHdZFoj+VtADi z8oy0i&J*c{L=*wlsD2#$u~^}_17vwF+j)Ns4AIfq|Bkna-uF>cVfMkh$nb6y+)B04 z(kv#n?Jy51^^zs^XeOacmFKbaBH>s(n4#n_nT~r~YVPY9-Zhi9t3P#QMW*0P2XHwO zTQ5PQRhwKv|0J(H=P+YKN)<2Jh4mIDViim^haR_*H08w|LsBN@yhp0WDz67>UuTr@ zrbpARFAioYb-@kWY@2Y4rn=3U-T7`uo3|U44EC$Y#5O$p3n#F&j8*p&1okDMq9x$0 zTYDN&h(qQzf+Mczy7c(Ab;tWq>8QnZs3;uQ7V&6RXtF!7+}m2@^J3_^q*{N+sL$20 z@g`*By6nfhQU9MWU^`=#jTQfdz38wA#q*_7<@k2}wuewVK^qRj=TcbAEA}%>iGoMC zIq>AwCQ>!Ae6h)$>+v05rsrih@)Z8oeg=Q2EE)g#=O4(e>??7ND0S-I0IsH0L(v77 z^_f}9k`+3`vcId1daiU|l5_Z{)%~~R^>xQn|1OkA;*Z>BLcYky1Hd*d`GQ-CxQzz( zs!kCJ*G8@bPT^?6wEp5zVa4Nya;}&Q82!HL3S@4UjhJ5fJt^4nO9H){ z4CA<~oSmrUyz+ljvk)SK=eW!Vo1e^^cEp%zcdZQUn%-2VU=RuF=sk`RG*noKq10yB zLc+1K_=iQ~3WQW|G%VmG8-CS4hBa*7{XjydCX(p`vkAJ#Q4fMGMiW~NQ|Z{It3s;7ngUz zF;X}o-Bdft2MJo;mFj(UMtuboWUY}0rTI5LZOn&aUzZ5woA%UjLMBTI;ErG)*AT*2 z6>e&hlRR19`2$R0>^nXlJi!!5Fl^~pD>q-y&Tls|pklSnM+ePWyOuwen6)xVn zH<{JO&3dyao)sY{E3||hT>x7@+6>I!VJ!G7gL6KiJf3%GAd5@ZX$vfYRCr!sh(!Q) zo|a0%K#XQ9zZ2{Sj|K{d%)g&d&o<@%A?-46>n!*RiE9&I9V-f_GoALbY+vQEuIMfA zLSk&FZK`y>T2lSp{^;dgmO0vHp?Nk(O+Z}t5H0ctRk4%=GlL?KO(D`)?!YmR{+g-D zhtLSxJlSaWXtUtpguE+D`KIXBMRT7E@tV|DvmY2CDHut1y^Z%v|H#8KhTS#1 zHhnINiM&J~dg!a6eOI{bXbMIWCC9sR4KKZ_XGYy@i>A0>qiV*>>>(0^4im^0A-2BG zFkPSyCoekuzK#B@qym5Ed9Fr1wr!JQk1>RNXR+^Jx&v3lg)3Rm{MirvFKyzJ$)nAB zHy{53#{sG`UPZ_gSMu1VBZv9>-|I1x@JGC{@GGRVi`wZ(i}Raip9Hzf4yB`QcpoG7 ztYAxNA7j+z7I#+Mo9~42PmqM&`yZC}a-m%xkfL7hZ5^RNF2s;2lJ@&4VZt&n@{=X? zXk-uBE9KI)>MZjT$WFUy@qK#9#+cbFvGnLE@%KHa5F+1~)&O&xnkn8#%NSvym2``D zBCWi?#V<4swPHLz-5M8C@~n?1b2sSCXj>^W{G8n6R1R4KVG1wDn!w|u`H@dyq1Vrq z&U*AamIIcr=krkuQmV~5ugqe{mi{uYc<5s2WbtO#NmjLAoKsW_{*t20fiI)5qcC>j z>vpYt9s?I2G`S5OryqGt#n6xs#~0^YPr_|>xJ%qpcrEUeCAzu|G50M#^7Gr?8w+Lde8A(qB`D(j+POVe5_x%JS<|_OInVY1 zt4977r|;IA?AUMsmfQNZpq)OCd>hu$8JEgqMFdDi!-btOb}3OZTB;RdX2)>$w#Ok5 zE0O<;SEaqre5Ly#c9U2#xLt|G{4W=N~y89~3+iMiW5!pa1)R_~j__`-@hbQRG$vUH`A& zu@+xOp07wSG@uGazQ;cSNIbQ8~)IXto-B&##1i0Bk2MH$QPBhlG*oHUSk_Bmg8^UjFm;Y8g#T2MJ2f z+ALoHN4#eAUO^OOC!}F}4++~Mkan;UAkNQ?-4wgsQtm+_sW=A6=5yj*lJ1+-RYeCc z^$GyBJP)z;UYUc=t)_fEkQl;Ej|PLHSI6Wb?0Gkt*Ylf1NcE`rOxw{12L&8B7~rSo z0~|acNElPw2?A#R!0CkXk3{2-x|q;X+ksx%5Wu(_l2lhQ4eT=HE&qqc6>BwD3-RgF zVdEm)0znfKfN-e@kb__m_Wp2IV_l@ysAqSkDw4S6x0LVuSL+**#y4i_H-=L?W#cFJ z;9!z1lOND@DjW(<88{P~4CYc1`8KaSa!5H86}zN0**fT*wEZ*xsMl)hrYDcl{v1Hm zt~1&@!P6oa^XfiGW)dk17~4S-P^|-PMxIBWRD)Dby~|tig{Ft*9hP_q%0wTW`)N*! z&Kk2zoa^edeZU`6uX)Gt;BD!XRXz*Rv14CG{7J{SKBrsC8X^@x0ZTlY%0Sb>Y~GEi zyWi4cEQ}Q<&Y-yD0C&ssJugbY0%LrAjvA+0%1)O)4%AFWttXSdzejTjzF98+*@Su| zl1;^jeH#|sQSg|=MXIrp`L%|<^x^_+h#j>JsdF^Zw?GfBo{RD5TPV-Jq4fHl6M$i@ zCGgfP067Vc-CrRE%sno+szKc0`d>pQ`ag{A;C{w|kChY1vUslsAfFzi?;S*Yx9opvD;r0e z`46IM<(eS3@DRO@Q3L0lC6^CYz8saduT9=Q<7C-$&^RX5kCbRTh|2^`_KTM2>x+Kk z&|QaWJegdRn8f|(0SccZ+qGMd6S+@H^z8H>553f4XvX`7MCn35Z5fD#HhR3~b}lio z()45kxK&m$g9>OQUBmj2^lJI)j`1I_qLP;b?a&8W#+PvpO=lk*ST3uQsPLC_4NU4N zD;gv?ylTlD)|2ctKH=k0PV~n!CIi5pZrwy z^ttHY6_3?%5e)G$E&5-5on=^*-P^B~kdTs=kZvRuqy-d2hHj)A5fG3R7$pUyTL~ov zh7Jko9uNd63CRJZ28JB+UE}lYz5nmt@3Ft}iGzcC-D|D;y3XtTwQ&;J@zoND)a@Yw zX^tzxktm0iJ28K_#((xg@r@{L$jV2-Vf2T8cg@c5hYpzi6lg&qmgoq?fC z7UT9e670XyD}Z1tWi|EPGl%~}+GZG+8<9{`Bn4%I+iVoLf>d{G-7oi5s|2D*Dh#a{ z1=>4$J35P?F8)7|w)K_E!Kd6aO}=tO|Dv|<{}*bzn<*dPLz0PE^@yD@Gq3ktkA%9J zluBErx}*4RpZO1mUIOTLz`4clpP`YqI99T0x$40A(nVMW0vS)$z~G{ZFFhAb*moUU z&H1KrkX7&pstC_6>a+jNzfN=6R}p(K!?SGv5w8d$ z-Dg|b$0$7|rc$=gJ>A@<_JB&%VD>e#8Mk9iV%S%`Ur_p3@+6J%`d|MfXUkNISitFl z)#p%ZUc7y0yKvaz>^UZUFQ0->BHhMK1%4f)Y-si)mIcOW*TcabJQ6?Zr3;L;(`w`~ zP3a(azf>8dpepF2UZUXZUjqjvsNWsg`GHS67Gh0h1`CClx#q`0IX}(@sX8>L9KZ}v zKA36o5*J_%-WydHukkkuyh2S9*i8%oPyFj+7`3%BPypN+_Y=QjQscd_1O zu=OoqE^dy#D>oQjUo-lYPml11K1nY*pG}B90LvRPUj=}(Nv59|4`4CL_WBLF18)l1 z>~AoaSAP7C!xtHJf5OmO+ref^yk#$XGyr^VW_&XFku_6sM;WbRuD(Gh<-UIG0N~!O z39BGuK3aYhB1US5+h8KG2Y}P5ygZQiBg$jClkb$uSl4I#h7sW!s%LX?M zX0cZ5i$)YVci>$N1u}-d~p!R+cxfcD{^ynR1zqI8!&Cco_mNg=x?6(LF{2TwUyGq)&ySVd5pZ_d3;`DH2sKnr^HP)psg# z>QFh%Kt0h9pW)A?yW@{s!QWic&ow#xn8x&8J4x-?ooLvLvr#0Fjxx-$4c-n>^wabF zv)d}igSXuy6TahMm7k$6WHFiUJ`cPv#gg5Mqiv8W2+4S*1xs@z#O6iX+yK#`f5 zkeeuM*3xVAiBjOEDcwq-LNbtnDZej&gxFsQi4qK3*1TdD`jtyCXhe0DKI~uMUWXZv zcRFLMo;cMOrhBR5E#zFj@X9K5<3+j?E`1_hGbNGi_C(|w_kd`7{q}PnIT@bbkO)=3 z$5FqBHcs~gCK5}z3m=S0gz&oyDP?u{<&(8^39hB@Ui_GM@I6KdLK~9aLg;QS(804+%>1Zawg0>dAb-}xyDuL2$7(OcEz`Q-K)^fdBpOeo zHIhEH&mr+Bjw%E*6b2vH#lQD179(6Wq))zY;%sv#rfM`kx!6IoJLPV$C{9)uj9Jns zqW6st#tUjA!6?Z2M!2+cM2TLF&QMf^JKV%7>rKQ9$@{QV`wG>m#%-_!Jz?2yJ-72k z8T$%?@AgYc@iFU1MJfdOJ$%3dzlYnsYhRS@aGGyTAP3ozMfwhx=pT0qitXR7n9wpf zOia!LNBL+tm1SB*J#dBdA{0z7O~j3O-bj_;q`k2irIh`p#Y!0!)kWb)>d(l)U@dC#Fi8BJ0d8zx0P!+#17qU!*Tfdzw8ZyfQCY6H`_OtT-zk$M9zq zf3iu!!D~~rSWXS~v_&xS)7xR>r070xI*#HjvJr|BPbp1yf$Yr({TRi1y;b`7j%P?t zn93>VvlZ>*fKyf2^x6#J$nY&&Y|A+h)hVq%+K^_Y$`GqeB(B{cv)2o4SIv#cDzidI zl~~4faCr)iDQGNgjY?qpe<<4I>}7$q(sQHJCgDR#y@oS*e%SNTjTeYm*F}&CZ&opr zfNKnZw5D3B$lIRm(^;vp%UF)6oB>z)kZ2rl+4coF&u5F9j!4lbAXFP`Bl& zC5L!uo}?+#4rvi?==XnxgVr`)O79TCGMs&^c<)p0u>WmS`j_BMRp&{ZD`c!^8&mCg=`@FVHya zzg6s;;zV7@lF9152GUQSR@25ajO=8a=ZWY}2W6lL!W6>w$Q?jzpZ&*06Lb&OyV zZtoxBYFJ~m=1=^#LEU`N3H-8J|e4ii29* z*-qiCGVG^Y$ZsbEmgOz4svOKPYZe(}3DPiKN4fE5_Xbq$gE>yVYF@aSV=A~|`Pk(1 z?)cY-g=+7ub&}e$h9j+X<}**mpB-rOk89B!CXOqjKHGYhadOhz8pH#Cr?OE&^39XS zszW=h+_(a^k9u0;jwWpbgsVg?t(24eB+D>!p5wV$2PF*S(fM;>e;0uKSn9JI(?m~g zL{Q*0=HXJcu@$$|`VHy{T&ibriX=rX5|4Oi8q(i)n|(8rNaG)oxE0!0J?36S!^G`P z%dE&vTH}6QT>oA_`#XsLg20;9So((!NE}0&fI&yiNPVBipu#~8&_1G*X#!FphzDz& zk6PAWn$GD6hG;_MPtUjxgG{L|SdHT3TZiU|9Ra{t&aV`IIuL}fe=X-F1%(zjXD!NO z7V#}vF50mbsRIaAYYd_4e000_R$`9uGBK1l}Ud-^?pr}rs(;~ zXta!!%qW}WUmxqT!jG4k4k-S;h8)qoiym2-sR?EY8*kJz6{%qR4MuSC#QdDk4Lw>#e_)`D#=J!Lr*&0nz^aGA{Ys2^5S#vKwySubB&5;)#g)Af4!c zycs!r)u$}0ogVpFT7rSmo0BDt+j6(}FW4#rN!b66-tlggT5QwO4@Oz%*nvn9jd&9( z<#*DmJgLP*HM8$>t_fmFA?-t$k8n8(1xz&yA!@5|Ib|Dk_vQNSVIL*mai|rxNmFg9 z2pt!?NNvgS@-QGXUFqUCulXy6ix%jRN0tu9Tde5D&vSnV-;ie*X0>nV-WGAwKg{ZF z@w=4rMeSWKjL}UQq=<5o+`xu>Ztwn&u_olLrsl$2ecpr|^qe=)^4!!1tXXCI<0*HR zuTv*hzI^f)o`EHq3;8&pdPPJ$6z>cD$EhMZgx!#dyoft72hg5_gG5a1g zi%k5C600#HpMP+Q5VVyWP*#6GdNLe@W=2=9qzZvI{e#lv;;ZJe#G*dq_$Jde=c&62Z97zMAmlhWEGGGw3w!C!lIotlr7f)TmII5tc=!5aDw)pV^0~5DUudG){ zl2xN^zw^D`RBVm9Mfl<*M9FHOB9Ye9hPZHfG;t(t@;2ww_H049#@x?e9X5BXJn~iA zf9RUC#}xS8MFx~vEh*}oH(KVoH12o~?-b8V%JIA`ikj>#CD2$LafX^s&}5&BqtR2n zvieyF`AdnoXC#OoxkR%mNqa<5T`RK37Gp7t$&vPO+vu(4rr|F6JI?fc0RSoG^`H^) zlw%&a_Kk(OH6Eu;FZXU}=yO*GY2P?vFdR5m__!6N_O+C%N8$qh2}Z>3bJbe)D5mBX6}A za+Ys40Od{BoL&IIV&~xip&;ku^b9*?BJ6uUw!1x_YcWcJ+j*t_{M%xb!&<<6KT(wZ z3(mI<1$Xr9dn$csBFOo2I6+bp;f+Py?i-$=Oq}$9S}oz%;iXkrF?!m-?q6qmIMk*A zJ^kryHsD>=6cNFaBwFT9t|5YsO0NR5^ELJv&f7Xzv61JL9M{lx6$XxTlbAXn3b3d2E;7SsC|BTc!`qz+^D1W} zX$aroM8Eu6H$x_^td;dvtyRT!&fM=$zEh%N!+TZ*k}}v~QeV92>k5`wl@`v#fQW?W z(VY1v&(+qUf`{y+a5wYOa4Q{I&yv1?Wz5d&EhG)t;@T?ZEf;a(mx@5g+q1k7>!)-@ z+(9`A?%Sh%DYa(eN>j_-^#f?i^TRhn{hLXM9k$d2n`v4C+ZiJS-2!gw>f3BgepE>Um9fleP65Zn8o(B1ALgfvGAYQi{UCXX_Fa! zv%m0&el!N3MhXn&9B-N>nX2v6W$-2cX~i^6$_R2p-drWDX)W=}n;}6-ZrX6ALi)mO zG*QYEj?4MRKSI&ju-_k}sC@ODMPFhbdUa8%6%s}v%9mFOi~+ShjQiU9mz#P#$MZLJ0j)2||(Hq<+0K>^%-raxJf?uMGNdKa9- zs5Ec8s`0!(24^9u0~325ucUk|s8Ky`8qGf+TP3&Q3je6#75Gri zUFGCuC&9cL${5MK-nd=wyWLwYJ7(lnct78|bW&~a(4EO!B0Hhg%&-xuxZyd8DBFjt zKczHB&G_$sKiup&BfDfL_foFBcGJq;7{WG!T!a*hRpm`mXAesyl`4o(C$81;HPLAN z6_`q;PM?!5Wm0Jf2Y1EvgNksOr1Tw%d}m5j0IkjIH+U;G=K|pKQ>;!hpzw^M@`6A6 z{pP`n)J|BUJEP52_xD8PJ~#sFn4qaq@IpX5*fM*R=z}-jzs)sP2@40sCTm zv=5-G7!z}B^2v+Y|4mC6-L)f zDNk>TQF1s9?a)$!qf11Ss%gkm`p*%6-M7v21{X+97>$*58l*^5R-B3gCz& z7$oNhoimD@iZC{EBi!%2tS>r_6LIMEKWt!22xEJKeP4`|vD4f-k62L1lY_5T%g@3| z@0|*_u-ls%&ow4EP3SJ~+X*AiD6BqEu>QlA(zZ;^b4`D(T(fanvrf_dF>sQ`+9jgs41arSh% zac4hJKe6XQ3SL#)Y;XZ+qg9JYg-g6Ov_cH&KXcXtg*`D_s==sCuLsrC+-KBf@0-Rv0=+(nPbPVYWvi{i0s$r1i}r> zg!FDv4KWGlNmO6b4&sx$cVk$iML#j0vUo-gqYxQ#=4jSS1n*N0NZ~u8B=1~5TU;17 zOZ#KeoR2<1t zdIzVmma0BG_vGMGp!OU`cu&po`&iliz)+ZBX6L7YOiHyf2TO8C0hrZbrc?x0L+qyg z50Ep#Bz;oHb`BR8dUY9Ep)le9^aO}FKiMpjE}vk!bc7vPtgDeCRNeK^SohnpOjzyq zLMA|_5$$DzG58SRI|IJ}q5+G7L3#acyBPL!9N6!IZuau5w6!)h9Elm~^xcK2$CdNMOfRw;ibO8D1Uxwz$zLu2(^{(~X%s>>!j1LDX z?1!_68%v-yq1?zd(3cSh}0^<080N-F$Ufx zF&w8yaw-1m1j^e`1B!rDK*jFGGp{-5gg6HffDUom6HAODwC-QJLY}WXX)C_B&16gkbrAIr)@{ zmsXpUkHb8ZO9z6ZLIcf!Hei6N0J89Kw}SV)0Fi;6(+3 zq<5NEN+uhPf!bV}PcTBIWM6_=F_1Rqz63#W)eRzkHJ_$@rlN6bP?c4j$p)eZzl+AF z&8a66Kc2cuQF*Um3W2+6vhZ!%xK$3aD1j|-!hnR z-{xKuO-;1_=+7F^9aN9YMGXN66;oYhz3LN(*pn4<9@D#l5XK}7nHe|9T21g31?Nk3 z*Y0hXar>F2If(CQ;>fr1Il4zJu!C>j`WH)~Zy|%GT@yv{w8j_z5li8wc6K(ryvcs0 z?MdVE2*pXJqP0c_M6(HjwVFn+sD{J5jIao^1l zKBp#i4!%XfIp6E1{VfJSbGM zzhy7?xj{ncqC?cmvJs8O3Jz-8(#+L6o~s@y{v$RLSpFYdIi z-Y<8|G>mfmTIoYlEq$|Fx)wT^DEwJXIqhwvK2Dx`6cXzXaFzBQw`ac5$h0dyX^c~~ z@Bj-ZWjjLt>ANjOGtiV`Lkv+s9v&nr$p?0{mJh}x3tn{jF)YOtO|qQ--RcE#dUXQ< z>8OilO!a925;^Lr>sQ;Z&}TbO!Uet9PX*{c&6m=As-z$EW)SD4wc3c(`%ZRqVb(P0 zL#*z6`0Sl~#fQhH2i4%aMmtv1buw^&QVQ=-R=b96Ue+{#q|u{02oHetI`2ygM7jfG z3q-uqkK5VEUa&gu8tus9oxaRBD-yl!oEbMEb=xmLcm6DlqM9Gl_Nvr`xwQp7_N_a_ zW0}C$3^@E0zmbNTJ5$Ko1l?udrv^K%0GiMd8iI-A+FnaW?cL2C1J~nW%M|+yex~et zBgU@sn;$^p$IZapC#HS*X;tgmX77W}^J2O!$XU&mDrLRX*V6nL6NcSV_klQiAr0?C?YCv)Bq%k^ci)ptXxC(;j3F~ z$j+aJL@0q&#j78&7k3T??Il*29z0Jc3viHI}@#@q?iAmn3Npe~;LC^Ps z6!%Z$r(mlurCA0GfC^9ml~#TqA3x=Oxu0Zr$o6gD;P6=&E-VV28^V@ct2xIy5MyD> z*K7REvHYo{)*q|~@gcT2RyG!psiOC2-L*_cya`i0j9rL;iSI1k|LcwoQ;*2Pdr`GC z!Ns;k$Q*0jnUTgaBipidhiqoBKOUB_VNXaV;d-xsYyp;Rdheo=1&oqt%E?KR8D=1C z@sn-|hWlj`+XVh-f8LM{m@)-0B%*uG*A@asF;j3M zm`D{slu4C;TSW%bKz?WSG-p$RbeGX%791KRJgIM4mpA-T9Kt~aO9$(%f8Tx*_v+wE zJ!@MrkL>P0cDJ?)<~YP?Mbe`o<^RxUE{8qbv5$ThVHfyU0oMk!dZ2}_6_XzIKO=~= zqBtfxk6X6>bQT75+CcvsN63#4sm6g@{x_H0|4+M_HjU3N5xfXMF5Vc4)im<{VgFT8 z9lDXfR!2}XH+liYW!+%TPAtDN?fKxQ*EJ*t${Lv0?QBPgFE6ns{PkmED8eMNlf&#C z9xn;|0p5Yl7TbNR>U5^EtbH|c{1x{vy%Ng_Zbiz>P5*)=7~g5!t#y*O?1W($V6htm zE?5CT&Up|GYmt&;kB%s?z}GJ-8n~l zuQEKkc)x`ElZFShhr}t%{ksVRRQQxPV%snXZAw$fCz4n4-gBM?dl*Q3`zC_=Ac*GM zCqs~)oSWDRQoc(bV6tbeitiVY(qMjm{N-nXkEj5v(;d*|26bn=~;G`})Ft=szA>S|k2Bv`p2CfXW%G3rqrB OQ+cfZs7%2k=)V9)7vvED literal 0 HcmV?d00001 diff --git a/padiff/cinn_diff/logs.py b/padiff/cinn_diff/logs.py new file mode 100644 index 0000000..e69de29 diff --git a/padiff/cinn_diff/read_file.py b/padiff/cinn_diff/read_file.py new file mode 100644 index 0000000..143d55a --- /dev/null +++ b/padiff/cinn_diff/read_file.py @@ -0,0 +1,253 @@ +import os + +from graph import Graph, Node, Cluster, Group, Pass, construct_graph_by_dot + +import collections + +INPUTS_NAME = "cluster_inputs.txt" +OUTPUTS_NAME = "cluster_outputs.txt" +OPS_NAME = "cluster_ops.txt" +GRAPH_NAME = "subgraph.txt" +PADDLE2CINN_VARMAP = "paddle2cinn_varmap.txt" +GRAPH_COMPLATION_KEY = "graph_compilation_key.txt" +VARMAPS_KEY_NAME = "graph_compilation_key" + + +def read_varmaps(varmaps_file): + graph2varmaps = {} + varmaps_file = os.path.join(varmaps_file, PADDLE2CINN_VARMAP) + with open(varmaps_file) as f: + lines = f.readlines() + cur_graph_key = None + for line in lines: + var_map = line.strip("\n").split(":") + if var_map[0] == VARMAPS_KEY_NAME: + cur_graph_key = var_map[1] + graph2varmaps[cur_graph_key] = {} + continue + if not cur_graph_key: + return + graph2varmaps[cur_graph_key][var_map[0]] = var_map[1] + + return graph2varmaps + + +def read_tensors(tensors_path): + tensors_map = {} + assert os.path.isdir(tensors_path) + for file in os.listdir(tensors_path): + # ['matmul_v2_grad', 'input', 'scale_0.tmp_0'] + var_info = file.split("-") + # bind var_name to var tensor file + if "share_buffer" in file: + continue + tensors_map[var_info[-1]] = os.path.join(tensors_path, file) + return tensors_map + + +def read_graph(graph_file, idx): + assert os.path.isfile(graph_file) + nodes = {} + edges = {} + def record_nodes_and_edges(line, type, nodes, edges): + if type == "nodes": + node = line.split(" : ") + node_id, node_desc = node[0], node[1] + name, node_type = node_desc[1:-1].split(", ") + if name in ["feed", "fetch"]: + return + nodes[node_id] = Node(name, node_type, node_id) + elif type == "edges": + edge = line.split(" -> ") + cur, next = edge[0], edge[1] + edges[cur] = next + else : + raise ValueError(type + "not support") + type = "nodes" + with open(graph_file) as f: + lines = f.readlines() + for line in lines: + line = line.strip("\n") + if line.startswith("nodes:"): # start to record node + type = "nodes" + continue + if line.startswith("edges:"): # start to record edge + type = "edges" + continue + record_nodes_and_edges(line, type, nodes, edges) + + def construct_graph(nodes, edges): + for k, v in edges.items(): + if k not in nodes or v not in nodes: + continue + nodes[k].add_output(nodes[v]) + nodes[v].add_input(nodes[k]) + graph = Graph(nodes.values(), idx) + return graph + graph = construct_graph(nodes, edges) + return graph + + +def read_strings(string_file): + assert os.path.isfile(string_file) + with open(string_file) as f: + line = f.readline() + line = line[1:-1] + rets = line.split(", ")[:-1] + return rets + +def read_string(string_file): + assert os.path.isfile(string_file) + with open(string_file) as f: + line = f.readline() + rets = line + return rets + +def read_cluster(path, idx): + assert os.path.isdir(path), f"{path} must be dir" + inputs = read_strings(os.path.join(path, INPUTS_NAME)) + outputs = read_strings(os.path.join(path, OUTPUTS_NAME)) + ops = read_strings(os.path.join(path, OPS_NAME)) + graph = read_graph(os.path.join(path, GRAPH_NAME), idx) + graph_key = read_string(os.path.join(path, GRAPH_COMPLATION_KEY)) + return Cluster(idx, graph, ops, inputs, outputs, graph_key) + + +def read_cinn_pass(path): + all_groups = {} + def read_graphviz_dot(path): + passes = os.listdir(path) + idx = path.split("_")[-1] + #print("group idx: " + str(idx)) + all_passes = {} + for pass_path in passes: + #print("pass_path: " + pass_path) + #print(pass_path.split("_")) + pass_idx = int(pass_path.split("_")[1]) + #print("pass_idx: " + str(pass_idx)) + if pass_idx not in all_passes: + all_passes[pass_idx] = Pass(pass_idx) + pass_name = pass_path.split("_")[2] + all_passes[pass_idx].set_pass_name(pass_name) + type = pass_path.split("_")[3] #after.txt + record_path = os.path.join(path, pass_path) + if type == "after.txt": + all_passes[pass_idx].set_after_txt(record_path) + elif type == "before.txt": + all_passes[pass_idx].set_before_txt(record_path) + elif type == "after.dot": + all_passes[pass_idx].set_after_dot(record_path) + elif type == "before.dot": + all_passes[pass_idx].set_before_dot(record_path) + else: + raise ValueError(type + "not support") + max_pass_id = max(all_passes.keys()) + #print("lass_pass_id: " + str(max_pass_id)) + group_cc = Group(idx, all_passes, max_pass_id) + all_groups[idx] = group_cc + + file_names = os.listdir(path) + for file_name in file_names: + read_graphviz_dot(os.path.join(path, file_name)) + + return all_groups + + +def read_cinn_graph(path): + all_cinn_graphs = {} + + def read_cinn_graph_dot(path, idx): + graph_path = os.listdir(path)[0] + file_path = os.path.join(path, graph_path) + nodes, _ = construct_graph_by_dot(file_path, sep="\n") + graph = Graph(nodes=nodes, name=str("cinn_graph_" + idx)) + return graph + + file_names = os.listdir(path) + for file_name in file_names: + idx = file_name.split("_")[-1] + graph = read_cinn_graph_dot(os.path.join(path, file_name), idx) + all_cinn_graphs[idx] = graph + return all_cinn_graphs + + +def set_node_cinn_name(all_clusters): + for cluster in all_clusters: + nodes = cluster.graph.nodes + for node in nodes: + if node.is_var(): + node.cinn_name = cluster.varmaps.get(node.name, '') + + +def set_cluster_varmaps(clusters, varmaps): + for cluster in clusters: + tmp_varmaps = varmaps.get(cluster.graph_key, "") + if not tmp_varmaps: + raise KeyError(f"can't find graph key {cluster.graph_key} in graph2varmaps") + cluster.set_varmaps(tmp_varmaps) + +def set_clusters_group(clusters, groups, cinn_graphs): + for cluster in clusters: + inputs = cluster.inputs + outputs = cluster.outputs + for idx, graph in cinn_graphs.items(): + graph_inputs = graph.graph_inputs() + graph_outputs = graph.graph_outputs() + if not graph_inputs and not graph_outputs: + raise ValueError(f"{graph} does not have inputs or outputs") + # print(graph_inputs) + # print(inputs) + if not set(inputs).difference(graph_inputs) and not set(outputs).difference(graph_outputs): + print(f"group_{idx} belongs to Cluster_{cluster.idx}") + cluster.cinn_group = groups[idx] + + + + + +def read_all(root_path="", type = "cinn"): + + assert root_path, f"{root_path} can't be None" + all_clusters = [] + #paddle2cinn_varmaps + graph2varmaps = {} + all_vars_paths = {} + all_cinn_groups = {} + all_cinn_graphs = {} + #exsist some bug + cinn2paddle_varmaps = {} + + allinone = collections.namedtuple('allinone', ['all_clusters', 'all_varmaps', 'all_vars_paths', 'all_cinn_groups', "cinn2paddle"]) + + all_paths = os.listdir(root_path) + for path in all_paths: + file_path = os.path.join(root_path, path) + assert os.path.isfile(file_path) or os.path.isdir(file_path), f'{file_path} must be path or dir' + if type == "cinn" and path.startswith("cluster"): + idx = path.split("_")[-1] + all_clusters.append(read_cluster(file_path, idx)) + + if type == "cinn" and path == "paddle2cinn_varmap": + graph2varmaps.update(read_varmaps(file_path)) + + if path == "saved_tensors": + all_vars_paths.update(read_tensors(file_path)) + + if type == "cinn" and path == "cinn_pass": + all_cinn_groups = read_cinn_pass(file_path) + + if type == "cinn" and path == "cinn_graph": + all_cinn_graphs = read_cinn_graph(file_path) + + set_cluster_varmaps(all_clusters, graph2varmaps) + + if type == "cinn": + set_node_cinn_name(all_clusters) + set_clusters_group(all_clusters, all_cinn_groups, all_cinn_graphs) + + + return allinone(all_clusters, graph2varmaps, all_vars_paths, all_cinn_groups, cinn2paddle_varmaps) + +if __name__ == "__main__": + read_all() + diff --git a/padiff/cinn_diff/requirments.txt b/padiff/cinn_diff/requirments.txt new file mode 100644 index 0000000..4a45c25 --- /dev/null +++ b/padiff/cinn_diff/requirments.txt @@ -0,0 +1,2 @@ +pygraphviz==1.11 +graphviz==0.20.1 \ No newline at end of file diff --git a/padiff/cinn_diff/run.py b/padiff/cinn_diff/run.py new file mode 100644 index 0000000..151ec35 --- /dev/null +++ b/padiff/cinn_diff/run.py @@ -0,0 +1,16 @@ +import os +from analyze import auto_diff +from env import Env + + +def run(run_script, base_env, cinn_env): + run_env = Env(run_script, base_env, cinn_env) + run_env.run_base_model() #可以提供选项选择不运行base model + run_env.run_cinn_model() + auto_diff(run_env.base_path, run_env.cinn_path, rtol=0, atol=0) + + +if __name__ == '__main__': + run_script = "/root/dev/PaddleNLP/model_zoo/bert/run_bert.sh" + run(run_script, None, None) + diff --git a/padiff/cinn_diff/utils.py b/padiff/cinn_diff/utils.py new file mode 100644 index 0000000..14aa6a0 --- /dev/null +++ b/padiff/cinn_diff/utils.py @@ -0,0 +1,63 @@ +from contextlib import contextmanager +from functools import wraps +import os, sys + + +@contextmanager +def suppress_stdout(): + with open(os.devnull, "w") as devnull: + old_stdout = sys.stdout + old_stderr = sys.stderr + old_stdin = sys.stdin + sys.stdout = devnull + sys.stderr = devnull + sys.stdin = devnull + try: + yield + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + sys.stdin = old_stdin + +def retry(max_times=1): + def retry_decorator(func): + @wraps(func) + def inner(*args, **kwargs): + retry_times = 0 + while retry_times <= max_times: + try: + ret = func(*args, **kwargs) + return ret + except Exception as e: + retry_times+=1 + return inner + return retry_decorator + + +# \033 [显示方式;字体色;背景色m ...... [\033[0m] +# 显示方式: 0（默认值）、1（高亮）、22（非粗体）、4（下划线）、24（非下划线）、 5（闪烁）、25（非闪烁）、7（反显）、27（非反显） +# 前景色: 30（黑色）、31（红色）、32（绿色）、 33（黄色）、34（蓝色）、35（洋 红）、36（青色）、37（白色） +# 背景色: 40（黑色）、41（红色）、42（绿色）、 43（黄色）、44（蓝色）、45（洋 红）、46（青色）、47（白色） + +class console: + + def __init__(self) -> None: + pass + + @classmethod + def red(self, str): + return + + @classmethod + def info(self, str): + return + + @classmethod + def error(self, str): + return + + @classmethod + def warning(self, str): + return + + \ No newline at end of file From 6fd7af56e5a91438fec0d97d9cff920e95d27a74 Mon Sep 17 00:00:00 2001 From: GGBond8488 <857631483@qq.com> Date: Mon, 15 Jan 2024 09:01:33 +0000 Subject: [PATCH 02/10] code format --- padiff/cinn_diff/README.md | 7 -- padiff/cinn_diff/__init__.py | 16 +++- padiff/cinn_diff/analyze.py | 51 +++++++---- padiff/cinn_diff/compare_utils.py | 79 ++++++++-------- padiff/cinn_diff/env.py | 84 +++++++++-------- padiff/cinn_diff/graph.py | 144 +++++++++++++++--------------- padiff/cinn_diff/logs.py | 13 +++ padiff/cinn_diff/read_file.py | 83 ++++++++++------- padiff/cinn_diff/requirments.txt | 2 +- padiff/cinn_diff/run.py | 20 ++++- padiff/cinn_diff/utils.py | 35 +++++--- 11 files changed, 313 insertions(+), 221 deletions(-) diff --git a/padiff/cinn_diff/README.md b/padiff/cinn_diff/README.md index 93edd40..09f4f1d 100644 --- a/padiff/cinn_diff/README.md +++ b/padiff/cinn_diff/README.md @@ -134,10 +134,3 @@ python run_pretrain.py \ 2. 【开发中】中间变量读取展示接口 更多功能正在研发中... - - - - - - - diff --git a/padiff/cinn_diff/__init__.py b/padiff/cinn_diff/__init__.py index 5ea3f0f..7ab1050 100644 --- a/padiff/cinn_diff/__init__.py +++ b/padiff/cinn_diff/__init__.py @@ -1,4 +1,18 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from .graph import * from .compare_utils import * from .utils import * -from .env import * \ No newline at end of file +from .env import * diff --git a/padiff/cinn_diff/analyze.py b/padiff/cinn_diff/analyze.py index 584d72d..34efdaa 100644 --- a/padiff/cinn_diff/analyze.py +++ b/padiff/cinn_diff/analyze.py @@ -1,4 +1,17 @@ -from logging import warning +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from read_file import read_all from compare_utils import Comparator @@ -7,7 +20,7 @@ def back_track_group(base, compare, cluster, cmp, graph, node): inputs = graph.inputs() all_inputs_equal = True paddle_output_name = "" - cur_cluster_cinn2paddle = { v : k for k, v in cluster.varmaps.items()} + cur_cluster_cinn2paddle = {v: k for k, v in cluster.varmaps.items()} for input in inputs: tmp = input paddle_name = cur_cluster_cinn2paddle.get(tmp.name, "") @@ -15,10 +28,10 @@ def back_track_group(base, compare, cluster, cmp, graph, node): print(f"can't find {node.name}'s paddle name") diff_ret = { "cluster": cluster.idx, - "group" : cluster.cinn_group, - "output" : paddle_output_name, - "output_cinn_var" : node.name, - "subgraph_id": node.graph_id if node else None + "group": cluster.cinn_group, + "output": paddle_output_name, + "output_cinn_var": node.name, + "subgraph_id": node.graph_id if node else None, } return diff_ret paddle_output_name = paddle_name @@ -35,10 +48,10 @@ def back_track_group(base, compare, cluster, cmp, graph, node): if all_inputs_equal: diff_ret = { "cluster": cluster.idx, - "group" : cluster.cinn_group, - "output" : paddle_output_name, - "output_cinn_var" : node.name, - "subgraph_id": node.graph_id if node else None + "group": cluster.cinn_group, + "output": paddle_output_name, + "output_cinn_var": node.name, + "subgraph_id": node.graph_id if node else None, } return diff_ret @@ -50,7 +63,7 @@ def auto_diff(base_path, compare_path, rtol=1e-6, atol=1e-6): # step1: 确认cluster的输入输出是否对齐 for cluster in compare.all_clusters: - #print(cluster.idx) + # print(cluster.idx) input_equals_flag = True output_equals_flag = True for input in cluster.inputs: @@ -61,7 +74,7 @@ def auto_diff(base_path, compare_path, rtol=1e-6, atol=1e-6): input_equals_flag = False cmp.record_input_diff(cluster.idx, input) continue - + if input_equals_flag: # step2: 找到cluster内部对不齐的点 for output in cluster.outputs: @@ -70,7 +83,7 @@ def auto_diff(base_path, compare_path, rtol=1e-6, atol=1e-6): ret = cmp.allclose(base_var_path, compare_var_path) if not ret: output_equals_flag = False - # step3: 找到对不齐变量对应的group + # step3: 找到对不齐变量对应的group output_cinn_var = cluster.varmaps.get(output, "") if not output_cinn_var: print("can't find var " + output + " corresponding cinn var name") @@ -83,25 +96,25 @@ def auto_diff(base_path, compare_path, rtol=1e-6, atol=1e-6): if node and not graph.is_input(node): # 找到对不齐的第一个输出，开始回溯 diff_ret = back_track_group(base, compare, cluster, cmp, graph, node) - if diff_ret: # 输入能对齐，输出无法对齐 + if diff_ret: # 输入能对齐，输出无法对齐 diff_ret["output"] = output cmp.record_group_output_diff(diff_ret) find_diff_group_flag = True break - + if not find_diff_group_flag: cmp.record_output_diff(cluster.idx, output, cluster.varmaps.get(output, "")) print("can't find diff group in cluster_" + cluster.idx + " but diff exsits") - + if output_equals_flag: print("cluster_" + cluster.idx + " has no diff") - + for diff in cmp.record: print(diff) return cmp.record - + if __name__ == "__main__": base_path = "/root/dev/PaddleClas/base" compare_path = "/root/dev/PaddleClas/cinn" - auto_diff(base_path, compare_path, atol=0, rtol=0) \ No newline at end of file + auto_diff(base_path, compare_path, atol=0, rtol=0) diff --git a/padiff/cinn_diff/compare_utils.py b/padiff/cinn_diff/compare_utils.py index 80b2934..1bce4ea 100644 --- a/padiff/cinn_diff/compare_utils.py +++ b/padiff/cinn_diff/compare_utils.py @@ -1,11 +1,22 @@ -from typing_extensions import Self +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import paddle import numpy as np -from colorama import Fore,Back,Style class Comparator: - def __init__(self, rtol=0, atol=0) -> None: self.cluster_ret = {} self.graph_ret = {} @@ -22,56 +33,50 @@ def allclose(self, base_path, compare_path): compare_var = self.load_var(compare_path) ret = np.allclose(base_var, compare_var, rtol=self.rtol, atol=self.atol) return ret - + def assert_allclose(self, base_path, compare_path): base_var = self.load_var(base_path) compare_var = self.load_var(compare_path) ret = np.testing.assert_allclose(base_var, compare_var, rtol=self.rtol, atol=self.atol) return ret - + def record_diff(self, diff, type): diff = { "event": diff, - "type" : type, + "type": type, } self.record.append(diff) - + def record_input_diff(self, cluster_idx, input): - self.record_diff( - { - "cluster_idx": cluster_idx, - "cluster_input_diff_name": input - }, - "cluster_input_diff" - ) - + self.record_diff({"cluster_idx": cluster_idx, "cluster_input_diff_name": input}, "cluster_input_diff") + def record_output_diff(self, cluster_idx, output, output_cinn_name): self.record_diff( - { - "cluster_idx": cluster_idx, - "cluster_output_diff_paddle_name": output, - "cluster_output_diff_cinn_name" : output_cinn_name - }, - "cluster_output_diff" - ) - + { + "cluster_idx": cluster_idx, + "cluster_output_diff_paddle_name": output, + "cluster_output_diff_cinn_name": output_cinn_name, + }, + "cluster_output_diff", + ) + def record_group_output_diff(self, diff_ret): self.record_diff( - { - "cluster_idx":diff_ret["cluster"], - "cluster_output_diff_paddle_name" : diff_ret["output"], - "group_idx": diff_ret["group"].group_id, - "group_output_diff_cinn_name": diff_ret["output_cinn_var"], - "group_graphviz_path": diff_ret["group"].dot_path, - "group_test_py_code_path": diff_ret["group"].txt_path, - "group_diff_subgraph_id": diff_ret["subgraph_id"], - }, - "group_output_diff" - ) - - + { + "cluster_idx": diff_ret["cluster"], + "cluster_output_diff_paddle_name": diff_ret["output"], + "group_idx": diff_ret["group"].group_id, + "group_output_diff_cinn_name": diff_ret["output_cinn_var"], + "group_graphviz_path": diff_ret["group"].dot_path, + "group_test_py_code_path": diff_ret["group"].txt_path, + "group_diff_subgraph_id": diff_ret["subgraph_id"], + }, + "group_output_diff", + ) + + if __name__ == "__main__": cmp = Comparator() base = cmp.load_var("/root/dev/PaddleClas/base/saved_tensors/batch_norm_grad-input-batch_norm_0.tmp_3@GRAD") cinn = cmp.load_var("/root/dev/PaddleClas/cinn/saved_tensors/batch_norm_grad-input-batch_norm_0.tmp_3@GRAD") - np.testing.assert_allclose(base, cinn, rtol=0, atol=0) \ No newline at end of file + np.testing.assert_allclose(base, cinn, rtol=0, atol=0) diff --git a/padiff/cinn_diff/env.py b/padiff/cinn_diff/env.py index f5af006..a4e577e 100644 --- a/padiff/cinn_diff/env.py +++ b/padiff/cinn_diff/env.py @@ -1,33 +1,51 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import subprocess -from sys import stderr class Env: - + base_dir_name = "base" cinn_dir_name = "cinn" cinn_pass_dir = "cinn_pass" cinn_graph_dir = "cinn_graph" - def __init__(self, script=None, base_env=None, cinn_env=None, ): + def __init__( + self, + script=None, + base_env=None, + cinn_env=None, + ): self._base_env = { - "CUDA_VISIBLE_DEVICES" : "7", - "NVIDIA_TF32_OVERRIDE" : "1", - "CUDA_LAUNCH_BLOCKING" : "1", - "FLAGS_save_static_runtime_data" : "1", - "FLAGS_static_runtime_data_save_path" : "./", - "FLAGS_cudnn_deterministc" : "1", - "FLAGS_cinn_cudnn_deterministc" : "1", - "FLAGS_prim_all" : "true" + "CUDA_VISIBLE_DEVICES": "7", + "NVIDIA_TF32_OVERRIDE": "1", + "CUDA_LAUNCH_BLOCKING": "1", + "FLAGS_save_static_runtime_data": "1", + "FLAGS_static_runtime_data_save_path": "./", + "FLAGS_cudnn_deterministc": "1", + "FLAGS_cinn_cudnn_deterministc": "1", + "FLAGS_prim_all": "true", } self._cinn_env = { - "FLAGS_use_cinn" : "1", - "FLAGS_deny_cinn_ops" :"reduce_sum", - "FLAGS_use_reduce_split_pass" : "1", - "FLAGS_nvrtc_compile_to_cubin" : "0", - "FLAGS_cinn_use_op_fusion" : "1", - "FLAGS_cinn_parallel_compile_size" : "8", + "FLAGS_use_cinn": "1", + "FLAGS_deny_cinn_ops": "reduce_sum", + "FLAGS_use_reduce_split_pass": "1", + "FLAGS_nvrtc_compile_to_cubin": "0", + "FLAGS_cinn_use_op_fusion": "1", + "FLAGS_cinn_parallel_compile_size": "8", "FLAGS_cinn_pass_visualize_dir": "", } self.base_env = base_env if base_env else self._base_env @@ -38,38 +56,36 @@ def __init__(self, script=None, base_env=None, cinn_env=None, ): self.script_path = os.path.dirname(script) self.script_name = os.path.basename(script) self.os_env = dict(os.environ) - + def init_base_env(self): if os.path.exists(self.base_path): print("base path exists, remove it") os.system("rm -rf " + self.base_path) - self.base_env['FLAGS_static_runtime_data_save_path'] = self.base_path - self.base_env['FLAGS_save_static_runtime_data'] = "1" - + self.base_env["FLAGS_static_runtime_data_save_path"] = self.base_path + self.base_env["FLAGS_save_static_runtime_data"] = "1" + def set_base_env(self, env): self.base_env = env def init_cinn_env(self): - self.base_env['FLAGS_static_runtime_data_save_path'] = self.cinn_path + self.base_env["FLAGS_static_runtime_data_save_path"] = self.cinn_path if os.path.exists(self.cinn_path): print("cinn path exists, remove it") os.system("rm -rf " + self.cinn_path) - self.cinn_env['FLAGS_cinn_pass_visualize_dir'] = os.path.join(self.cinn_path, self.cinn_pass_dir) - self.cinn_env['FLAGS_cinn_subgraph_graphviz_dir'] = os.path.join(self.cinn_path, self.cinn_graph_dir) - + self.cinn_env["FLAGS_cinn_pass_visualize_dir"] = os.path.join(self.cinn_path, self.cinn_pass_dir) + self.cinn_env["FLAGS_cinn_subgraph_graphviz_dir"] = os.path.join(self.cinn_path, self.cinn_graph_dir) + def set_cinn_env(self, env): self.cinn_env = env - - + def set_script(self, name): self.script = name - + def run_model(self, run_env, log): print(self.script) - ret = subprocess.run(["sh", self.script_name], env=run_env, stdout=log, stderr=log) + ret = subprocess.run(["sh", self.script_name], env=run_env, stdout=log, stderr=log) print(ret) - def run_base_model(self): self.init_base_env() os.chdir(self.script_path) @@ -90,11 +106,3 @@ def run_cinn_model(self): cinn_log = open("cinn.log", "w") self.run_model(run_env, cinn_log) cinn_log.close() - - - - - - - - diff --git a/padiff/cinn_diff/graph.py b/padiff/cinn_diff/graph.py index 6cbe39a..83f02ba 100644 --- a/padiff/cinn_diff/graph.py +++ b/padiff/cinn_diff/graph.py @@ -1,15 +1,30 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from lib2to3.pytree import Node import graphviz import pygraphviz as pgv -from utils import retry, suppress_stdout +from utils import retry + @retry(max_times=1) def get_graph(dot_path): return pgv.AGraph(dot_path) + def construct_graph_by_dot(dot_path, sep="\\n"): - #print("dot_path:" + dot_path) + # print("dot_path:" + dot_path) graph_source = get_graph(dot_path) # ['color', 'label', 'style'] all_nodes = [] @@ -22,14 +37,14 @@ def construct_graph_by_dot(dot_path, sep="\\n"): subgraph_id = subgraph.get_name().split("_")[-1] tmp_nodes = [] for node in subgraph.nodes(): - name = node.attr['label'].split(sep)[0] + name = node.attr["label"].split(sep)[0] idx = node.get_name() - cls_node = Node(name = name, idx = idx, type="unknown", graph_id = subgraph_id) + cls_node = Node(name=name, idx=idx, type="unknown", graph_id=subgraph_id) idx2nodes[idx] = cls_node all_nodes.append(cls_node) tmp_nodes.append(cls_node) - ret_subgraphs.append(Graph(nodes=tmp_nodes, name = f"group_{subgraph_id}")) - + ret_subgraphs.append(Graph(nodes=tmp_nodes, name=f"group_{subgraph_id}")) + for edge in graph_source.edges(): start, end = edge if start not in idx2nodes.keys() or end not in idx2nodes.keys(): @@ -38,15 +53,15 @@ def construct_graph_by_dot(dot_path, sep="\\n"): idx2nodes[start].add_output(idx2nodes[end]) # 输入边 idx2nodes[end].add_input(idx2nodes[start]) - + return all_nodes, ret_subgraphs -class Graph: +class Graph: def __init__(self, nodes, name) -> None: self.nodes = nodes self.name = str(name) - + def add_node(self, node): if isinstance(node, Node): if node in self.nodes: @@ -55,16 +70,16 @@ def add_node(self, node): self.nodes.append(node) else: raise ValueError(" param type must be Node") - + # just for cinn graph def graph_inputs(self): inputs = set() for node in self.nodes: if node.name == "feed": inputs.add(node.outputs[0].name) - + return inputs - + def inputs(self): inputs = [] for node in self.nodes: @@ -78,7 +93,6 @@ def inputs(self): inputs.append(node) return inputs - # just for cinn graph def graph_outputs(self): outputs = set() @@ -86,9 +100,9 @@ def graph_outputs(self): if node.name == "fetch": outputs.add(node.inputs[0].name) if not node.outputs: - outputs.add(node.name) + outputs.add(node.name) return outputs - + def outputs(self): outputs = [] for node in self.nodes: @@ -107,21 +121,21 @@ def find(self, cinn_var_name): if node.name == cinn_var_name: return node return None - + def is_input(self, node): return node in self.inputs() def is_output(self, node): return node in self.outputs() - + def export_dot(self): - dot = graphviz.Digraph(comment = self.name) + dot = graphviz.Digraph(comment=self.name) for item in self.nodes: dot.node(item.idx, item.idx + "\n" + item.name + ":" + item.node_type) for next in item.outputs: dot.edge(item.idx, next.idx) return dot - + def __str__(self) -> str: return "graph_" + str(self.name) @@ -130,7 +144,6 @@ def __repr__(self) -> str: class Pass: - def __init__(self, id, pass_name=None, before_txt=None, after_txt=None, before_dot=None, after_dot=None) -> None: self.pass_id = id self.pass_name = pass_name @@ -138,31 +151,30 @@ def __init__(self, id, pass_name=None, before_txt=None, after_txt=None, before_d self.after_txt = after_txt self.before_dot = before_dot self.after_dot = after_dot - + def set_pass_name(self, pass_name): self.pass_name = pass_name - + def set_before_txt(self, before_txt): self.before_txt = before_txt - + def set_after_txt(self, after_txt): self.after_txt = after_txt def set_before_dot(self, before_dot): self.before_dot = before_dot - + def set_after_dot(self, after_dot): self.after_dot = after_dot - + def __str__(self) -> str: - return "pass_" + str(self.pass_id) + "_" + self.pass_name + return "pass_" + str(self.pass_id) + "_" + self.pass_name def __repr__(self) -> str: return "pass_" + str(self.pass_id) + "_" + self.pass_name class Group: - def __init__(self, group_id, all_passes, last_pass_id) -> None: self.group_id = group_id self.passes = all_passes @@ -171,52 +183,51 @@ def __init__(self, group_id, all_passes, last_pass_id) -> None: self.all_nodes, self.subgraphs = construct_graph_by_dot(self.dot_path) self.fetch = None self.feed = None - - + def export_graph(self): self.graph = Graph(self.all_nodes, self.__str__) dot = self.graph.export_dot() - dot.render(self.__str__(), format='png', cleanup=True) - + dot.render(self.__str__(), format="png", cleanup=True) + def export_dot(self): dot = graphviz.Source(self.dot_path) - dot.render(self.__str__(), format='png', cleanup=True) - + dot.render(self.__str__(), format="png", cleanup=True) + def __str__(self) -> str: - return "fusion_group_" + str(self.group_id) + return "fusion_group_" + str(self.group_id) def __repr__(self) -> str: return "fusion_group_" + str(self.group_id) - - + + class Node: - def __init__(self, name, type, idx, graph_id = None) -> None: - self.name = name #var name, like arg_1 + def __init__(self, name, type, idx, graph_id=None) -> None: + self.name = name # var name, like arg_1 self.node_type = type if type else "unknown" - self.idx = idx # node name like node1 + self.idx = idx # node name like node1 self.inputs = [] self.outputs = [] self.cinn_name = "" self.graph_id = graph_id - + def is_op(self): return self.node_type == "op" - + def is_var(self): return self.node_type == "var" - + def is_leaf(self): return self.outputs == [] or self.outputs[0].name == ["fetch"] - + def is_root(self): return self.inputs == [] or self.inputs[0].name == ["feed"] - + def set_outputs(self, outputs): self.outputs = outputs def set_inputs(self, inputs): self.inputs = inputs - + def add_input(self, node): if isinstance(node, Node): if node in self.inputs: @@ -225,7 +236,7 @@ def add_input(self, node): self.inputs.append(node) else: raise ValueError("Node input must be Node") - + def add_output(self, node): if isinstance(node, Node): if node in self.outputs: @@ -234,28 +245,27 @@ def add_output(self, node): self.outputs.append(node) else: raise ValueError("Node output must be Node") - + def __str__(self) -> str: - return self.name + "_" + self.idx + " : " + self.node_type + return self.name + "_" + self.idx + " : " + self.node_type def __repr__(self) -> str: - return self.name + "_" + self.idx + " : " + self.node_type + return self.name + "_" + self.idx + " : " + self.node_type class Cluster: - def __init__(self, idx, graph, ops, inputs, outputs, graph_key, varmaps=None) -> None: - self.idx = idx - self.graph = graph - self.ops = ops - self.inputs = inputs - self.outputs = outputs - self.graph_key = graph_key - self.varmaps = varmaps - self.cinn_group = None - + self.idx = idx + self.graph = graph + self.ops = ops + self.inputs = inputs + self.outputs = outputs + self.graph_key = graph_key + self.varmaps = varmaps + self.cinn_group = None + def set_varmaps(self, varmaps: dict): - self.varmaps = varmaps + self.varmaps = varmaps def set_associate_groups(self, group): if isinstance(group, [list, set, tuple]): @@ -264,8 +274,7 @@ def set_associate_groups(self, group): self.associate_groups.append(group) else: raise ValueError(f"group must be str or sequence type, but got {type(group)}") - - + def __str__(self) -> str: return "Cluster_" + str(self.idx) @@ -274,13 +283,4 @@ def __repr__(self) -> str: def print_varmaps(self): for paddle_name, cinn_name in self.varmaps.items(): - print({ - 'graph_key': self.graph_key, - 'paddle_name': paddle_name, - 'cinn_name': cinn_name - }) - - - - - + print({"graph_key": self.graph_key, "paddle_name": paddle_name, "cinn_name": cinn_name}) diff --git a/padiff/cinn_diff/logs.py b/padiff/cinn_diff/logs.py index e69de29..fd05a92 100644 --- a/padiff/cinn_diff/logs.py +++ b/padiff/cinn_diff/logs.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/padiff/cinn_diff/read_file.py b/padiff/cinn_diff/read_file.py index 143d55a..3a89f6a 100644 --- a/padiff/cinn_diff/read_file.py +++ b/padiff/cinn_diff/read_file.py @@ -1,13 +1,27 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os from graph import Graph, Node, Cluster, Group, Pass, construct_graph_by_dot import collections -INPUTS_NAME = "cluster_inputs.txt" +INPUTS_NAME = "cluster_inputs.txt" OUTPUTS_NAME = "cluster_outputs.txt" -OPS_NAME = "cluster_ops.txt" -GRAPH_NAME = "subgraph.txt" +OPS_NAME = "cluster_ops.txt" +GRAPH_NAME = "subgraph.txt" PADDLE2CINN_VARMAP = "paddle2cinn_varmap.txt" GRAPH_COMPLATION_KEY = "graph_compilation_key.txt" VARMAPS_KEY_NAME = "graph_compilation_key" @@ -43,12 +57,13 @@ def read_tensors(tensors_path): continue tensors_map[var_info[-1]] = os.path.join(tensors_path, file) return tensors_map - + def read_graph(graph_file, idx): assert os.path.isfile(graph_file) nodes = {} edges = {} + def record_nodes_and_edges(line, type, nodes, edges): if type == "nodes": node = line.split(" : ") @@ -61,32 +76,34 @@ def record_nodes_and_edges(line, type, nodes, edges): edge = line.split(" -> ") cur, next = edge[0], edge[1] edges[cur] = next - else : + else: raise ValueError(type + "not support") + type = "nodes" with open(graph_file) as f: lines = f.readlines() for line in lines: line = line.strip("\n") - if line.startswith("nodes:"): # start to record node + if line.startswith("nodes:"): # start to record node type = "nodes" continue - if line.startswith("edges:"): # start to record edge + if line.startswith("edges:"): # start to record edge type = "edges" continue record_nodes_and_edges(line, type, nodes, edges) - + def construct_graph(nodes, edges): for k, v in edges.items(): if k not in nodes or v not in nodes: continue nodes[k].add_output(nodes[v]) - nodes[v].add_input(nodes[k]) + nodes[v].add_input(nodes[k]) graph = Graph(nodes.values(), idx) return graph + graph = construct_graph(nodes, edges) return graph - + def read_strings(string_file): assert os.path.isfile(string_file) @@ -96,6 +113,7 @@ def read_strings(string_file): rets = line.split(", ")[:-1] return rets + def read_string(string_file): assert os.path.isfile(string_file) with open(string_file) as f: @@ -103,6 +121,7 @@ def read_string(string_file): rets = line return rets + def read_cluster(path, idx): assert os.path.isdir(path), f"{path} must be dir" inputs = read_strings(os.path.join(path, INPUTS_NAME)) @@ -115,21 +134,22 @@ def read_cluster(path, idx): def read_cinn_pass(path): all_groups = {} + def read_graphviz_dot(path): passes = os.listdir(path) idx = path.split("_")[-1] - #print("group idx: " + str(idx)) + # print("group idx: " + str(idx)) all_passes = {} for pass_path in passes: - #print("pass_path: " + pass_path) - #print(pass_path.split("_")) + # print("pass_path: " + pass_path) + # print(pass_path.split("_")) pass_idx = int(pass_path.split("_")[1]) - #print("pass_idx: " + str(pass_idx)) + # print("pass_idx: " + str(pass_idx)) if pass_idx not in all_passes: all_passes[pass_idx] = Pass(pass_idx) pass_name = pass_path.split("_")[2] all_passes[pass_idx].set_pass_name(pass_name) - type = pass_path.split("_")[3] #after.txt + type = pass_path.split("_")[3] # after.txt record_path = os.path.join(path, pass_path) if type == "after.txt": all_passes[pass_idx].set_after_txt(record_path) @@ -142,10 +162,10 @@ def read_graphviz_dot(path): else: raise ValueError(type + "not support") max_pass_id = max(all_passes.keys()) - #print("lass_pass_id: " + str(max_pass_id)) + # print("lass_pass_id: " + str(max_pass_id)) group_cc = Group(idx, all_passes, max_pass_id) all_groups[idx] = group_cc - + file_names = os.listdir(path) for file_name in file_names: read_graphviz_dot(os.path.join(path, file_name)) @@ -176,7 +196,7 @@ def set_node_cinn_name(all_clusters): nodes = cluster.graph.nodes for node in nodes: if node.is_var(): - node.cinn_name = cluster.varmaps.get(node.name, '') + node.cinn_name = cluster.varmaps.get(node.name, "") def set_cluster_varmaps(clusters, varmaps): @@ -186,6 +206,7 @@ def set_cluster_varmaps(clusters, varmaps): raise KeyError(f"can't find graph key {cluster.graph_key} in graph2varmaps") cluster.set_varmaps(tmp_varmaps) + def set_clusters_group(clusters, groups, cinn_graphs): for cluster in clusters: inputs = cluster.inputs @@ -202,40 +223,39 @@ def set_clusters_group(clusters, groups, cinn_graphs): cluster.cinn_group = groups[idx] - - - -def read_all(root_path="", type = "cinn"): +def read_all(root_path="", type="cinn"): assert root_path, f"{root_path} can't be None" all_clusters = [] - #paddle2cinn_varmaps + # paddle2cinn_varmaps graph2varmaps = {} all_vars_paths = {} all_cinn_groups = {} all_cinn_graphs = {} - #exsist some bug + # exsist some bug cinn2paddle_varmaps = {} - allinone = collections.namedtuple('allinone', ['all_clusters', 'all_varmaps', 'all_vars_paths', 'all_cinn_groups', "cinn2paddle"]) + allinone = collections.namedtuple( + "allinone", ["all_clusters", "all_varmaps", "all_vars_paths", "all_cinn_groups", "cinn2paddle"] + ) all_paths = os.listdir(root_path) for path in all_paths: file_path = os.path.join(root_path, path) - assert os.path.isfile(file_path) or os.path.isdir(file_path), f'{file_path} must be path or dir' + assert os.path.isfile(file_path) or os.path.isdir(file_path), f"{file_path} must be path or dir" if type == "cinn" and path.startswith("cluster"): idx = path.split("_")[-1] all_clusters.append(read_cluster(file_path, idx)) - + if type == "cinn" and path == "paddle2cinn_varmap": graph2varmaps.update(read_varmaps(file_path)) - + if path == "saved_tensors": all_vars_paths.update(read_tensors(file_path)) - + if type == "cinn" and path == "cinn_pass": all_cinn_groups = read_cinn_pass(file_path) - + if type == "cinn" and path == "cinn_graph": all_cinn_graphs = read_cinn_graph(file_path) @@ -244,10 +264,9 @@ def read_all(root_path="", type = "cinn"): if type == "cinn": set_node_cinn_name(all_clusters) set_clusters_group(all_clusters, all_cinn_groups, all_cinn_graphs) - return allinone(all_clusters, graph2varmaps, all_vars_paths, all_cinn_groups, cinn2paddle_varmaps) + if __name__ == "__main__": read_all() - diff --git a/padiff/cinn_diff/requirments.txt b/padiff/cinn_diff/requirments.txt index 4a45c25..27f2041 100644 --- a/padiff/cinn_diff/requirments.txt +++ b/padiff/cinn_diff/requirments.txt @@ -1,2 +1,2 @@ pygraphviz==1.11 -graphviz==0.20.1 \ No newline at end of file +graphviz==0.20.1 diff --git a/padiff/cinn_diff/run.py b/padiff/cinn_diff/run.py index 151ec35..c8e23d6 100644 --- a/padiff/cinn_diff/run.py +++ b/padiff/cinn_diff/run.py @@ -1,16 +1,28 @@ -import os +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from analyze import auto_diff from env import Env def run(run_script, base_env, cinn_env): run_env = Env(run_script, base_env, cinn_env) - run_env.run_base_model() #可以提供选项选择不运行base model + run_env.run_base_model() # 可以提供选项选择不运行base model run_env.run_cinn_model() auto_diff(run_env.base_path, run_env.cinn_path, rtol=0, atol=0) -if __name__ == '__main__': +if __name__ == "__main__": run_script = "/root/dev/PaddleNLP/model_zoo/bert/run_bert.sh" run(run_script, None, None) - diff --git a/padiff/cinn_diff/utils.py b/padiff/cinn_diff/utils.py index 14aa6a0..f71b1a0 100644 --- a/padiff/cinn_diff/utils.py +++ b/padiff/cinn_diff/utils.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from contextlib import contextmanager from functools import wraps import os, sys @@ -12,13 +26,14 @@ def suppress_stdout(): sys.stdout = devnull sys.stderr = devnull sys.stdin = devnull - try: + try: yield finally: sys.stdout = old_stdout sys.stderr = old_stderr sys.stdin = old_stdin + def retry(max_times=1): def retry_decorator(func): @wraps(func) @@ -29,29 +44,31 @@ def inner(*args, **kwargs): ret = func(*args, **kwargs) return ret except Exception as e: - retry_times+=1 + retry_times += 1 + return inner + return retry_decorator - + # \033 [显示方式;字体色;背景色m ...... [\033[0m] # 显示方式: 0（默认值）、1（高亮）、22（非粗体）、4（下划线）、24（非下划线）、 5（闪烁）、25（非闪烁）、7（反显）、27（非反显） # 前景色: 30（黑色）、31（红色）、32（绿色）、 33（黄色）、34（蓝色）、35（洋 红）、36（青色）、37（白色） # 背景色: 40（黑色）、41（红色）、42（绿色）、 43（黄色）、44（蓝色）、45（洋 红）、46（青色）、47（白色） -class console: +class console: def __init__(self) -> None: pass - + @classmethod def red(self, str): - return - + return + @classmethod def info(self, str): return - + @classmethod def error(self, str): return @@ -59,5 +76,3 @@ def error(self, str): @classmethod def warning(self, str): return - - \ No newline at end of file From d61b6f845b24916244079550fc94925ae1b2359d Mon Sep 17 00:00:00 2001 From: GGBond8488 <857631483@qq.com> Date: Tue, 16 Jan 2024 18:44:03 +0800 Subject: [PATCH 03/10] fix package bug --- README.md | 18 ++++++++++++++++++ padiff/__init__.py | 1 + padiff/cinn_diff/README.md | 14 ++++---------- padiff/cinn_diff/analyze.py | 4 ++-- padiff/cinn_diff/env.py | 2 +- padiff/cinn_diff/graph.py | 3 +-- padiff/cinn_diff/read_file.py | 2 +- 7 files changed, 28 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 7b9455c..5b5f1fe 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,25 @@ for i in range(6): assert check_params(f"./torch/step_{i}", f"./paddle/step_{i}") == True ``` +### 框架与编译器对齐 +使用文档 [CINN](padiff/cinn_diff/README.md) +```python +import os +from padiff import cinn_diff + + +def run(run_script, base_env, cinn_env): + run_env = cinn_diff.Env(run_script, base_env, cinn_env) + run_env.run_base_model() #可以注释掉选择不运行base model + run_env.run_cinn_model() #也可以注释掉选择不运行cinn model + cinn_diff.auto_diff(run_env.base_path, run_env.cinn_path, rtol=1e-3, atol=1e-3) + + +if __name__ == '__main__': + run_script = "/root/workspace/PaddleNLP/model_zoo/bert/run_bert.sh" + run(run_script, None, None) +``` ## 已支持 `Special Init` 的组件 diff --git a/padiff/__init__.py b/padiff/__init__.py index c7be604..4ead7bd 100644 --- a/padiff/__init__.py +++ b/padiff/__init__.py @@ -27,6 +27,7 @@ from .report.hooks import info_hook from .datas import global_json_laoder as jsons +from . import cinn_diff def module_filter(name): diff --git a/padiff/cinn_diff/README.md b/padiff/cinn_diff/README.md index 09f4f1d..857838c 100644 --- a/padiff/cinn_diff/README.md +++ b/padiff/cinn_diff/README.md @@ -9,20 +9,18 @@ **example** ```python import os -from analyze import auto_diff -from env import Env +from padiff import cinn_diff def run(run_script, base_env, cinn_env): - run_env = Env(run_script, base_env, cinn_env) + run_env = cinn_diff.Env(run_script, base_env, cinn_env) run_env.run_base_model() #可以注释掉选择不运行base model run_env.run_cinn_model() #也可以注释掉选择不运行cinn model - auto_diff(run_env.base_path, run_env.cinn_path, rtol=1e-3, atol=1e-3) + cinn_diff.auto_diff(run_env.base_path, run_env.cinn_path, rtol=1e-3, atol=1e-3) if __name__ == '__main__': - run_script = "/root/dev/PaddleNLP/model_zoo/bert/run_bert.sh" - run_script = "/root/dev/PaddleClas/run_resnet.sh" + run_script = "/root/workspace/PaddleNLP/model_zoo/bert/run_bert.sh" run(run_script, None, None) ``` **run_script** @@ -129,8 +127,4 @@ python run_pretrain.py \ ![运行结果图](./img/run_ret.png) -## 功能扩展 -1. 【开发中】参数式启动 -2. 【开发中】中间变量读取展示接口 - 更多功能正在研发中... diff --git a/padiff/cinn_diff/analyze.py b/padiff/cinn_diff/analyze.py index 34efdaa..8685a91 100644 --- a/padiff/cinn_diff/analyze.py +++ b/padiff/cinn_diff/analyze.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from read_file import read_all -from compare_utils import Comparator +from .read_file import read_all +from .compare_utils import Comparator # 需要优化 def back_track_group(base, compare, cluster, cmp, graph, node): diff --git a/padiff/cinn_diff/env.py b/padiff/cinn_diff/env.py index a4e577e..728f403 100644 --- a/padiff/cinn_diff/env.py +++ b/padiff/cinn_diff/env.py @@ -41,7 +41,7 @@ def __init__( } self._cinn_env = { "FLAGS_use_cinn": "1", - "FLAGS_deny_cinn_ops": "reduce_sum", + "FLAGS_deny_cinn_ops": "", "FLAGS_use_reduce_split_pass": "1", "FLAGS_nvrtc_compile_to_cubin": "0", "FLAGS_cinn_use_op_fusion": "1", diff --git a/padiff/cinn_diff/graph.py b/padiff/cinn_diff/graph.py index 83f02ba..a574136 100644 --- a/padiff/cinn_diff/graph.py +++ b/padiff/cinn_diff/graph.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from lib2to3.pytree import Node import graphviz import pygraphviz as pgv -from utils import retry +from .utils import retry @retry(max_times=1) diff --git a/padiff/cinn_diff/read_file.py b/padiff/cinn_diff/read_file.py index 3a89f6a..89dbbe5 100644 --- a/padiff/cinn_diff/read_file.py +++ b/padiff/cinn_diff/read_file.py @@ -14,7 +14,7 @@ import os -from graph import Graph, Node, Cluster, Group, Pass, construct_graph_by_dot +from .graph import Graph, Node, Cluster, Group, Pass, construct_graph_by_dot import collections From 775c630ea8a5cdb4af2ac3cff5722493e4c01149 Mon Sep 17 00:00:00 2001 From: GGBond8488 <857631483@qq.com> Date: Wed, 17 Jan 2024 15:44:09 +0800 Subject: [PATCH 04/10] add log --- padiff/cinn_diff/.gitignore | 2 ++ padiff/cinn_diff/__init__.py | 1 + padiff/cinn_diff/analyze.py | 13 ++++----- padiff/cinn_diff/compare_utils.py | 2 +- padiff/cinn_diff/env.py | 13 ++++----- padiff/cinn_diff/graph.py | 5 ++-- padiff/cinn_diff/logs.py | 45 +++++++++++++++++++++++++++++++ padiff/cinn_diff/read_file.py | 17 ++++++------ padiff/cinn_diff/run.py | 29 ++++++++++++++++---- 9 files changed, 99 insertions(+), 28 deletions(-) create mode 100644 padiff/cinn_diff/.gitignore diff --git a/padiff/cinn_diff/.gitignore b/padiff/cinn_diff/.gitignore new file mode 100644 index 0000000..056dc05 --- /dev/null +++ b/padiff/cinn_diff/.gitignore @@ -0,0 +1,2 @@ +*.json +*.log \ No newline at end of file diff --git a/padiff/cinn_diff/__init__.py b/padiff/cinn_diff/__init__.py index 7ab1050..0511ee5 100644 --- a/padiff/cinn_diff/__init__.py +++ b/padiff/cinn_diff/__init__.py @@ -16,3 +16,4 @@ from .compare_utils import * from .utils import * from .env import * +from .analyze import * diff --git a/padiff/cinn_diff/analyze.py b/padiff/cinn_diff/analyze.py index 8685a91..af0887b 100644 --- a/padiff/cinn_diff/analyze.py +++ b/padiff/cinn_diff/analyze.py @@ -14,6 +14,7 @@ from .read_file import read_all from .compare_utils import Comparator +from .logs import logger # 需要优化 def back_track_group(base, compare, cluster, cmp, graph, node): @@ -25,7 +26,7 @@ def back_track_group(base, compare, cluster, cmp, graph, node): tmp = input paddle_name = cur_cluster_cinn2paddle.get(tmp.name, "") if not paddle_name: - print(f"can't find {node.name}'s paddle name") + logger.info(f"can't find {node.name}'s paddle name") diff_ret = { "cluster": cluster.idx, "group": cluster.cinn_group, @@ -63,7 +64,7 @@ def auto_diff(base_path, compare_path, rtol=1e-6, atol=1e-6): # step1: 确认cluster的输入输出是否对齐 for cluster in compare.all_clusters: - # print(cluster.idx) + # logger.info(cluster.idx) input_equals_flag = True output_equals_flag = True for input in cluster.inputs: @@ -86,7 +87,7 @@ def auto_diff(base_path, compare_path, rtol=1e-6, atol=1e-6): # step3: 找到对不齐变量对应的group output_cinn_var = cluster.varmaps.get(output, "") if not output_cinn_var: - print("can't find var " + output + " corresponding cinn var name") + logger.info("can't find var " + output + " corresponding cinn var name") else: find_diff_group_flag = False # step4 : 从对不齐的输出出发，找到第一次出现输出对不齐的group（输入能对齐，输出无法对齐） @@ -104,13 +105,13 @@ def auto_diff(base_path, compare_path, rtol=1e-6, atol=1e-6): if not find_diff_group_flag: cmp.record_output_diff(cluster.idx, output, cluster.varmaps.get(output, "")) - print("can't find diff group in cluster_" + cluster.idx + " but diff exsits") + logger.info("can't find diff group in cluster_" + cluster.idx + " but diff exsits") if output_equals_flag: - print("cluster_" + cluster.idx + " has no diff") + logger.info("cluster_" + cluster.idx + " has no diff") for diff in cmp.record: - print(diff) + logger.info(diff) return cmp.record diff --git a/padiff/cinn_diff/compare_utils.py b/padiff/cinn_diff/compare_utils.py index 1bce4ea..8d11233 100644 --- a/padiff/cinn_diff/compare_utils.py +++ b/padiff/cinn_diff/compare_utils.py @@ -42,8 +42,8 @@ def assert_allclose(self, base_path, compare_path): def record_diff(self, diff, type): diff = { - "event": diff, "type": type, + "event": diff, } self.record.append(diff) diff --git a/padiff/cinn_diff/env.py b/padiff/cinn_diff/env.py index 728f403..932651f 100644 --- a/padiff/cinn_diff/env.py +++ b/padiff/cinn_diff/env.py @@ -14,6 +14,7 @@ import os import subprocess +from .logs import logger class Env: @@ -59,7 +60,7 @@ def __init__( def init_base_env(self): if os.path.exists(self.base_path): - print("base path exists, remove it") + logger.info("base path exists, remove it") os.system("rm -rf " + self.base_path) self.base_env["FLAGS_static_runtime_data_save_path"] = self.base_path self.base_env["FLAGS_save_static_runtime_data"] = "1" @@ -70,7 +71,7 @@ def set_base_env(self, env): def init_cinn_env(self): self.base_env["FLAGS_static_runtime_data_save_path"] = self.cinn_path if os.path.exists(self.cinn_path): - print("cinn path exists, remove it") + logger.info("cinn path exists, remove it") os.system("rm -rf " + self.cinn_path) self.cinn_env["FLAGS_cinn_pass_visualize_dir"] = os.path.join(self.cinn_path, self.cinn_pass_dir) self.cinn_env["FLAGS_cinn_subgraph_graphviz_dir"] = os.path.join(self.cinn_path, self.cinn_graph_dir) @@ -82,15 +83,15 @@ def set_script(self, name): self.script = name def run_model(self, run_env, log): - print(self.script) + logger.info(self.script) ret = subprocess.run(["sh", self.script_name], env=run_env, stdout=log, stderr=log) - print(ret) + logger.info(ret) def run_base_model(self): self.init_base_env() os.chdir(self.script_path) run_env = self.base_env.copy() - print(run_env) + logger.info(run_env) run_env.update(self.os_env) base_log = open("base.log", "w") self.run_model(run_env, base_log) @@ -101,7 +102,7 @@ def run_cinn_model(self): os.chdir(self.script_path) run_env = self.cinn_env.copy() run_env.update(self.base_env) - print(run_env) + logger.info(run_env) run_env.update(self.os_env) cinn_log = open("cinn.log", "w") self.run_model(run_env, cinn_log) diff --git a/padiff/cinn_diff/graph.py b/padiff/cinn_diff/graph.py index a574136..630f2e8 100644 --- a/padiff/cinn_diff/graph.py +++ b/padiff/cinn_diff/graph.py @@ -15,6 +15,7 @@ import graphviz import pygraphviz as pgv from .utils import retry +from .logs import logger @retry(max_times=1) @@ -23,7 +24,7 @@ def get_graph(dot_path): def construct_graph_by_dot(dot_path, sep="\\n"): - # print("dot_path:" + dot_path) + # logger.info("dot_path:" + dot_path) graph_source = get_graph(dot_path) # ['color', 'label', 'style'] all_nodes = [] @@ -282,4 +283,4 @@ def __repr__(self) -> str: def print_varmaps(self): for paddle_name, cinn_name in self.varmaps.items(): - print({"graph_key": self.graph_key, "paddle_name": paddle_name, "cinn_name": cinn_name}) + logger.info({"graph_key": self.graph_key, "paddle_name": paddle_name, "cinn_name": cinn_name}) diff --git a/padiff/cinn_diff/logs.py b/padiff/cinn_diff/logs.py index fd05a92..51f1bb9 100644 --- a/padiff/cinn_diff/logs.py +++ b/padiff/cinn_diff/logs.py @@ -11,3 +11,48 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import os +import sys +import logging +from logging import handlers + + +class Logger(object): + def __init__( + self, filename, level=logging.INFO, when="D", backCount=3, fmt="%(asctime)s" "-%(levelname)s: %(message)s" + ): + + self.logger = logging.getLogger(filename) # 根据文件名创建一个日志 + self.logger.setLevel(level) # 设置默认日志级别 + self.format_str = logging.Formatter(fmt) # 设置日志格式 + + # screen_handler = logging.StreamHandler() # 屏幕输出处理器 + # screen_handler.setFormatter(self.format_str) # 设置屏幕输出显示格式 + + # 定时写入文件处理器 + time_file_handler = handlers.TimedRotatingFileHandler( + filename=filename, # 日志文件名 + when=when, # 多久创建一个新文件 + interval=1, # 写入时间间隔 + backupCount=backCount, # 备份文件的个数 + encoding="utf-8", + ) # 编码格式 + + time_file_handler.setFormatter(self.format_str) + + # 添加日志处理器 + # self.logger.addHandler(screen_handler) + self.logger.addHandler(time_file_handler) + + def loggerImp(self): + return self.logger + + +logger = Logger(os.path.join(sys.path[0], "cinn_diff_log.log")).loggerImp() + + +def log_init(file_name): + global logger + if file_name: + logger = Logger(os.path.join(sys.path[0], file_name)).loggerImp() diff --git a/padiff/cinn_diff/read_file.py b/padiff/cinn_diff/read_file.py index 89dbbe5..fd086ba 100644 --- a/padiff/cinn_diff/read_file.py +++ b/padiff/cinn_diff/read_file.py @@ -15,6 +15,7 @@ import os from .graph import Graph, Node, Cluster, Group, Pass, construct_graph_by_dot +from .logs import logger import collections @@ -138,13 +139,13 @@ def read_cinn_pass(path): def read_graphviz_dot(path): passes = os.listdir(path) idx = path.split("_")[-1] - # print("group idx: " + str(idx)) + # logger.info("group idx: " + str(idx)) all_passes = {} for pass_path in passes: - # print("pass_path: " + pass_path) - # print(pass_path.split("_")) + # logger.info("pass_path: " + pass_path) + # logger.info(pass_path.split("_")) pass_idx = int(pass_path.split("_")[1]) - # print("pass_idx: " + str(pass_idx)) + # logger.info("pass_idx: " + str(pass_idx)) if pass_idx not in all_passes: all_passes[pass_idx] = Pass(pass_idx) pass_name = pass_path.split("_")[2] @@ -162,7 +163,7 @@ def read_graphviz_dot(path): else: raise ValueError(type + "not support") max_pass_id = max(all_passes.keys()) - # print("lass_pass_id: " + str(max_pass_id)) + # logger.info("lass_pass_id: " + str(max_pass_id)) group_cc = Group(idx, all_passes, max_pass_id) all_groups[idx] = group_cc @@ -216,10 +217,10 @@ def set_clusters_group(clusters, groups, cinn_graphs): graph_outputs = graph.graph_outputs() if not graph_inputs and not graph_outputs: raise ValueError(f"{graph} does not have inputs or outputs") - # print(graph_inputs) - # print(inputs) + # logger.info(graph_inputs) + # logger.info(inputs) if not set(inputs).difference(graph_inputs) and not set(outputs).difference(graph_outputs): - print(f"group_{idx} belongs to Cluster_{cluster.idx}") + logger.info(f"group_{idx} belongs to Cluster_{cluster.idx}") cluster.cinn_group = groups[idx] diff --git a/padiff/cinn_diff/run.py b/padiff/cinn_diff/run.py index c8e23d6..09bd3ad 100644 --- a/padiff/cinn_diff/run.py +++ b/padiff/cinn_diff/run.py @@ -12,17 +12,36 @@ # See the License for the specific language governing permissions and # limitations under the License. -from analyze import auto_diff -from env import Env +from padiff import cinn_diff +import json def run(run_script, base_env, cinn_env): - run_env = Env(run_script, base_env, cinn_env) + run_env = cinn_diff.Env(run_script, base_env, cinn_env) run_env.run_base_model() # 可以提供选项选择不运行base model run_env.run_cinn_model() - auto_diff(run_env.base_path, run_env.cinn_path, rtol=0, atol=0) + ret = cinn_diff.auto_diff(run_env.base_path, run_env.cinn_path, rtol=0, atol=0) + with open("./cmp_ret.json", "w") as jsonf: + json.dump(ret, jsonf, indent=4) if __name__ == "__main__": + _base_env = { + "CUDA_VISIBLE_DEVICES": "7", + "NVIDIA_TF32_OVERRIDE": "1", + "CUDA_LAUNCH_BLOCKING": "1", + "FLAGS_cudnn_deterministc": "1", + "FLAGS_cinn_cudnn_deterministc": "1", + "FLAGS_prim_all": "true", + } + _cinn_env = { + "FLAGS_use_cinn": "1", + "FLAGS_deny_cinn_ops": "reduce_sum", + "FLAGS_use_reduce_split_pass": "1", + "FLAGS_nvrtc_compile_to_cubin": "0", + "FLAGS_cinn_use_op_fusion": "1", + "FLAGS_cinn_parallel_compile_size": "8", + "FLAGS_cinn_pass_visualize_dir": "", + } run_script = "/root/dev/PaddleNLP/model_zoo/bert/run_bert.sh" - run(run_script, None, None) + run(run_script, _base_env, _cinn_env) From 0ebfa25884a24984a599634347965e96d9a2fbb6 Mon Sep 17 00:00:00 2001 From: GGBond8488 <857631483@qq.com> Date: Wed, 17 Jan 2024 16:36:12 +0800 Subject: [PATCH 05/10] opt env --- padiff/cinn_diff/env.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/padiff/cinn_diff/env.py b/padiff/cinn_diff/env.py index 932651f..86eb9c2 100644 --- a/padiff/cinn_diff/env.py +++ b/padiff/cinn_diff/env.py @@ -101,7 +101,10 @@ def run_cinn_model(self): self.init_cinn_env() os.chdir(self.script_path) run_env = self.cinn_env.copy() - run_env.update(self.base_env) + base_env = self.base_env.copy() + for key in base_env: + if key not in run_env: + run_env[key] = base_env[key] logger.info(run_env) run_env.update(self.os_env) cinn_log = open("cinn.log", "w") From 36a6b3bd723a4e4f83c0bef21b17fb348e69c6a2 Mon Sep 17 00:00:00 2001 From: GGBond8488 <857631483@qq.com> Date: Wed, 17 Jan 2024 17:02:23 +0800 Subject: [PATCH 06/10] opt env --- padiff/cinn_diff/env.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/padiff/cinn_diff/env.py b/padiff/cinn_diff/env.py index 86eb9c2..dcf93c8 100644 --- a/padiff/cinn_diff/env.py +++ b/padiff/cinn_diff/env.py @@ -89,6 +89,7 @@ def run_model(self, run_env, log): def run_base_model(self): self.init_base_env() + root_path = os.getcwd() os.chdir(self.script_path) run_env = self.base_env.copy() logger.info(run_env) @@ -96,9 +97,11 @@ def run_base_model(self): base_log = open("base.log", "w") self.run_model(run_env, base_log) base_log.close() + os.chdir(root_path) def run_cinn_model(self): self.init_cinn_env() + root_path = os.getcwd() os.chdir(self.script_path) run_env = self.cinn_env.copy() base_env = self.base_env.copy() @@ -110,3 +113,4 @@ def run_cinn_model(self): cinn_log = open("cinn.log", "w") self.run_model(run_env, cinn_log) cinn_log.close() + os.chdir(root_path) From c6860b04e8a4df3f82f72bc089af2dd618a52419 Mon Sep 17 00:00:00 2001 From: GGBond8488 <857631483@qq.com> Date: Wed, 17 Jan 2024 17:03:55 +0800 Subject: [PATCH 07/10] opt log --- padiff/cinn_diff/logs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/padiff/cinn_diff/logs.py b/padiff/cinn_diff/logs.py index 51f1bb9..956c0f4 100644 --- a/padiff/cinn_diff/logs.py +++ b/padiff/cinn_diff/logs.py @@ -49,7 +49,7 @@ def loggerImp(self): return self.logger -logger = Logger(os.path.join(sys.path[0], "cinn_diff_log.log")).loggerImp() +logger = Logger(os.path.join(sys.path[0], "cinn_diff.log")).loggerImp() def log_init(file_name): From 73666e73b4d9e37033588e39815daa80b7339a3f Mon Sep 17 00:00:00 2001 From: GGBond8488 <857631483@qq.com> Date: Wed, 17 Jan 2024 21:02:27 +0800 Subject: [PATCH 08/10] remove Chinese comments --- padiff/cinn_diff/README.md | 4 ++-- padiff/cinn_diff/analyze.py | 16 ++++++++-------- padiff/cinn_diff/compare_utils.py | 1 + padiff/cinn_diff/graph.py | 8 ++++---- padiff/cinn_diff/logs.py | 23 ++++++++--------------- padiff/cinn_diff/read_file.py | 15 +++++---------- padiff/cinn_diff/run.py | 2 +- 7 files changed, 29 insertions(+), 40 deletions(-) diff --git a/padiff/cinn_diff/README.md b/padiff/cinn_diff/README.md index 857838c..576b3a7 100644 --- a/padiff/cinn_diff/README.md +++ b/padiff/cinn_diff/README.md @@ -14,8 +14,8 @@ from padiff import cinn_diff def run(run_script, base_env, cinn_env): run_env = cinn_diff.Env(run_script, base_env, cinn_env) - run_env.run_base_model() #可以注释掉选择不运行base model - run_env.run_cinn_model() #也可以注释掉选择不运行cinn model + run_env.run_base_model() + run_env.run_cinn_model() cinn_diff.auto_diff(run_env.base_path, run_env.cinn_path, rtol=1e-3, atol=1e-3) diff --git a/padiff/cinn_diff/analyze.py b/padiff/cinn_diff/analyze.py index af0887b..857af6c 100644 --- a/padiff/cinn_diff/analyze.py +++ b/padiff/cinn_diff/analyze.py @@ -16,7 +16,7 @@ from .compare_utils import Comparator from .logs import logger -# 需要优化 +# Needs optimization in the future def back_track_group(base, compare, cluster, cmp, graph, node): inputs = graph.inputs() all_inputs_equal = True @@ -62,9 +62,8 @@ def auto_diff(base_path, compare_path, rtol=1e-6, atol=1e-6): compare = read_all(compare_path) cmp = Comparator(rtol=rtol, atol=atol) - # step1: 确认cluster的输入输出是否对齐 + # step1: Confirm whether the input and output of the cluster are aligned for cluster in compare.all_clusters: - # logger.info(cluster.idx) input_equals_flag = True output_equals_flag = True for input in cluster.inputs: @@ -77,27 +76,28 @@ def auto_diff(base_path, compare_path, rtol=1e-6, atol=1e-6): continue if input_equals_flag: - # step2: 找到cluster内部对不齐的点 + # step2: Find the misaligned output of the cluster for output in cluster.outputs: base_var_path = base.all_vars_paths[output] compare_var_path = compare.all_vars_paths[output] ret = cmp.allclose(base_var_path, compare_var_path) if not ret: output_equals_flag = False - # step3: 找到对不齐变量对应的group + # step3: Find the group corresponding to the misaligned output output_cinn_var = cluster.varmaps.get(output, "") if not output_cinn_var: logger.info("can't find var " + output + " corresponding cinn var name") else: find_diff_group_flag = False - # step4 : 从对不齐的输出出发，找到第一次出现输出对不齐的group（输入能对齐，输出无法对齐） + # step4 : Starting from the misaligned output, find the group where the output misalignment + # occurs for the first time (the input can be aligned, but the output cannot be aligned) group = cluster.cinn_group for graph in group.subgraphs: node = graph.find(output_cinn_var) if node and not graph.is_input(node): - # 找到对不齐的第一个输出，开始回溯 + # Find the first misaligned output and start backtracking diff_ret = back_track_group(base, compare, cluster, cmp, graph, node) - if diff_ret: # 输入能对齐，输出无法对齐 + if diff_ret: # Input can be aligned, but output cannot be aligned diff_ret["output"] = output cmp.record_group_output_diff(diff_ret) find_diff_group_flag = True diff --git a/padiff/cinn_diff/compare_utils.py b/padiff/cinn_diff/compare_utils.py index 8d11233..0ab4647 100644 --- a/padiff/cinn_diff/compare_utils.py +++ b/padiff/cinn_diff/compare_utils.py @@ -76,6 +76,7 @@ def record_group_output_diff(self, diff_ret): if __name__ == "__main__": + # test code， intermediate variables can be read this way cmp = Comparator() base = cmp.load_var("/root/dev/PaddleClas/base/saved_tensors/batch_norm_grad-input-batch_norm_0.tmp_3@GRAD") cinn = cmp.load_var("/root/dev/PaddleClas/cinn/saved_tensors/batch_norm_grad-input-batch_norm_0.tmp_3@GRAD") diff --git a/padiff/cinn_diff/graph.py b/padiff/cinn_diff/graph.py index 630f2e8..4272244 100644 --- a/padiff/cinn_diff/graph.py +++ b/padiff/cinn_diff/graph.py @@ -49,9 +49,9 @@ def construct_graph_by_dot(dot_path, sep="\\n"): start, end = edge if start not in idx2nodes.keys() or end not in idx2nodes.keys(): continue - # 输出边 + # Output edge idx2nodes[start].add_output(idx2nodes[end]) - # 输入边 + # Input edge idx2nodes[end].add_input(idx2nodes[start]) return all_nodes, ret_subgraphs @@ -87,7 +87,7 @@ def inputs(self): inputs.append(node.outputs[0]) if not node.inputs: inputs.append(node) - # 输入在另一个子图中，也算作当前子图的输入 + # Inputs in another subgraph are also counted as inputs to the current subgraph. for node in node.inputs: if node not in self.nodes: inputs.append(node) @@ -110,7 +110,7 @@ def outputs(self): outputs.append(node.inputs[0]) if not node.outputs: outputs.append(node) - # 输出在另一个子图中，也算作当前子图的输出 + # The output is in another subgraph and is also counted as the output of the current subgraph. for node in node.outputs: if node not in self.nodes: outputs.append(node) diff --git a/padiff/cinn_diff/logs.py b/padiff/cinn_diff/logs.py index 956c0f4..4ea896d 100644 --- a/padiff/cinn_diff/logs.py +++ b/padiff/cinn_diff/logs.py @@ -23,26 +23,19 @@ def __init__( self, filename, level=logging.INFO, when="D", backCount=3, fmt="%(asctime)s" "-%(levelname)s: %(message)s" ): - self.logger = logging.getLogger(filename) # 根据文件名创建一个日志 - self.logger.setLevel(level) # 设置默认日志级别 - self.format_str = logging.Formatter(fmt) # 设置日志格式 + self.logger = logging.getLogger(filename) + self.logger.setLevel(level) + self.format_str = logging.Formatter(fmt) - # screen_handler = logging.StreamHandler() # 屏幕输出处理器 - # screen_handler.setFormatter(self.format_str) # 设置屏幕输出显示格式 - - # 定时写入文件处理器 time_file_handler = handlers.TimedRotatingFileHandler( - filename=filename, # 日志文件名 - when=when, # 多久创建一个新文件 - interval=1, # 写入时间间隔 - backupCount=backCount, # 备份文件的个数 + filename=filename, + when=when, + interval=1, + backupCount=backCount, encoding="utf-8", - ) # 编码格式 + ) time_file_handler.setFormatter(self.format_str) - - # 添加日志处理器 - # self.logger.addHandler(screen_handler) self.logger.addHandler(time_file_handler) def loggerImp(self): diff --git a/padiff/cinn_diff/read_file.py b/padiff/cinn_diff/read_file.py index fd086ba..79eac01 100644 --- a/padiff/cinn_diff/read_file.py +++ b/padiff/cinn_diff/read_file.py @@ -14,10 +14,11 @@ import os +import collections + from .graph import Graph, Node, Cluster, Group, Pass, construct_graph_by_dot from .logs import logger -import collections INPUTS_NAME = "cluster_inputs.txt" OUTPUTS_NAME = "cluster_outputs.txt" @@ -139,18 +140,14 @@ def read_cinn_pass(path): def read_graphviz_dot(path): passes = os.listdir(path) idx = path.split("_")[-1] - # logger.info("group idx: " + str(idx)) all_passes = {} for pass_path in passes: - # logger.info("pass_path: " + pass_path) - # logger.info(pass_path.split("_")) pass_idx = int(pass_path.split("_")[1]) - # logger.info("pass_idx: " + str(pass_idx)) if pass_idx not in all_passes: all_passes[pass_idx] = Pass(pass_idx) pass_name = pass_path.split("_")[2] all_passes[pass_idx].set_pass_name(pass_name) - type = pass_path.split("_")[3] # after.txt + type = pass_path.split("_")[3] record_path = os.path.join(path, pass_path) if type == "after.txt": all_passes[pass_idx].set_after_txt(record_path) @@ -163,7 +160,6 @@ def read_graphviz_dot(path): else: raise ValueError(type + "not support") max_pass_id = max(all_passes.keys()) - # logger.info("lass_pass_id: " + str(max_pass_id)) group_cc = Group(idx, all_passes, max_pass_id) all_groups[idx] = group_cc @@ -217,8 +213,7 @@ def set_clusters_group(clusters, groups, cinn_graphs): graph_outputs = graph.graph_outputs() if not graph_inputs and not graph_outputs: raise ValueError(f"{graph} does not have inputs or outputs") - # logger.info(graph_inputs) - # logger.info(inputs) + if not set(inputs).difference(graph_inputs) and not set(outputs).difference(graph_outputs): logger.info(f"group_{idx} belongs to Cluster_{cluster.idx}") cluster.cinn_group = groups[idx] @@ -233,7 +228,7 @@ def read_all(root_path="", type="cinn"): all_vars_paths = {} all_cinn_groups = {} all_cinn_graphs = {} - # exsist some bug + cinn2paddle_varmaps = {} allinone = collections.namedtuple( diff --git a/padiff/cinn_diff/run.py b/padiff/cinn_diff/run.py index 09bd3ad..30cd2cb 100644 --- a/padiff/cinn_diff/run.py +++ b/padiff/cinn_diff/run.py @@ -18,7 +18,7 @@ def run(run_script, base_env, cinn_env): run_env = cinn_diff.Env(run_script, base_env, cinn_env) - run_env.run_base_model() # 可以提供选项选择不运行base model + run_env.run_base_model() run_env.run_cinn_model() ret = cinn_diff.auto_diff(run_env.base_path, run_env.cinn_path, rtol=0, atol=0) with open("./cmp_ret.json", "w") as jsonf: From d63d5ed94da7a6eb569c868e83dc976c69eff10a Mon Sep 17 00:00:00 2001 From: GGBond8488 <857631483@qq.com> Date: Wed, 17 Jan 2024 21:05:17 +0800 Subject: [PATCH 09/10] remove Chinese comments --- padiff/cinn_diff/analyze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/padiff/cinn_diff/analyze.py b/padiff/cinn_diff/analyze.py index 857af6c..f29cc8a 100644 --- a/padiff/cinn_diff/analyze.py +++ b/padiff/cinn_diff/analyze.py @@ -16,7 +16,7 @@ from .compare_utils import Comparator from .logs import logger -# Needs optimization in the future +# TODO(GGBond8488): Needs optimization in the future def back_track_group(base, compare, cluster, cmp, graph, node): inputs = graph.inputs() all_inputs_equal = True From 7a370c623e6b58746440d0eae30bd8079becfea0 Mon Sep 17 00:00:00 2001 From: GGBond8488 <857631483@qq.com> Date: Wed, 17 Jan 2024 21:07:10 +0800 Subject: [PATCH 10/10] remove Chinese comments --- padiff/cinn_diff/analyze.py | 1 + 1 file changed, 1 insertion(+) diff --git a/padiff/cinn_diff/analyze.py b/padiff/cinn_diff/analyze.py index f29cc8a..513223f 100644 --- a/padiff/cinn_diff/analyze.py +++ b/padiff/cinn_diff/analyze.py @@ -116,6 +116,7 @@ def auto_diff(base_path, compare_path, rtol=1e-6, atol=1e-6): if __name__ == "__main__": + # test code, you can simple use cinn_diff.auto_diff this way base_path = "/root/dev/PaddleClas/base" compare_path = "/root/dev/PaddleClas/cinn" auto_diff(base_path, compare_path, atol=0, rtol=0)