From 39dea7f9e8a9b96e95b4f48fab36735d3d3be638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=95=86=E5=85=83=E4=B9=89?= Date: Mon, 17 Jun 2024 06:04:05 +0000 Subject: [PATCH] =?UTF-8?q?!1331=20=E6=B7=BB=E5=8A=A0Qwen1.5-0.5B=E9=80=82?= =?UTF-8?q?=E9=85=8D=20Merge=20pull=20request=20!1331=20from=20=E5=95=86?= =?UTF-8?q?=E5=85=83=E4=B9=89/master?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 19 +- README_en.md | 19 +- examples/qwen15/README.md | 277 +++++++++++++++++- examples/qwen15/README_en.md | 275 ++++++++++++++++- .../qwen15/evaluate_qwen15_0point5b_ptd.sh | 67 +++++ .../qwen15/generate_qwen15_0point5b_ptd.sh | 63 ++++ .../qwen15/pretrain_qwen15_0point5b_ptd.sh | 101 +++++++ .../qwen15/qwen15_0point5b_inference.png | Bin 0 -> 20563 bytes 8 files changed, 793 insertions(+), 28 deletions(-) create mode 100644 examples/qwen15/evaluate_qwen15_0point5b_ptd.sh create mode 100644 examples/qwen15/generate_qwen15_0point5b_ptd.sh create mode 100644 examples/qwen15/pretrain_qwen15_0point5b_ptd.sh create mode 100644 sources/images/qwen15/qwen15_0point5b_inference.png diff --git a/README.md b/README.md index 1ffca5d..c0e602b 100644 --- a/README.md +++ b/README.md @@ -317,7 +317,16 @@ ModelLink旨在为华为 [昇腾芯片](https://open.codehub.huawei.com/OpenBaiz 【昇腾贡献】 - Qwen1.5 + Qwen1.5 + 0.5B + pretrain + generate + -- + -- + -- + eval + 【社区贡献】 + 1.8B pretrain generate @@ -675,7 +684,13 @@ ModelLink旨在为华为 [昇腾芯片](https://open.codehub.huawei.com/OpenBaiz - Qwen1.5 + Qwen1.5 + 0.5B + 1x8 + BF16 + 22834 + 25306 + 1.8B 1x8 BF16 diff --git a/README_en.md b/README_en.md index 9b7e05a..64eb5ac 100644 --- a/README_en.md +++ b/README_en.md @@ -316,7 +316,16 @@ Current ModelLink supports pre-training and fine-tuning for the following models 【Ascend】 - Qwen1.5 + Qwen1.5 + 0.5B + pretrain + generate + -- + -- + -- + eval + 【Community】 + 1.8B pretrain generate @@ -671,7 +680,13 @@ For the supported models listed above, we provide training scripts and readme in 345 - Qwen1.5 + Qwen1.5 + 0.5B + 1x8 + BF16 + 22834 + 25306 + 1.8B 1x8 BF16 diff --git a/examples/qwen15/README.md b/examples/qwen15/README.md index 68422fa..4b032b9 100644 --- a/examples/qwen15/README.md +++ b/examples/qwen15/README.md @@ -6,6 +6,13 @@

# 目录 +- [Qwen1.5-0.5B](#Qwen1.5-0.5b) + - [训练-0.5B](#训练-0.5b) + - [脚本-0.5B](#脚本-0.5b) + - [性能-0.5B](#性能-0.5b) + - [吞吐-0.5B](#吞吐-0.5b) + - [推理-0.5B](#推理-0.5b) + - [评估-0.5B](#评估-0.5b) - [Qwen1.5-1.8B](#Qwen1.5-1.8b) - [训练-1.8B](#训练-1.8b) - [脚本-1.8B](#脚本-1.8b) @@ -48,6 +55,252 @@ - [吞吐-72B](#吞吐-72b) - [推理-72B](#推理-72b) - [评估-72B](#评估-72b) +# Qwen1.5-0.5B + +## 训练-0.5B +Qwen1.5-0.5B 训练的硬件配置: + +| 硬件 | 配置 | +|:---:|:---------------:| +| NPU | 8 x Ascend NPUs | +### 脚本-0.5B + +1. 克隆仓库到本地服务器 + + ```shell + git clone https://gitee.com/ascend/ModelLink.git + git clone https://github.com/NVIDIA/Megatron-LM.git + cd Megatron-LM + git checkout core_r0.6.0 + cp -r megatron ../ModelLink/ + cd .. + cd ModelLink + mkdir logs + mkdir model_from_hf + mkdir dataset + mkdir ckpt + ``` +2. 搭建环境 + + ```bash + # python3.8 + conda create -n test python=3.8 + conda activate test + + # 安装 torch 和 torch_npu + pip install torch-2.2.0-cp38-cp38m-manylinux2014_aarch64.whl + pip install torch_npu-2.2.0*-cp38-cp38m-linux_aarch64.whl + pip install apex-0.1_ascend*-cp38-cp38m-linux_aarch64.whl + + # 修改 ascend-toolkit 路径 + source /usr/local/Ascend/ascend-toolkit/set_env.sh + + # 安装加速库 + git clone https://gitee.com/ascend/MindSpeed.git + cd MindSpeed + git checkout 2b0edd2 + pip install -r requirements.txt + pip3 install -e . + cd .. + + # 安装其余依赖库 + pip install -r requirements.txt + ``` +3. 下载 Qwen1.5-0.5B 的 [预训练权重和词表](https://huggingface.co/Qwen/Qwen1.5-0.5B/tree/main) + + ```shell + #!/bin/bash + mkdir ./model_from_hf/qwen15-0.5b-hf/ + cd ./model_from_hf/qwen15-0.5b-hf/ + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/config.json + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/generation_config.json + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/merges.txt + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/model.safetensors + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/tokenizer.json + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/tokenizer_config.json + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/vocab.json + cd ../../ + ``` +4. 权重转换 + + 4.1 将权重从 huggingface 格式转化为 magatron 格式 + ***（该场景一般用于使能开源的HuggingFace模型在Megatron上进行训练）*** + + ```bash + # 修改 ascend-toolkit 路径 + source /usr/local/Ascend/ascend-toolkit/set_env.sh + + # 权重格式转换 + python tools/checkpoint/convert_ckpt.py \ + --model-type GPT \ + --loader llama2_hf \ + --saver megatron \ + --target-tensor-parallel-size 1 \ + --target-pipeline-parallel-size 1 \ + --add-qkv-bias \ + --load-dir ./model_from_hf/qwen15-0.5b-hf/ \ + --save-dir ./model_weights/qwen15-0.5b-hf-v0.1-tp1-pp1/ \ + --tokenizer-model ./model_from_hf/qwen15-0.5b-hf/tokenizer.json + ``` + + 4.2 任意并行切分策略的 Megatron 权重 格式转化为 HuggingFace权重 + ***（该场景一般用于将训练好的megatron模型重新转回HuggingFace格式）*** + + ```shell + # 请按照您的真实环境修改 set_env.sh 路径 + source /usr/local/Ascend/ascend-toolkit/set_env.sh + python tools/checkpoint/convert_ckpt.py \ + --model-type GPT \ + --loader megatron \ + --saver megatron \ + --save-model-type save_huggingface_llama \ + --load-dir ./ckpt/ \ + --target-tensor-parallel-size 1 \ + --target-pipeline-parallel-size 1 \ + --add-qkv-bias \ + --save-dir ./model_from_hf/qwen15-0.5b-hf/ # <-- 需要填入原始HF模型路径，新权重会存于./model_from_hf/qwen15-0.5b-hf/mg2hg/ + ``` + + 权重转换适用于预训练、微调、推理和评估，根据任务不同调整参数 `target-tensor-parallel-size`和 `target-pipeline-parallel-size`。 +5. 预训练 + + 5.1 准备数据集 + + 下载 Qwen1.5-0.5B [数据集](https://huggingface.co/datasets/tatsu-lab/alpaca/resolve/main/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet) + + ```shell + # 下载数据 + cd ./dataset + wget https://huggingface.co/datasets/tatsu-lab/alpaca/resolve/main/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet + cd .. + # 处理数据 + mkdir ./dataset/qwen15-0.5b-hf/ + python ./tools/preprocess_data.py \ + --input ./dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet \ + --tokenizer-name-or-path ./model_from_hf/qwen15-0.5b-hf/ \ + --output-prefix ./dataset/qwen15-0.5b-hf/alpaca \ + --workers 4 \ + --log-interval 1000 \ + --tokenizer-type PretrainedFromHF + ``` + + 5.2 预训练 + + ```shell + # 设置 ascend-toolkit 路径 + source /usr/local/Ascend/ascend-toolkit/set_env.sh + + # 根据实际情况配置词表、数据集、模型参数保存路径 + CKPT_SAVE_DIR="./ckpt/qwen15-0.5b-hf/" + TOKENIZER_PATH="./model_from_hf/qwen15-0.5b-hf" #词表路径 + DATA_PATH="./dataset/qwen15-0.5b-hf/alpaca_text_document" #数据集路径 + CKPT_LOAD_DIR="./model_weights/qwen15-0.5b-hf-v0.1-tp1-pp1" + ``` + 启动 Qwen1.5-0.5B 预训练脚本: examples/qwen15/pretrain_qwen15_0point5b_ptd.sh + +6. 微调 + + 6.1 准备微调数据集 + 下载微调数据集 [这里](https://huggingface.co/datasets/tatsu-lab/alpaca/resolve/main/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet) + + ```shell + # 下载数据集 + mkdir finetune_dataset + cd ./finetune_dataset + wget https://huggingface.co/datasets/tatsu-lab/alpaca/resolve/main/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet + cd .. + + # 处理微调数据集 + mkdir ./finetune_dataset/qwen15-0.5b-hf/ + python ./tools/preprocess_data.py \ + --input ./dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet \ + --tokenizer-name-or-path ./model_from_hf/qwen15-0.5b-hf/ \ + --output-prefix ./finetune_dataset/qwen15-0.5b-hf/alpaca \ + --workers 4 \ + --log-interval 1000 \ + --tokenizer-type PretrainedFromHF \ + --handler-name GeneralInstructionHandler \ + --append-eod + ``` + + 6.2 全参微调 + 全参微调的配置脚本基本和预训练脚本一致. *区别是数据集，以及增加训练参数--is-instruction-dataset* + + 增加微调参数--finetune，参数如下： + + ```bash + DATA_PATH="./finetune_dataset/qwen15-0.5b-hf/alpaca" + TOKENIZER_PATH="./model_from_hf/qwen15-0.5b-hf/" + CKPT_PATH="./model_weights/qwen15-0.5b-hf-v0.1-tp1-pp1/" + --load ${CKPT_PATH} \ + --finetune \ + --is-instruction-dataset \ + --tokenizer-type PretrainedFromHF \ + --tokenizer-name-or-path ${TOKENIZER_PATH} \ + --tokenizer-not-use-fast \ + ``` +### 性能-0.5B + +#### 吞吐-0.5B + +Qwen1.5-0.5B 在 **昇腾芯片** 和 **参考芯片** 上的性能对比： + +| 设备 | 模型 | 迭代数 | tokens吞吐 (tokens/s/p) | +| :--: |:------------:|:----:|:---------------------:| +| NPUs | Qwen1.5-0.5B | 2000 | 22834 | +| 参考 | Qwen1.5-0.5B | 2000 | 25306 | + +## 推理-0.5B + +配置Qwen1.5-0.5B 推理脚本: examples/qwen15/generate_qwen1.5_0point5b_ptd.sh + +```bash +# 根据您自己的 ascend-toolkit 路径，执行set_env.sh +source /usr/local/Ascend/ascend-toolkit/set_env.sh + +# 修改模型权重路径和词表路径 +CHECKPOINT="./model_weights/qwen15-0.5b-hf-v0.1-tp1-pp1" +TOKENIZER_PATH="./model_from_hf/qwen15-0.5b-hf/" +``` + +启动Qwen1.5-0.5B 推理脚本 + +```bash +bash examples/qwen15/generate_qwen15_0point5b_ptd.sh +``` + +推理示例如下： + +![Inference](../../sources/images/qwen15/qwen15_0point5b_inference.png) + +## 评估-0.5B + +使用 MMLU数据集评估模型. 数据集下载路径 [这里](https://huggingface.co/datasets/cais/mmlu). +配置Qwen1.5-0.5B 评估脚本: examples/qwen15/evaluate_qwen1.5_0point5b_ptd.sh + +```bash +# ascend-toolkit 路径 +source /usr/local/Ascend/ascend-toolkit/set_env.sh + +# 修改模型参数路径和词表路径 +TOKENIZER_PATH="./model_from_hf/qwen15-0.5b-hf/" #词表路径 +CHECKPOINT="./model_weights/qwen15-0.5b-hf-v0.1-tp1-pp1" #模型路径 +# 配置任务和数据集路径 +DATA_PATH="./mmlu/data/test/" +TASK="mmlu" +``` + +启动评估 + +```bash +bash examples/qwen15/evaluate_qwen15_0point5b_ptd.sh +``` + +评估结果如下 + +| 数据集 | 总学科数 | 总问题数 | 参考准确率 | NPU准确率 | +| :----: | :------: | :------: |:-----:|:------:| +| MMLU | 57 | 14042 | 0.318 | 0.318 | # Qwen1.5-1.8B ## 训练-1.8B @@ -260,7 +513,7 @@ TOKENIZER_PATH="./model_from_hf/qwen15-1.8b-hf/" 启动Qwen1.5-1.8B 推理脚本 ```bash -bash examples/qwen15/generate_qwen15_1.8b_ptd.sh +bash examples/qwen15/generate_qwen15_1point8b_ptd.sh ``` 推理示例如下： @@ -498,7 +751,7 @@ Qwen1.5-4B 在 **昇腾芯片** 和 **参考芯片** 上的性能对比： ## 推理-4B -配置Qwen1.5-4B 推理脚本: examples/qwen1.5/generate_qwen1.5_4b_ptd.sh +配置Qwen1.5-4B 推理脚本: examples/qwen15/generate_qwen15_4b_ptd.sh ```bash # 根据您自己的 ascend-toolkit 路径，执行set_env.sh @@ -522,7 +775,7 @@ bash examples/qwen15/generate_qwen15_4b_ptd.sh ## 评估-4B 使用 MMLU数据集评估模型. 数据集下载路径 [这里](https://huggingface.co/datasets/cais/mmlu). -配置Qwen1.5-4B 评估脚本: examples/qwen1.5/evaluate_qwen1.5_4b_ptd.sh +配置Qwen1.5-4B 评估脚本: examples/qwen15/evaluate_qwen15_4b_ptd.sh ```bash # ascend-toolkit 路径 @@ -809,9 +1062,9 @@ TASK="mmlu" # ceval任务配置为 "ceval" bash examples/qwen15/evaluate_qwen15_7b_ptd.sh ``` -| 数据集 | 总学科数 | 总问题数 | 参考准确率 | NPU准确率 | -|:-----:|:----:|:-----:|:-------------------------------------------------------:|:------:| -| MMLU | 57 | 14042 | [61.0](https://qwenlm.github.io/zh/blog/qwen1.5) | 60.3 | +| 数据集 | 总学科数 | 总问题数 | 参考准确率 | NPU准确率 | +|:-----:|:----:|:-----:|:----------------------------------------------------:|:------:| +| MMLU | 57 | 14042 | [61.0](https://qwenlm.github.io/zh/blog/qwen1.5) | 0.603 | # Qwen1.5-14B @@ -1225,8 +1478,8 @@ bash examples/qwen15/evaluate_qwen15_14b_ptd.sh # 根据实际情况配置词表、数据集、模型参数保存路径 CKPT_SAVE_DIR="./ckpt/Qwen1.5-32B/" TOKENIZER_PATH="./model_from_hf/Qwen1.5-32B/" #词表路径 - DATA_PATH="./dataset/Qwen1.5-32B-hf/alpaca_text_document" #数据集路径 - CKPT_LOAD_DIR="./model_weights/Qwen1.5-32B-v0.1-tp8-pp4-vpp2/" + DATA_PATH="./dataset/qwen1.5-32B-hf/alpaca_text_document" #数据集路径 + CKPT_LOAD_DIR="./model_weights/qwen1.5-32B-v0.1-tp8-pp4-vpp2/" ``` 启动 Qwen1.5-32B 预训练脚本: examples/qwen15/pretrain_qwen15_32b_ptd.sh @@ -1250,7 +1503,7 @@ bash examples/qwen15/evaluate_qwen15_14b_ptd.sh # 处理微调数据集 mkdir ./finetune_dataset/qwen-1.5-32b-hf/ python ./tools/preprocess_data.py \ - --input ./dataset/ train-00000-of-00001-a09b74b3ef9c3b56.parquet \ + --input ./finetune_dataset/ train-00000-of-00001-a09b74b3ef9c3b56.parquet \ --tokenizer-name-or-path ./model_from_hf/Qwen1.5-32B/ \ --output-prefix ./finetune_dataset/qwen-1.5-32b-hf/alpaca \ --workers 4 \ @@ -1340,7 +1593,7 @@ bash ./examples/qwen15/generate_qwen15_32b_lora_chat_ptd.sh ``` lora微调后的推理效果如下： -![Inference](../../sources/images/qwen15/qwen15_32b_lora_inference.png) +![Inference](../../../../qwen1.5/ModelLink/sources/images/qwen15/qwen15_32b_lora_inference.png) ## 评估-32B @@ -1589,7 +1842,7 @@ Qwen1.5-72B 在 **昇腾芯片** 和 **参考芯片** 上的性能对比： ## 推理-72B -配置 qwen1.5-72b 推理脚本：examples/qwen15/generate_qwen15_72b_ptd.sh +配置 qwen1.5-72b 推理脚本：examples/qwen15/generate_qwen15_72b_chat_ptd.sh ```bash # ascend-toolkit 路径 @@ -1623,7 +1876,7 @@ bash ./examples/qwen15/generate_qwen15_72b_lora_chat_ptd.sh ``` lora微调后的推理效果如下： -![Inference](../../sources/images/qwen15/qwen15_72b_lora_inference.png) +![Inference](../../../../qwen1.5/ModelLink/sources/images/qwen15/qwen15_72b_lora_inference.png) ## 评估-72B diff --git a/examples/qwen15/README_en.md b/examples/qwen15/README_en.md index deb80b5..d17824e 100644 --- a/examples/qwen15/README_en.md +++ b/examples/qwen15/README_en.md @@ -5,6 +5,13 @@

# Contents +- [Qwen1.5-0.5B](#Qwen1.5-0.5b) + - [Training-0.5B](#training-0.5b) + - [Script-0.5B](#script-0.5b) + - [Performance-0.5B](#performance-0.5b) + - [Machine performance 0.5B](#machine-performance-0.5b) + - [Inference-0.5B](#inference-0.5b) + - [Evaluation-0.5B](#evaluation-0.5b) - [Qwen1.5-1.8B](#Qwen1.5-1.8b) - [Training-1.8B](#training-1.8b) - [Script-1.8B](#script-1.8b) @@ -47,6 +54,250 @@ - [Machine performance 72B](#machine-performance-72b) - [Inference-72B](#Inference-72b) - [Evaluation-72B](#Evaluation-72b) +# Qwen1.5-0.5B + +## Training-0.5B +Here's a hardware summary of pre-training Qwen-0.5B: + +| Hardware | Value | +| :------: | :---------------------------------------------: | +| NPU | 8 x Ascend NPUs | +### Script + +1. Clone the repository to your local server + + ```shell + git clone https://gitee.com/ascend/ModelLink.git + git clone https://github.com/NVIDIA/Megatron-LM.git + cd Megatron-LM + git checkout core_r0.6.0 + cp -r megatron ../ModelLink/ + cd .. + cd ModelLink + mkdir logs + mkdir model_from_hf + mkdir dataset + mkdir ckpt + ``` +2. Build environment + + ```bash + # python3.8 + conda create -n test python=3.8 + conda activate test + + # install torch 和 torch_npu + pip install torch-2.2.0-cp38-cp38m-manylinux2014_aarch64.whl + pip install torch_npu-2.2.0*-cp38-cp38m-linux_aarch64.whl + pip install apex-0.1_ascend*-cp38-cp38m-linux_aarch64.whl + + # modify ascend-toolkit path + source /usr/local/Ascend/ascend-toolkit/set_env.sh + + # install MindSpeed + git clone https://gitee.com/ascend/MindSpeed.git + cd MindSpeed + git checkout 2b0edd2 + pip install -r requirements.txt + pip3 install -e . + cd .. + + # install other packages + pip install -r requirements.txt + ``` +3. Download Qwen1.5-0.5B [pretrained weights and tokenizer](https://huggingface.co/Qwen/Qwen1.5-1.8B/tree/main) + + ```shell + #!/bin/bash + mkdir ./model_from_hf/qwen15-0.5b-hf/ + cd ./model_from_hf/qwen15-4b-hf/ + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/config.json + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/generation_config.json + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/merges.txt + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/model.safetensors + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/tokenizer.json + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/tokenizer_config.json + wget https://huggingface.co/Qwen/Qwen1.5-0.5B/resolve/main/vocab.json + cd ../../ + ``` +4. weight conversion in ptd mode + + 4.1 Convert weights from huggingface format to megatron format + ***(This scenario is generally used to train open-source HuggingFace models on Megatron)*** + + ```bash + # modify the script according to your own ascend-toolkit path + source /usr/local/Ascend/ascend-toolkit/set_env.sh + + # convert to ptd weights + python tools/checkpoint/convert_ckpt.py \ + --model-type GPT \ + --loader llama2_hf \ + --saver megatron \ + --target-tensor-parallel-size 1 \ + --target-pipeline-parallel-size 1 \ + --params-dtype bf16 \ + --add-qkv-bias \ + --load-dir ./model_from_hf/qwen15-0.5b-hf/ \ + --save-dir ./model_weights/qwen15-0.5b-hf-v0.1-tp1-pp1/ \ + --tokenizer-model ./model_from_hf/qwen15-0.5b-hf/tokenizer.json + ``` + + 4.2 Any Megatron weights with parallel slicing strategy --> Any Megatron weights with parallel slicing strategy + ***(This scenario is generally used to convert the trained megatron model back to the HuggingFace format)*** + + ```shell + # Modify the ascend-toolkit path + source /usr/local/Ascend/ascend-toolkit/set_env.sh + python tools/checkpoint/convert_ckpt.py \ + --model-type GPT \ + --loader megatron \ + --saver megatron \ + --save-model-type save_huggingface_llama \ + --load-dir ./ckpt/ \ + --target-tensor-parallel-size 1 \ + --target-pipeline-parallel-size 1 \ + --add-qkv-bias \ + --save-dir ./model_from_hf/qwen15-0.5b-hf/ # <-- Fill in the original HF model path here, new weights will be saved in ./model_from_hf/qwen15-0.5b-hf/mg2hg/ + ``` + +5. pre-training + + 5.1 prepare dataset + + Download the Qwen1.5-0.5B datasets from [here](https://huggingface.co/datasets/tatsu-lab/alpaca/resolve/main/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet) + + ```shell + # download datasets + cd ./dataset + wget https://huggingface.co/datasets/tatsu-lab/alpaca/resolve/main/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet + cd .. + # process datasets + mkdir ./dataset/qwen15-0.5b-hf/ + python ./tools/preprocess_data.py \ + --input ./dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet \ + --tokenizer-name-or-path ./model_from_hf/qwen15-0.5b-hf/ \ + --output-prefix ./dataset/qwen15-0.5b-hf/alpaca \ + --workers 4 \ + --log-interval 1000 \ + --tokenizer-type PretrainedFromHF + ``` + + 5.2 pre-training + + ```shell + # modify the script according to your own ascend-toolkit path + source /usr/local/Ascend/ascend-toolkit/set_env.sh + + # modify config according to your own actual situation + CKPT_SAVE_DIR="./ckpt/qwen15-0.5b-hf/" + TOKENIZER_PATH="./model_from_hf/qwen15-0.5b-hf" #tokenizer path + DATA_PATH="./dataset/qwen15-0.5b-hf/alpaca_text_document" #processed dataset + CKPT_LOAD_DIR="./model_weights/qwen15-0.5b-hf-v0.1-tp1-pp1" + ``` + Config Qwen1.5-0.5B pre-training script: examples/qwen15/pretrain_qwen15_0point5b_ptd.sh + +6. fine-tuning + + 6.1 Prepare fine-tuning dataset + Download the Qwen1.5-0.5B datasets from [here](https://huggingface.co/datasets/tatsu-lab/alpaca/resolve/main/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet) + + ```shell + # download datasets + mkdir finetune_dataset + cd ./finetune_dataset + wget https://huggingface.co/datasets/tatsu-lab/alpaca/resolve/main/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet + cd .. + + # process datasets + mkdir ./finetune_dataset/qwen15-0.5b-hf/ + python ./tools/preprocess_data.py \ + --input ./dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet \ + --tokenizer-name-or-path ./model_from_hf/qwen15-0.5b-hf/ \ + --output-prefix ./finetune_dataset/qwen15-0.5b-hf/alpaca \ + --workers 4 \ + --log-interval 1000 \ + --tokenizer-type PretrainedFromHF \ + --handler-name GeneralInstructionHandler \ + --append-eod + ``` + + 6.2 Full Parameters Fine-Tuning + The configuration script for full parameters fine-tuning is basically the same as that for pretrain_qwen15_0point5b_ptd.sh.*The difference is that the dataset and the training parameter is-instruction-dataset are added.* + + Add the fine-tuning parameter '--finetune' so that fine-tuning starts from the first step. + + ```bash + DATA_PATH="./finetune_dataset/qwen15-0.5b-hf/alpaca" + TOKENIZER_PATH="./model_from_hf/qwen15-0.5b-hf/" + CKPT_PATH="./model_weights/qwen15-0.5b-hf-v0.1-tp1-pp1/" + --load ${CKPT_PATH} \ + --finetune \ + --is-instruction-dataset \ + --tokenizer-type PretrainedFromHF \ + --tokenizer-name-or-path ${TOKENIZER_PATH} \ + --tokenizer-not-use-fast \ + ``` +### Performance-0.5B + +#### Machine performance + +The performance of Qwen1.5-0.5B in **Ascend NPU** and **Reference**: + +| Device | Model | total Iterations | throughput rate (tokens/s/p) | +| :--: |:------------:|:----------------:|:----------------------------:| +| NPUs | Qwen1.5-0.5B | 2000 | 22834 | +| Reference | Qwen1.5-0.5B | 2000 | 25306 | + +## Inference-0.5B + +Config Qwen1.5-0.5B inference script: examples/qwen15/generate_qwen1.5_0point5b_ptd.sh + +```bash +# modify the script according to your own ascend-toolkit path +source /usr/local/Ascend/ascend-toolkit/set_env.sh + +# modify script model path and tokenizer path +CHECKPOINT="./model_weights/qwen15-0.5b-hf-v0.1-tp1-pp1" +TOKENIZER_PATH="./model_from_hf/qwen15-0.5b-hf/" +``` + +Config Qwen1.5-0.5B inference script + +```bash +bash examples/qwen15/generate_qwen15_0point5b_ptd.sh +``` + +Some inference samples are as follows: +![Inference](../../sources/images/qwen15/qwen15_0point5b_inference.png) + +## Evaluation-0.5B + +We use MMLU benchmark to evaluate our model. Benchmark Download [here](https://huggingface.co/datasets/cais/mmlu). +Config Qwen1.5-0.5B evaluation script: examples/qwen15/evaluate_qwen15_0point5b_ptd.sh + +```bash +source /usr/local/Ascend/ascend-toolkit/set_env.sh + +# modify script model path and tokenizer path +TOKENIZER_PATH="./model_from_hf/qwen15-0.5b-hf/" #tokenizer path +CHECKPOINT="./model_weights/qwen15-0.5b-hf-v0.1-tp1-pp1" #model path +# configure task and data path +DATA_PATH="./mmlu/data/test/" +TASK="mmlu" +``` + +Launch evaluation script: + +```bash +bash examples/qwen15/evaluate_qwen15_0point5b_ptd.sh +``` + +Evaluation results + +| dataset | subject_num | question_num | reference_acc | NPU acc | +| :----: | :------: | :------: |:-------------:|:-------:| +| MMLU | 57 | 14042 | 0.318 | 0.318 | # Qwen1.5-1.8B ## Training-1.8B @@ -216,7 +467,7 @@ Here's a hardware summary of pre-training Qwen-1.8B: ``` 6.2 Full Parameters Fine-Tuning - The configuration script for full parameters fine-tuning is basically the same as that for pretrain_qwen15_4b_ptd.sh.*The difference is that the dataset and the training parameter is-instruction-dataset are added.* + The configuration script for full parameters fine-tuning is basically the same as that for pretrain_qwen15_1point8b_ptd.sh.*The difference is that the dataset and the training parameter is-instruction-dataset are added.* Add the fine-tuning parameter '--finetune' so that fine-tuning starts from the first step. @@ -244,7 +495,7 @@ The performance of Qwen1.5-1.8B in **Ascend NPU** and **Reference**: ## Inference-1.8B -Config Qwen1.5-1.8B inference script: examples/qwen1.5/generate_qwen1.5_1point8b_ptd.sh +Config Qwen1.5-1.8B inference script: examples/qwen1.5/generate_qwen15_1point8b_ptd.sh ```bash # modify the script according to your own ascend-toolkit path @@ -262,7 +513,7 @@ bash examples/qwen1.5/generate_qwen1.5_1point8b_ptd.sh ``` Some inference samples are as follows: -![Inference](../../sources/images/qwen15/qwen15_1point8b_inference.png) +![Inference](../../sources/images/qwen15/qwen15_0point5b_inference.png) ## Evaluation-1.8B @@ -288,9 +539,9 @@ bash examples/qwen15/evaluate_qwen15_1point8b_ptd.sh Evaluation results -| dataset | subject_num | question_num | reference_acc | NPU acc | -| :----: | :------: | :------: |:------------------------------------------------------:|:-------:| -| MMLU | 57 | 14042 | [0.468](https://qwenlm.github.io/zh/blog/qwen1.5) | 0。462 | +| dataset | subject_num | question_num | reference_acc | NPU acc | +| :----: | :------: | :------: |:-------------------------------------------------:|:-------:| +| MMLU | 57 | 14042 | [0.468](https://qwenlm.github.io/zh/blog/qwen1.5) | 0.462 | # Qwen1.5-4B ## Training-4B @@ -492,7 +743,7 @@ The performance of Qwen1.5-4B in **Ascend NPU** and **Reference**: ## Inference-4B -Config Qwen1.5-4B inference script: examples/qwen1.5/generate_qwen1.5_4b_ptd.sh +Config Qwen1.5-4B inference script: examples/qwen15/generate_qwen15_4b_ptd.sh ```bash # modify the script according to your own ascend-toolkit path @@ -506,7 +757,7 @@ TOKENIZER_PATH="./model_from_hf/qwen15-4b-hf/" Config Qwen1.5-4B inference script ```bash -bash examples/qwen1.5/generate_qwen1.5_4b_ptd.sh +bash examples/qwen15/generate_qwen15_4b_ptd.sh ``` Some inference samples are as follows: @@ -770,7 +1021,7 @@ bash examples/qwen15/generate_qwen15_7b_ptd.sh ``` Some inference samples are as follows: -![Inference](../../sources/images/qwen15/qwen1.5_7b_inference.png) +![Inference](../../../../qwen1.5/ModelLink/sources/images/qwen15/qwen1.5_7b_inference.png) ## Evaluation @@ -1321,7 +1572,7 @@ bash ./examples/qwen15/generate_qwen15_32b_lora_chat_ptd.sh ``` Some inference samples of Qwen1.5-32B after lora are as follows： -![Inference](../../sources/images/qwen15/qwen15_32b_lora_inference.png) +![Inference](../../../../qwen1.5/ModelLink/sources/images/qwen15/qwen15_32b_lora_inference.png) ## Evaluation @@ -1566,7 +1817,7 @@ The performance of Qwen1.5-72B in **Ascend NPU** and **Reference**: ## Inference -Config qwen1.5-72b inference script:examples/qwen15/generate_qwen15_72b_ptd.sh +Config qwen1.5-72b inference script:examples/qwen15/generate_qwen15_72b_chat_ptd.sh ```bash # ascend-toolkit path @@ -1599,7 +1850,7 @@ bash ./examples/qwen15/generate_qwen15_72b_lora_chat_ptd.sh ``` Some inference samples of Qwen1.5-72B after lora are as follows： -![Inference](../../sources/images/qwen15/qwen15_32b_lora_inference.png) +![Inference](../../../../qwen1.5/ModelLink/sources/images/qwen15/qwen15_32b_lora_inference.png) ## Evaluation diff --git a/examples/qwen15/evaluate_qwen15_0point5b_ptd.sh b/examples/qwen15/evaluate_qwen15_0point5b_ptd.sh new file mode 100644 index 0000000..81dc166 --- /dev/null +++ b/examples/qwen15/evaluate_qwen15_0point5b_ptd.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# The number of parameters is not aligned +export LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib:/root/miniconda3/lib:$LD_LIBRARY_PATH +export HCCL_CONNECT_TIMEOUT=1200 +export COMBINED_ENABLE=1 +export CUDA_DEVICE_MAX_CONNECTIONS=1 + +# Change for multinode config +MASTER_ADDR=localhost +MASTER_PORT=6001 +NNODES=1 +NODE_RANK=0 +NPUS_PER_NODE=1 +WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES)) + +# please fill these path configurations +CHECKPOINT="your model ckpt path" +TOKENIZER_PATH="your tokenizer path" +DATA_PATH="your data path" +TASK="mmlu" + +TP=1 +PP=1 + +DISTRIBUTED_ARGS=" + --nproc_per_node $NPUS_PER_NODE \ + --nnodes $NNODES \ + --node_rank $NODE_RANK \ + --master_addr $MASTER_ADDR \ + --master_port $MASTER_PORT +" + +# Different task needs different max_new_tokens value, please follow the instruction in readme. +torchrun $DISTRIBUTED_ARGS evaluation.py \ + --task-data-path $DATA_PATH \ + --task ${TASK} \ + --tensor-model-parallel-size ${TP} \ + --pipeline-model-parallel-size ${PP} \ + --seq-length 8192 \ + --max-new-tokens 1 \ + --max-position-embeddings 8192 \ + --num-layers 24 \ + --hidden-size 1024 \ + --ffn-hidden-size 2816 \ + --num-attention-heads 16 \ + --disable-bias-linear \ + --swiglu \ + --position-embedding-type rope \ + --load $CHECKPOINT \ + --normalization RMSNorm \ + --tokenizer-type PretrainedFromHF \ + --tokenizer-name-or-path ${TOKENIZER_PATH} \ + --tokenizer-not-use-fast \ + --micro-batch-size 1 \ + --exit-on-missing-checkpoint \ + --no-load-rng \ + --no-load-optim \ + --add-qkv-bias \ + --make-vocab-size-divisible-by 1 \ + --padded-vocab-size 151936 \ + --rotary-base 1000000 \ + --no-gradient-accumulation-fusion \ + --attention-softmax-in-fp32 \ + --seed 42 \ + --no-chat-template \ + | tee logs/eval_qwen15_0point5b_${TASK}.log \ No newline at end of file diff --git a/examples/qwen15/generate_qwen15_0point5b_ptd.sh b/examples/qwen15/generate_qwen15_0point5b_ptd.sh new file mode 100644 index 0000000..391dc52 --- /dev/null +++ b/examples/qwen15/generate_qwen15_0point5b_ptd.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# The number of parameters is not aligned +export LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib:/root/miniconda3/lib:$LD_LIBRARY_PATH +export HCCL_CONNECT_TIMEOUT=1200 +export COMBINED_ENABLE=1 +export CUDA_DEVICE_MAX_CONNECTIONS=1 + +# please fill these path configurations +CHECKPOINT="your model ckpt path" +TOKENIZER_PATH="your tokenizer path" + +# Change for multinode config +MASTER_ADDR=localhost +MASTER_PORT=6010 +NNODES=1 +NODE_RANK=0 +NPUS_PER_NODE=8 +WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES)) + +TP=8 +PP=1 + +DISTRIBUTED_ARGS=" + --nproc_per_node $NPUS_PER_NODE \ + --nnodes $NNODES \ + --node_rank $NODE_RANK \ + --master_addr $MASTER_ADDR \ + --master_port $MASTER_PORT +" + +torchrun $DISTRIBUTED_ARGS inference.py \ + --tensor-model-parallel-size ${TP} \ + --pipeline-model-parallel-size ${PP} \ + --num-layers 24 \ + --hidden-size 1024 \ + --num-attention-heads 16 \ + --ffn-hidden-size 2816 \ + --max-position-embeddings 32768 \ + --seq-length 8192 \ + --make-vocab-size-divisible-by 1 \ + --padded-vocab-size 151936 \ + --rotary-base 1000000 \ + --micro-batch-size 1 \ + --swiglu \ + --disable-bias-linear \ + --tokenizer-type PretrainedFromHF \ + --tokenizer-name-or-path ${TOKENIZER_PATH} \ + --load ${CHECKPOINT} \ + --normalization RMSNorm \ + --position-embedding-type rope \ + --norm-epsilon 1e-6 \ + --hidden-dropout 0 \ + --attention-dropout 0 \ + --tokenizer-not-use-fast \ + --add-qkv-bias \ + --max-new-tokens 256 \ + --no-gradient-accumulation-fusion \ + --exit-on-missing-checkpoint \ + --attention-softmax-in-fp32 \ + --seed 42 \ + --bf16 \ + | tee logs/generate_qwen15_0point5b.log \ No newline at end of file diff --git a/examples/qwen15/pretrain_qwen15_0point5b_ptd.sh b/examples/qwen15/pretrain_qwen15_0point5b_ptd.sh new file mode 100644 index 0000000..1e6f5ae --- /dev/null +++ b/examples/qwen15/pretrain_qwen15_0point5b_ptd.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export NPU_ASD_ENABLE=0 + +NPUS_PER_NODE=8 +MASTER_ADDR=localhost +MASTER_PORT=6000 +NNODES=1 +NODE_RANK=0 +WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES)) + +# please fill these path configurations +CKPT_LOAD_DIR="your model ckpt path" +CKPT_SAVE_DIR="your model save ckpt path" +DATA_PATH="your data path" +TOKENIZER_PATH="your tokenizer path" + +TP=1 +PP=1 + +DISTRIBUTED_ARGS=" + --nproc_per_node $NPUS_PER_NODE \ + --nnodes $NNODES \ + --node_rank $NODE_RANK \ + --master_addr $MASTER_ADDR \ + --master_port $MASTER_PORT +" + +GPT_ARGS=" + --tensor-model-parallel-size ${TP} \ + --pipeline-model-parallel-size ${PP} \ + --sequence-parallel \ + --num-layers 24 \ + --hidden-size 1024 \ + --ffn-hidden-size 2816 \ + --num-attention-heads 16 \ + --load ${CKPT_LOAD_DIR} \ + --tokenizer-type PretrainedFromHF \ + --tokenizer-name-or-path ${TOKENIZER_PATH} \ + --seq-length 8192 \ + --max-position-embeddings 8192 \ + --micro-batch-size 1 \ + --global-batch-size 64 \ + --make-vocab-size-divisible-by 1 \ + --padded-vocab-size 151936 \ + --rotary-base 1000000 \ + --lr 1.25e-6 \ + --train-iters 5000 \ + --lr-decay-style cosine \ + --disable-bias-linear \ + --attention-dropout 0.0 \ + --init-method-std 0.01 \ + --hidden-dropout 0.0 \ + --position-embedding-type rope \ + --normalization RMSNorm \ + --use-fused-rmsnorm \ + --swiglu \ + --use-flash-attn \ + --use-fused-rotary-pos-emb \ + --use-rotary-position-embeddings \ + --use-fused-swiglu \ + --use-mc2 \ + --no-masked-softmax-fusion \ + --attention-softmax-in-fp32 \ + --min-lr 1.25e-7 \ + --weight-decay 1e-1 \ + --lr-warmup-fraction 0.01 \ + --clip-grad 1.0 \ + --adam-beta1 0.9 \ + --adam-beta2 0.95 \ + --add-qkv-bias \ + --initial-loss-scale 4096 \ + --use-distributed-optimizer \ + --no-gradient-accumulation-fusion \ + --no-load-optim \ + --no-load-rng \ + --seed 42 \ + --bf16 +" + +DATA_ARGS=" + --data-path $DATA_PATH \ + --split 100,0,0 +" + +OUTPUT_ARGS=" + --log-interval 1 \ + --save-interval 10000 \ + --eval-interval 10000 \ + --eval-iters 0 \ +" + +torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \ + $GPT_ARGS \ + $DATA_ARGS \ + $OUTPUT_ARGS \ + --distributed-backend nccl \ + --jit-compile \ + --save ${CKPT_SAVE_DIR} \ + | tee logs/train_qwen15_0point5b.log \ No newline at end of file diff --git a/sources/images/qwen15/qwen15_0point5b_inference.png b/sources/images/qwen15/qwen15_0point5b_inference.png new file mode 100644 index 0000000000000000000000000000000000000000..7f2f3478082fe3a576cbbd1bbf5dcbbd934332d1 GIT binary patch literal 20563 zcmdqIc{tQ<{6AW$C`mo_ME0VPEfKO!DlxLlma>GH?E6fm zOiYYjW-!Sz7-JhV!>niY^gQ3^cb)58*E#2pbDeYMx^U0s^SRg8=ly=YUibZZYJbU6 zVyDc`O`A4JSX=$&uxXQ6qOd$Cwq5v}-p=zC{@EPtV0m^^d6(R*@XJ=;GZ)Wn+EjrO z=X!4we&2D&$~Ab?Ci$3+pUtSLWuHx(>}IV0I^z`PMH`KUmNwRI>UREW#;BKOB|13R z92+*9JQQ{M{?<>5KR?ssb#Z1S?rPd`jWq$LNx)k~3PgmJZhU@p?oiqMcX5-_A68p_ z7f-7sZ2eulZ>6;3cX88;(|`O^JRGI+Pw}rU*8dcrY~2{wug{+U7gmh?j1=&2lE((x zAr2jVCh94_NA|(w533c+WQF+P9EWV~(E`(!B!`UO<7ifnQbGPTJ8nc#|3AD<#}k0T z0MKbGwFD#?n5#%0!iSTBJuv`G4w6RoO=F>`tb+I8sg-3O?6_Hh@%lv@*vkfw#uc@? z?8fpycRcr|XJhJ6JwhSSuHeM)^$4@vV$FZj+q#^A$FYX6NV32f_&A1>hC+Ljdagxj zr7cn!mHVu^MH3(S3pi{pKg{NM`@J|cf%hTW>A*)yO*6rv_!1(?A?Kz8XLfFeUUj)- z2$_p}_~<9Y?E$i!oVzUJO)jRz(;D(H5C3PpZ*&t>B=y%=jJ=8&?a;0%Ns`6V89&7? z$8e(@afsXw=oRtjGgP|2NrH0q3p8QT-_QNak!JbBTc0?MJxUaXRc{B&r4pIK_9Ksq z8=u(%rD~*+#3ZnIO@V`O(!?SbDD)E8DgR&n9uHDvQr2C>3fb!Kl z5&EVHtJY#C&pB-@Pqq7cN>Unn60$DO#t)t`#S3LgrWAP)qbBlFh?o+Y3 z&4*zhtZ8v#S*5FYnhpDByINOoE43dHw-MW{Vz?dt9DMW7S1*Vo6+e=}X^Xb0T~uF1 zBVAGveyC>8kM5#i)wffXyT4`(of|_H=K5NbbU+z_qKPBDbvGeD#fC7KHhWh!dU(Jy za#Iv{s#s55HpO|pY~&*Dk)~<1KS-^W@`XXa(HRz6bvp4HSA##k8H~|NZ_&*gSG{9| zTYY7kCi(P|1!5ot{H(yQcXJbTb!r zsthM|te4)Sy?h6ljJgav_#3#*RacbSSHL^zbqBtYL*6qIvVye~D@B*jz;_jjF=}{i zDAR<=po^PTc(PhH9P;5T7U-bUoz_fkX2Jag;;E|KzuH&Q#?G(6Bp@2<95G1M6CXXK z3Eo9g9gtm?8&lF?m~dm)^p927AqI*WWhg@Ln6yP7CVzV!mT`3HH3HAJzD*aS2_`W1Z{@#gG-Zz~@RU$0VlZoFOA zrEnB>Xys#=CN3|6{UIv#bfQx^XS2#UO|F0-X$z*^x&P^^jI4>43|- sPW(|B{?KM=K-5RS*>j`2FBG1{=uZNM{z%?p ztzDsg_S$(YQqYY>Cvm@5+T&`J;*}jdYrX1*D8AHmP?U=LQepGrh;V3w%qF@P_3anW zD3v%|YHxIk5Q{{&Yg_E*0yuA8fwaL>;)>amEz+Gw$mmqhG zts}1=Y~Djv@+Y}}yz0JkEg_PQSzA4wn8!O%C*@I24LQ(ZzV%8cpwt?gy-f=Wye0VQs5q zt+2HxKuz#-J)|I9?!nGRYLQYrEx6F2HYL|Pch5}F4#)?W#`q|eQhy0>)q#=Tv?=2~ zhX+4C6tS*-aK=VquT^hd54TgEP$dXSf+@96O@O>87Owx&hOn&B(}~KGICmB=E=r}r ze8$_Vb&r!h5j{ONVQV9aOLAMX(AoND1Ou?>u&eQ{OE%?I|Mr`b?4m|mUlJ`h-=Oy3 z=M!6P?r}WOLl5$}v6S>)`POLL3LZKOU^d@Ng{Z08gZrxH)=fpAvqsfd0|TJ zLD(ULvbDmd#NKY^`4?~$c_aCGv}VecNUj&Xifwvm@v^{JQ&e#hth7COk`%jH&DJ8P zgtKc1Jqc?@jIRk7_U!$(%V-&^UP@CbYEM?xR}Olo%B;Cy-rRkVQ~CTi1jbSf6X(SM z5C1>}9VsZ58vD@$u-ExD$5(~m1Ti0I51u{!jZaAIFEHK>&aXBT15_k-_}2p25qd3f z7;(3XH7!j3mA&NsS?UjdEzDWT+FzhCpGNTkcQ~)j=1nn$(Q7T~lxL<+YUq{51>*!2 zOxaDfQZ#Bx*4J2ZlyO$U;s<`sbx=|Yiq%y9yI1!^8Q)JQwi%vi4p(YNrcqfq%24`Z z&yeq~SAVa+`>N#Us@xnugKeJ4bvic~Rey>(q!1moo+vwp>si_E=@^Ytxxx^#5Xsq2 zdG+^)(QgifZc7~L5v(-7%jaoKoy3$)(&?NkHOU_{Zb6jc6 z_Qb7Ckp1ml&f(w9Nc;vw=~)EdQW(P}%izG}&KpnaU0{+5*guNmKp9DJ0Q%4eRtUZ0}+Wq7ox_v(e zE(GQL83uOgN2u{p?j)Wf48w@8<^Xe3m7I92hZKYx7N%q!#yKZ=)m%+T6=pE!+3~Q< zyk=R&dh*c$PRqyb#bBgt{K zs3;YzC*Pi%-tsEebuSsso#+G(~QyQAX^{U+F@?DTEm{s7TL{n?dt zMMc8i6)cRl=UQd`!ePJ}0WiTcdzW?o2|KRM*= zlAZg9^SAVd7OBvkn-13bL>(+ObbPl9Hb=BH8Y1HMh+P&|O9C`85A;=vnHPItom9`L zJzPrC1ewIo+VHN8<_g2sAB5od0xf!9(Hlu4S8!}J-Pi9aWuYT<=XZ7aA@>2uIl+O~ z47(lYELuzZlL;5b&~DQ1=NK=PBU&dV*5XLCNu29!kY z8HKfG>OCV3I>=x?$zD1}(G&`tUW#%ui;}%rAyUa+jf`5IMlEem&4|ACYWu$`Pr**9 zUGZ|b(U?l}#lzePTR+m}L-${eQ*7c-9avqK<4g$&KRG|S$XQC{>Tc|E;X9|Jzn5*% z>hC+dU^dW|%;c5TOEx~gDaTU$)J}=fBE%=kw@C7N@(7$YYd4yLi&XlBi5vWK`6DXvS69xPx1hUXH z_m0``wthgLN2ySio(Z+&p(n4PSl@7Ak``zyl7X$dK*hA5=9XgrniO4?d#xOH_kI;Q z5{^_&xZ!ue>CnGc#oC0B!q|M=2is$5r)0g*6Q$z2?_Wz}u~g1#g|W^%D1a-doS^lA z1#-+ktxhQa&)sFmqEv*^{g8{YWWV%76>~J%IZEZ+zp%egE|IYcI=JQV%C#qg@?6mU z&t6K_Mr6hWouiSh$NoD6c`+aE<2QL=0oZB4o1}QKgi3=4V34vVl;`!e+rl;?uv4wL zM3g+fa>vx#a)Tz$pJA#MrLunVY&Jr_U-kElEV1W=4EdFk_|X{=IiUtAJDq5n{(3rC zIbC_{yMB~_&HuJV*(OS*s<~f(s}GlbIuR8Y{x-{BCt>o|o_0;0I3Ka!HF2NEole}Z z*()mkeDn4y@JAupIq+o91M|xJFZf!@eYFmIKADOYkW@|h4r0e$wY znye}2qtiW6w8c88AY7;(zKNKzLA|oqev+&!H)RIxxn!0wnQHjr-=(=v2I(cJ2-GvV z*j9dxklD)W+C=Syy|OxV{JJsZNcRmZOPWj{gBrCPK4gW6RI>IKxUNgBIEkw?9C@O$ z=ii0G;)J4L=dl?+Z?DLf!c7kxgq)_^LM!3TkvpJ}xJwFFy)tV(GV{wM;aU%oO@PAW zaPTjOoqJ05{-=tYO4e%;2gK~TrjjatyC%Pm9Xlbu)o<6QuYFD?2#xP~r?Ro83HeUj zEk&%$L$|*82lQBcppe$Cz7RdXU&Pvtyn0z&r8olPB%bQRpTD>-N+p)&nv*7j|L5|x zY^I2H+lL41gl^Mg2`Xe^U?dVm6C)y#1w0ef*M0x2=F1Qf>)T$OI2HP}B)Y?TPx^y)O)vD*|`EHkzY}SXa{4VeY;E7<627A?xuo!QGqZ z(hg`v$yWK_yBVcoxD^qw*Xjg0M|8>bFywcYAN*mrl&s&&eoYWuuo*Y5QA$!lW?Ik6 zz=!P4BG9?HnSj0%l5QE4s%ah3#Ba4H`JXQ;M65jdq44|uA20=>>fdFs)_m#Xet~Br z7VC};Zuqz_AeTVv06)i`P+ zPbH`X=5AY=7ajT!NG8UGiriE>KS1N_;jJ1%EOYs5rh7CmpOg$L9yWDW?|yw~F_^xR zxy`HEWvj2LNrH;eA99EPeM%9iqOFNr$0L?I6|0ne0wX6%gaZ; zn=&HSe9j_PvF3vG@9En_`M=_&zGj|I#7}X;TbPXk?SA>f6h`cI9)rQ) zt@nr~2!}?_1qv_%9@V_bT>I6HjN|`Y+f{SWDrkL87>Ak>z^f!}vZ8MKB$+jY13j=# z-SMyC>%lmkNcD}Ne#QAL!-UuK*-{Uz&9UeLnyA*k4L^G0^P!<6N@X0ZG9IVQIQ@dk z@*TnK4G8*h!u#)kZ`zbq%0HjD+V%pC3R1F;06jJQ*Z1ZADQw{4ELcS#bVLhixc^of zD5la7fsA0BQ4xmqPfU{Le;VAqJuzP?5xt1A_2ba~+us)A-h{L zNF?}slX(BVT6BxG(Al&Lm$iQXpVcs4JD(`bE_eTT*XH>DX9A&WLSL#O0=P35@aaEI zm1-qS3Y}ZVBAJE#ID{)`Cd>b~#fwoYLQhuri3*z?wwC$7Of?vy(tzmNVvR9JPw^K1 z+o9XyDm%0{99(u}!~R;c|7B){akc%w4g8ydHvQkV*L$S@4UHe*|C18`f9mr8gVFy- zI9HpHsqi_HKP)vx3(1Aasz$cO7ZolWt9~$1i+j4kV>kHh2AB0cRxo>^KtH=Uxlk{q zSIE{sOr$Kd|IplGeP6LoE*+8Skn=;czyS8wY}Tg5SNcc!!flAV!tID=;Z{V}-#6VW zba>h2WV4Z0P|l62537V>6)_TYjSaSeJ*Y}1PVOxT-qQTAH+@aWtv_#YY$3l6`unE7 z3>!aVNA-)sdriG&b9QwFg(|f(!g-ke<^)dw76SkJ)&vp*$|1C8SGt zTSiT4!Sj?6_ylib8$^d!c&a322c}-9X*f<`URb6XpB7 z&bE#G%188xD{pAR*>D-06LQ7SnAImPnJ-}ws;>mv@eo$N2? z>o4IaWwy^?y7*A>q7`%OrRD~ij#u3}4mntJg;VO#Sn}vLrjy}vY+{k-`5d4CA2nO2 zKIM0`CbvA^93Wc|&XYP|@;F!_F|Kv=!SZ0m&z08L^7m?(Cet>IT;TjP>8DiuXl-{F zodERwg+kv&7cx1R)S0+$Dd^<>CE(32lh|(vxeUw8>UL;l#io!s$h@y4|L}4U*_}vfXEiA-r$=lT!LhoL_cJEq% z@!_paW8Gai825!AjKiRlbCDqh)X;Z|wwwz&J^Z5JLB*qW2Z<}rRMt3bjIlb+o z?(&s*gNqQTLxH*x5Wb$|Vn%+&`WT|3b*06l#*E1yxQL%m~;BN&;I6dY$ys|JfDyyZv zJo;+o(I1!q^+?Q{rgq7F0e4A|qywLQHPRnW|1vl5epHV>v?zN~5=9aC-0nVm-NYgDiuC=3X zaQA1DpDYwAK@*JDwad3zuCDL$E6FYv`r_0lpD6g%k(UB@DkLK(B~S7*A1*-#=*v)e z_6v{cyH~0`8Yj#f*O{fq0}HSb-&i`Q+zTnH8gt zBp%-_CkSfEjWIyQ`snqyFS+)aBKq?x7A4Db$=CS9Al`R3CgGM~*rG8|?t*{)3I@)^ zm1ZQfrYq==*g~WGYy2;LS@ijU=`?!5>=B&!IpjjuH=eK z{S*!tJnz*6eAI^A71lDDMCPu3jtvlF9gKsdKuuT;X450 z+v>TCjsWR7^b}$`K^QVIy9=8GWia-5s!km;hrJBPr4mWK0q@?5vAWiE3r<)wF@He} zm^5?2dab&A9Yv7C#`ih&ACH!jz|m4XZ7h#b!92Ex9Wuy&J*9X5X!l}j{Az2)S^?a7 z`Z4pwuxnHPDlW3N%kg;aP)7sKV0rR%;=XCZ(8(G~AyaQsP}d2_lxfRp2{FvvcNU;F z@0b+N!&7lupkWCpSyVN~e|2K1f;#`)*;WDy(3?jR!bUYy7%x`Mz8e8^UXbhdKX9v4 zD3Rm|hUsz3EHI8-P@x$+G=m4}xr|o=u3q%Ph&?Nxoqk|1`URY!@6{G;7@q+qKP?Ek z^A1cSxdTHMs{ZhP@3OE@ z^|6nrgru?C_7B7@Z&3DB0lYC#&<#rDC#q`q`;wx+*1iGmY%Zx@TI@Jwu2Qm-qXLIX zPU-VZ_?Xck-GP*en)e@WzpQQyS3~di+!B!M@-1YzHF#u=KH@{i{)=84~@ zQvr&}g{BOO4YDGoHpYA;%@3J9TmeGPKsK#_wc75gdYW)PL9eF8`s>iER<;wK{gq8N5dXq;R;heKOHdMRm z4?5vFziCN?$HKKO9zqW)4m0B!xOpO;+x+|u@8);Gxa5JH5o8*42+-U$S57)WKEAJa z&n})Cm=G7Km8I$rAK%>4FI*^j5U-q zK#R*XY$`U;e75&be14M`De~_7lU`*fr&Jwfl5&m${YP{GBR*EGk&(|E`}zK$x#N}# zH57Zhac_Fu1ZvqZdt$VW{2;D&&z!g3lYIA>FQ8B&HIVdlo&44EJO35*DoHM3umlw# z8cE{40DMQWG(DR*^Vofn>HSiY-->NoxG4he1GfU$wR0gN`~WdJc!s@@~CNQ_S$# zorw`6YD|~|VSsi_*nCF?|8~RGnr-Bzb!8G=vL9)#78g5~XG#Y**g9kIv|_;4yhRGh zZC)NQU{Qo`U0qL3dtDvIGk#xxkmv=~B~H$4Sw1M#n=!OdzvKfcNh@eZSSUD?9E~5oL*QtJ)o5kh zd=1Z)Bg>8uz=}QGaM2vfJ0IiVvUO|`-^TCd!FZFA@|X!G@mfxsRt=lf>+maum?>sM z^!&b!X*_HUQ1C2xGj<-=@H4*RmYI$!4?59WF|VW5O0a&+1MRV6M_5?~yY))n=5Y<^ zb{Pzi8ZI|Lz~BP{fX~Y&Y8U8XaUJzNSG^dplaogTq1C!Wk2r-ftaaVRL&sO67y>oF z=k-lAY*h=>xVk5fox6p_;RU)W%OJ_uxDr#@TRI8vdv3lg&?~6`MPc6avVVor>(SwVNDJM_DiL?k$pH(PY8J4Nh38XyveHd%IDB_R*Mi zl?s5jvntMebDhGqqNfeHR=uB<6K?pw2=Zw0fEOz?8h6Yczvz-XE1vLxADd{;XW1nX z4$${WSsVhA_61<}JjdEf)7QRdCfpfA`M@uUPu}(%{mEt9iS(xDUb@vcOTXIJ=|^VS zassTyAM2Y3Ea+7~U2^TP9FSmCW2M1&!Xm7ooVVM2V)I(eZd_P^e5eJc81|nV4+qJ1 zVZWBa@|OLF&!e7EW{ERFLc8QfO%~NrF|eU{agMp<0^uK=#ctcc#Wij`tb(uho~bsA4Ta=>8<-O^%8Wx2H2T^#qd>I8wBJad1Fj);BT$5Y z;4XdTyn@QwPIO<_+RIwx{HR`>ScT2?R)*Opw$&d;Pvuf6?1juhhw<=FV#~{*RfBH7!WE8&GQ5%Um{h8fSQdk0Gx$ zsZO;J4Ip*okdTgB(z4DtoMDQlx$NfWL*fM18ie?^u+;fb2sx;x+d6oDS~5)Y}>W#-aDqATz|*7d7C zD`IitQO&zztgl0Qpo>4}HFdKs`0S_EK8K{4;!~A?+1kiM{ny?Kl#_1e4wmT)<%8ep zmmL*Ary_!T+?1}zn`*nx^uvI-nH7v}&=xExKY%4*}NV(o> ztiHBMHlz3Tx-Rf`mah;UGK7<|X~!l`VZhJ4Ls z;Cd%GNEy+hATEfCQcUQm)mVh&UR`wYBGP`-K32_R^k2t4Zws>5i z`2`fiu0ePhzu`#^-TLMncE`L4R~pZGynJk*;HQajtGdh+E)T$)+W|7A;$y)#Lce8B zV@QZO|Ge53OmbFnZleSRu$ zc)9M(Hx7*@=RQ5_@f_RZbe;;%zRS%;9w05<97Z!}I*TTw4QGMWL49kFiO<2IAowk$ zpZk1L>+;2+UH*l=;ToAHl`zUxsv%S&7sXRc?pcBj)tZA&5m=*tk|v!3hssTJq&5ZT zZgAOxLkan)rQ3@2tOL8D6Ogst=vo=&y%WH|JluU&sop>e&HPr|nB-;^_{_@jGb8w04E& zJIj!X&V}c$bcgRto%<0bKL0*})O<*hNb&;(@xrO+Sc~ri-Ut;>264U;$fDgYdfsLp zWb2Ayag%1SCxwy6P7Z6);*PvWDg!*T;Y`6`5*G4D^U8fAgD&Ipu%EGS<%wQH<`Y0F ziW3@L+2Xe$hdc)_cQ$*lnZS|30zH6sXmyyW|2X~SCIw!oxA*8_{r_%Z3guhd@uUT;%kzD>K88$EKD(+cv%q-$;;OpRulNKFE$VICz79Bz(;r#Z;#+8WqTER6AigTqe?VisSDrKOy zRt`OPn}KPzoTFE#4XRL`2Ufts9b_F=dbA0FC2c{HUa50#n|Ec_Sci=zVZDF8k)^ND z8hkR%zOA|M$0O=P{NU|j-^5W^y*zDk+R=Xe0<1A)-@`pNB!h2#!&5q@t(j%C80Pns zk6il4I}u`yYJrQ@19HyNF1zapA5^`qu^$QnO<{HFH#!VnRQ^(@AY zQuA@so7b9Mxh+_bNS~3IZ_@hSNEmafKa~O!k4nJ2zjJKS#2KA7v0R0OFd@a>xvy{HdPeY#6h@Pd5MnzEBQ?U?I(gKC)&d_qs6Xa{Oz3 zh4LGQ3twoN(d<)qxiB*}WK?pMoYV%PqOQFq@8ci=+jkGim$%3%yEo6XCvHTZ3I)Gg zJ*^UPITK?=C7$8>FRm3vzb_Ls>yM3|3;{FfspT8q%r!cULsaj7t zbF|+H%5O7|-%h6%^6Ic6RJvQPtc8GxRBl_RQ%Yz1*3Id5*Ll+1I~)|aTw}-{>3I;X zV1K05!!2uH$pk;4?hs8#CfyExqY#sGze`?+OMfObR!ayJ`s5Uz?4|_)B-|a0qfOiR z%TElTDD{gml-a7!2u|PBy}t&BK1E0m##CHnOmzA5NZ~1JDDN0)q__iI0Ge3rJ)U}r zpbxLQTXs`gGJgBM`J&*W=Oq(@yBx-!{p|527>KuDLbwehzql`^VJgHkTc4HYFjhO- zKZvmUMVkcY6U%hARLV}bFZnp8zn98IiBsav6dUiA5{+VC^HAJcE2T|uHof}n#C(n2 z7)#=oI@a>&ClT?eZu3UoS=Y9qW97_js%i~c*GP5!>~X{ky{$%5oK@((8;8OImaisj z#gkOO<}E1H?1`}F;dIreimDjr*r|7Wdh^kJy%mQd*I~h+^{t*`9K!Uv>kgP~pv*Ip z`1Q~H+k)e%lLGD7DY-cvt=E~5=Z+ks+B;(g$<8&|3*g1iE2WW9a3@}PBxhts*3&pJ zyk&Z)UNUu14)08M1azOx_2rVZhF>s&R%BgnU zQ!RpkoLt2}@ps>n57`|8ERTDxZF%dD9SY37E-wkSmswnBl-x;)yuK0M+WVPsyusDm z$O+#=v{9%(2kZxSZs4RfzS=WQT_P2*JEBTHi=3o1Ui+pniJ;(wgKCgglWp$t`G5KuwtX>{W=i_GnZI?BnG0C&eq+c;qJ`Y!bJrcNXTs6z zzOjsn-hCv2J;6I-RJ%`8vB2Q6JS(P|mf}iVbD9PXU!ea*Hg7sH)7Chws{x-PT{ptH znDelZId{}?^W6GH1*n;?X;&FR64ABC=tREZ_Csa8S&T2kl!pHE9Nl-#gxwR1M4%kf zHo9VIiP#iO&B6k6O?;f>1t=V_eAA#LYc2@)y=^yhbifpfIwToGT&F<= z`)pV@?xdQCSerL~aW+)y^djvyr;5gRVwY}ZZ3X$3?>@^oQ*JI6pyR=QipiaiokhG) zjiO;TgH5nLH1Vi)shH8bqjBuPv)sbRA0+Wo4+>~!+>G?6{9RNYt(cvt&?*SI8xW=6 zPrHfeLJN$x7gXH*>L?6uGWc3+;P%tq+h{EC&mhOd!#`8`O%ZGi+$&jShPdR6jW!XpGI&?^Sc;?W5O(r0@G zYKUv`MBMt36apb`G%xaT5O9OUr%W=M-f>>_^Bv%w02AURxs=f}xXE4RzP@BO!@lG#|GcXr`)~?%yJ8H>>gpln=5MP}JE!p_8e;~x!V*O0P za26Nee6Za>!l@6aDG^O*-PAY|5@yfw7DlSNew;om8S~*Q*|^G>(v_cmv<5CU?|a)- z?2@6f=*YnKnDC1}2LrD+)yiYTjxFTooA=f35jsW_(#a{_M&^od-Qr^BFU{x_MBa7) zkW35|L(d*$jpoVTUOKVtTh_p14}St<3c+V?Db5;aMz56AvRY_$ zVzla2m_2q3BA5F>m?3XNyS@hwP9+e|jboi@%6)Z7E4BGtslyZ$K*vHL|0rBLQo$50 z;IFNDUgK5V;Ps?=`&RJR?&b;Z`E>}gSM*3ar4z#x3;EL^DhAf;GC{#9C5Fuu&6z^nOG`TT$Yqulvs!L&~{sxkO+FpAU! zhH);^AOkoeXUrqiqh#Wcj_^&DmpSgcU^8SVv>e4vQV0 zRXouz+Q8nr^kg-gI@ta$)D{sFM2lBHZyOKER?pVcvhSFUOp(n{uI-cSC~6-e?}dAh&Q_B*ajx1PIzFe?JiO)`ab3Nr34Fx zO#vEtBgeq}yOFPLotaX)m;KHVYp}~$X9pf_WKHfwO;J!LiNwFfpz_+5)m>_P-lS4u zr|h+IqigRD#eHxB_{IUyX9Qrn2+y#(sR%{(AyFiKym`99#Xlu{K?!Xqd5C3Z;DQQET&y;~{YO!xL$6$X{_@O`{!xagaq0_NQIY zpH$;Ml<%l%q_ewxZ^g09ak|O$Gv&5Zr>bi{g(w`V$#8>@lE#LmlQJJ8fPPhNhTN59}yRn=51o3{M5r$$h z*yv~b2uqFi(mRMefu+Y%G>`hwgNKiaUTsO|V5}OBVBtwP<|)T;TO0u1!`wCW6Jm0- zPX+1)w}~waIu$=fb{$puQ!2rhG9)Ck5<~LwouVaIL44ypY%;UhN%T(t`B&LZy z(-AP7sxY8>*2HrFmr%saeBWOiC~!c6(WtD^n7JWN+=SzX6ze1yR!3bC$aj<(Yu{LC z-21($i=2gAC10hm$a-yyGE`InjwL|sk}EfnYuX^rbL@#-U&!?eE7h3L;{Lk~+X+rv zjqgSVIa1Xqv*J$Fk*0i0=+YJxL_T-}p3e!n;$RFcz7XDe$`Dud2HM2uU(0A4=~?Nh z83g8}Ch8ykf?fNDc{g%}1_&y5qYk6(q zIRvZ*%R%R!{(|W{SC_WLi^);OW&gz=@BR4Ot2ki*Jn=XiZSlxxQ0C zjYqX|TMn-9j>3Ho#0%yh@^+G5I~g8A>wZUL2Lm%6*X8Ys)a1=iRYw9m^_D%xhr!I@ zgmMd?H6a?>6Ji`W-y420jy~a$G4jK*(F7l0;W<$LJ#TWHkY{Y3*9!2m>%fm=Z$2}R zw*L85Vrgu$!ZF%H{wkjT_3SI-Wz4gO`@pKJQG-K2Q!dc*5b>+qCU0X`*KdXs(849L zAr<{nV38*9a zYxobMz%(qumslscD+@xhnn)l(4@$sFYg@y|s5H$fe+VBu3b7rNs;vdA@J||MAuAAT zRR}JJ?le}zbo;J=8_h$i7{)bOk4+Cwd2_Yq>7s4q%TUeHP!h2!3tKVLRN>2*PFg+i zVyQW}rYvnxDU-ph5c@eZU$>Y%wiJ>$F5~IGh{I(!c6>p&i`_feA+SFBTmf7rls zcT53^s%kVT{GkZ1W&$Lk%b9rfWKE2ZOmN+5`x{1D-QyZ|eN`zDIc?)t;#Wq=|0Q#J zYf3=t7=7RQlrxtN0nthcKRv=+B1bP9^mE$ykwES0*tl@XsS3{n6hf{KWmMqwH=z&6v+@WE~nDfTCSnv-tUi5;M5Pgx0?1&LrW;J@$gSWao~> ziW=V|0t@2~O8C``0%i+fXm!@cC#xLFfm&v+mw(Sf$J+$oOczi4I8JKIR|UpH#9q+cqD5u)eYs@eH0vA~w6^4qJ}8Z|Zu#GwS8^w*AarM5uB zyPu=5>olFdPw$!|ae2n_b3Xo;-|z7nU(3*X)%zL-cK~4defRa_4YS`b+2+>CKxCp# zQ<=kw9ja4h)r5WnPQ)X^O0#C1;jxs*^9yGe6=I}%Xpz;PZeIt8_%Y_yNP!mMUOJv? z1i-mCKL**Dm+;;a9K&Lr8h1QLJGPjhe1y@*X*O#d?fQw(QNRy-D?bWVzQbRuTGgZ5 zMe*I8N|SI5a=TDC=OU>nI!9o%oBx7=D~$tE+2#Gq_I&t+XS$tjZdz84a92}*a3TTc z=Ba&=F<9i=5P7>bJI)}{ADjduN&0CEjmj?PV@V+l#PDrltkH^Gxy4KA)o=9G`WJ0<1$CLWO%VyE*peeZr+Ql>-pp<4R+O8AdvbqiGHUdp>nO&w>W@)PDgxCoC5ZQX;rCw`rceMgow^P=VlE;6E zo!UUh^6A9-J{AU9OGM;cW$2%iatKzToi%{&4-R`A3*sh<`;W*GV$FJfe_5 zSwHAA;cK-xXQ*mI5^ph)Sogz5nRi=eO88s_>%d8hWr!Bbfpb~Nt31_YTJYQ{$eTK~ z?BGP==T~@(*+292eus_b4(=Pl0b%}#M}%|7YKEK@eCoSA*DrvB7v}pH)XM>$MW zo(T08@CTM@^Uofkz%`rY>AxGQc_$ik>LA<9+rrH{BpBuK4Xjs0U@w8FwrLo&BN zUUG&<&>X21Gs1bfnw<@etH0kE`iD#}8zlZQoJV=so7`nyHaigQ$60;WZLU}aiaBRH z$62pE02=G9X+jtT%A%VvAvtN*#zXxv9Ow9uNPQ~SuI0tSj@E}W9=vc=N$hb!a))V_ zVCHr8x!W;y$FlX>@hV(%>UYuFEjEjdd!^@a8vrrG3YcEk{su^1(|OGpINc~8$};^0 zkzCYc_3TJD zW>mD@Le@*cJt+U;G^efZnl_Q#Z(!m%8b_+EGz#>zvn;rmEY~h5V+iFby0^>y3Wwlv z!Q#7Nzw0f%R>fDsm$}JrB5q7r!uh`tG_x5fHiX5WBH5=k(J!e{5hFBYC`J+2SG&aP zdB47g^dQj ziZvW9_*gCsGotr#IsrLHcYP?hw;Srth@9;qjL(0;OU4KrY{EUQlm;)R!u}R$?xzWM1-m3SHW}*M`u~+NzwPM zg6zFFr>z)_boWbzOTIs;u)hp~S*RO|X4U$hC6VN@%ziZMeWMsSa(&7&K!DO5vK~7C zxoqs)Er2dmnmWi1-*PTSll8TCGlGAvInpQSD8BohvjST4=d~99IgKe==uQ~8jAYiy zgLL${@{lJL3XOMjoEcDalZD5FiXTg#1F4+%a3WZGg2^Fe@Iv3!xth^taWl;d9Qp;i zzbm%fMIkL8PCnr6)iAr3cd-CuCTZYcRxLar>pp|Vas;Id) zuTG`BUwkUeO^_3Uk%gR~TYFZB#G6e0I;_+!LH2E))aS^|E_l9a3=G-;>A<;A%IJ#{ z^i>x@Y{jc9HvaqDy363Wa-GL@mv(VtDBIUDdC$|%Y(GCD;w_t;SM;D<=T!n`sJhcLOWn|4D~v1-f_vw)R$Tz z#F0zqWmbKQq#Rms#!B#%N}e*4%kl#x+>qY-y6}?OJz}Qu1-G}+m&lbrY(+j-=%@$E zC)m~KB`}jL{DLm{xa556{ZQ|j*7w1k>xtP<^VDd4-=aZTN|PTd-|ZV4bNwa$x?Aft zkMB(5RPp;bYh=`SJ}1C?xo7CITXh_%_h&;fSE!w)J|wRqU5{4}=W{}LYP&B0WYKQ- z7#l}A3GWl|B1E?AFkh#AkVDvb4_ zVe&RLn2ac0AI`wm3isBNn?G*^w5)C<(1$yAGWjtBHyOJu36mk3WgLSChMBlY{Fw@ZnPSIh| zYQQ-;ZB2u0v=yvtm736fX@w_4w8Geh#+w7ZO}%PgJVRl?fBBhfc~Z7Lr=mX9(l3Ci zdpDaFU;OgFwR7iBN#|i4z_)R;OQo4w%{=H@&9ZE@l@$f1!)?)4&9*cRl^i7xC@sa? z%8**GPK+fFrd78xl=3K1zUz^0d6t37SMVS*BP2lN@~cZW)BmtD``hQ|&pgk}=b7jI z`sm#jNNE}cc|F+n8*u1ydimLSslaXSnGcRs6B#GlG-@*2CA_8aoAzXDM_Lm+Dlqv@ z?6JKKBeD!?GV+#Y;N7K9ruS1GQD^h+qwMV-YF=}vJJfQey+z-fPe0$dS8{fEk?%rj zl2TsWdrGWu*OEp~L;x4Gp;>G*glR24(OH5r&yrvrdHtR3G$G@5gX>4&vRw5;OpI0g zmKx5&L;kC`H20^3V6V}=3bQ;GulVmAnoS=3FJlSLf^So&;osB^38OQ4b)t?+@7X<; z9Y`#q?Ym55Z1@hdHO+T&lZ=vgs0-EFi%_UNWXy}(9Cy^e*s47D+-6{a8!|O9oOPgu zy8N6={q;~%Mi0gsY8-zCX(~G9I6hpGS*Tw8OappUDx^3F%Dcm~;BYrC_+;yrhY6eg zd{y24vr9SqD7yX48LUlb?(ar!PL2bFK$+v!@6Z8HlCF<;Jo#R~(8qbJp6y(Y{*u?NK;YkF=(y5$Dlvyi}`G=n_p77)) zmNtU>-uFw{ta8wihXbNd(n81FN&2t`H+eTPiOtC}?!l#LD8ITIU87l)O=rG@d@Vg! znSK~qT6NyvlrRF@b~QF99Q&aTVWvGKoc-P#b2Za${>GFS{vrqq!*322RjspXyX`vj zQu4&Jfd;HE2o_jU5@mh3Mdc1>K4^6WYA@mTz{!z{Eu`3-q?rB0tScNR=Z*Tua`0#* zWAUOs+~P3sYgti%NH{5G+2q$=kO6fc05kJEdhMnjyA7dK@AD$G!CQF>Am7l(<1OBU zu|Zs*I>U#kypmFyVv-teFc$|ogTY(`YgrFw9YmJJK$_)UaugrV5fLConmcgNN7cKl z3Ejl%E|UTk$W+JF6Sj!VS3z=ln4tir+%q%3=t*B>~pSQ!_)u zkeB9Kg1egCrlF*t5+Fcj+)}E_sPhU}-t4$%l{QZjPF8Dm@U;#+RtqM&O=?-o^Hx48 z+$AWD1TttQgJ+Ju9Q55FAS8gC)ll_u2bes2YP+$+^eV6Ia$y@(&yCkTH@*lsb1Mw; z-rPAn#D1<9S9J{Wx!U4AMF9_Pk^_wc#YEl+_XpJ7KUD9|en*t5-P0ebGNa z72}X7!a3B?BAp`Bvq{m=@OYA7F&cSFXNAW225Eq#yBFa2dE!}uZuIDGLP3l%kV;=0 z{HA|ZdkYp{M(}JlmA;&0N-m4j|Me;c cbYq%DlL%eblxrnk@fo&Zz5Ncd_D5X$7f4QX-2eap literal 0 HcmV?d00001