commit 8542a1f9ef9ebb31f470a46d44788a41b44caaeb Author: Xi Xu Date: Wed Sep 25 18:29:02 2024 +0800 Initial commit diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..684ae6a --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,12 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: Current File", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal" + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..b9813e4 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "terminal.integrated.defaultProfile.windows": "Powershell (conda)" +} diff --git a/01_getting_started/01_getting_started.ipynb b/01_getting_started/01_getting_started.ipynb new file mode 100644 index 0000000..55460fc --- /dev/null +++ b/01_getting_started/01_getting_started.ipynb @@ -0,0 +1,6005 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cc79e031-c5ef-4dfc-8402-eacd0b7c6da3", + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "source": [ + "# 实验环境配置及基础编程训练\n", + "\n", + "## 目录\n", + "\n", + "- 实验配置补充说明\n", + "- Python 基础操作\n", + " - Numpy\n", + " - Matplotlib\n", + "- PyTorch 基础操作\n", + " - 基础数据操作\n", + "- 数据预处理\n", + "- 查阅文档\n", + "- 线性代数" + ] + }, + { + "cell_type": "markdown", + "id": "2a15f7b7-bed3-45af-bff6-96734985b337", + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "source": [ + "## 实验配置补充说明" + ] + }, + { + "cell_type": "markdown", + "id": "5a0c00c0-007d-4763-b902-c0d98a1b8cd2", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### 命令行\n", + "\n", + "基础安装配置已在另一单独文档讲解,如果能正常运行到这里,说明实验环境已经基本配置成功了。\n", + "\n", + "即使之后的实验提示缺乏必要的软件包,也可以在 `JupyterLab` 集成的命令行终端中补充安装。\n", + "\n", + "> 注意:任何在 `JupyterLab` 集成的命令行终端中补充安装的软件包都会被安装到与启动 `JupyterLab` 相同的环境中。" + ] + }, + { + "cell_type": "markdown", + "id": "cb1d4d58-aa2a-4086-8517-5c72796debab", + "metadata": {}, + "source": [ + "具体操作步骤:\n", + "\n", + "1. `File -> New -> Terminal`;或 `File -> New Launcher`,然后选择 `Terminal`\n", + "2. 正常使用命令行工具更改运行环境" + ] + }, + { + "cell_type": "markdown", + "id": "2b26d1b4-cf1a-4940-8e31-7856fc959ae6", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "### 笔记中调用命令\n", + "\n", + "在Jupyter笔记本文件中也是可以直接调用终端命令的,例如查看当前环境的Python版本:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "ea3122b2-935a-4814-a6f5-32724d90e05b", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Python 3.8.16\n" + ] + } + ], + "source": [ + "!python --version" + ] + }, + { + "cell_type": "markdown", + "id": "2f6f5920-aa1d-4c7d-8d88-2ea72e738cdf", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "### 调试器\n", + "\n", + "JupyterLab也集成了调试器:使用前需要先点击右上的`Enable Debugger`按钮。\n", + "\n", + "然后可以正常使用设置断点、单步执行、查看变量等操作。" + ] + }, + { + "cell_type": "markdown", + "id": "c596f9c2-1cbf-443d-ba3d-8062e30ba7b0", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "source": [ + "## Python 基础操作" + ] + }, + { + "cell_type": "markdown", + "id": "536bf0be-5558-4704-83f3-29810c4d5b33", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "Python本身是一种强大的通用编程语言,但在一些流行的库(`numpy`、`scipy`、`matplotlib`)的帮助下,它成为一种更加强大的科学计算环境。\n", + "\n", + "我们希望你们中的许多人对`Python`和`numpy`有一些经验;对于其余的人,本节将作为`Python`编程语言和`Python`在科学计算中的应用的快速入门课程。" + ] + }, + { + "cell_type": "markdown", + "id": "09ed020e-157c-46ba-a3ba-23a46cb1c6e4", + "metadata": {}, + "source": [ + "Python是一种高级的、动态类型的多范式编程语言。人们常说Python代码几乎就是伪代码,因为它允许你用很少的几行代码来表达非常强大的思想,同时又非常可读。" + ] + }, + { + "cell_type": "markdown", + "id": "454e6f4f-d5d2-464d-bd35-0b21f8fd0b82", + "metadata": { + "id": "NwS_hu4xL9eo", + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "source": [ + "### 基础数据类型" + ] + }, + { + "cell_type": "markdown", + "id": "62880ea3-9a6a-4274-b944-ca76d6d922de", + "metadata": { + "id": "DL5sMSZ9L9eq" + }, + "source": [ + "#### 数值型\n", + "\n", + "整数和浮点数的工作方式与其他语言几乎一样。" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "92debbb1-0b9b-447d-b31e-adeba820bc73", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "KheDr_zDL9es", + "outputId": "1db9f4d3-2e0d-4008-f78a-161ed52c4359", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3 \n" + ] + } + ], + "source": [ + "x = 3\n", + "print(x, type(x))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "d5ccd284-e91b-4bd9-9dde-640ca8b69791", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 86 + }, + "id": "sk_8DFcuL9ey", + "outputId": "dd60a271-3457-465d-e16a-41acf12a56ab", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4\n", + "2\n", + "6\n", + "9\n" + ] + } + ], + "source": [ + "print(x + 1) # Addition\n", + "print(x - 1) # Subtraction\n", + "print(x * 2) # Multiplication\n", + "print(x ** 2) # Exponentiation" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "144bc983-be47-4e00-b0fc-1f5e511be462", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "U4Jl8K0tL9e4", + "outputId": "07e3db14-3781-42b7-8ba6-042b3f9f72ba", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4\n", + "8\n" + ] + } + ], + "source": [ + "x += 1\n", + "print(x)\n", + "x *= 2\n", + "print(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ce0b350d-6308-4418-b450-6a480e5d38fb", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "w-nZ0Sg_L9e9", + "outputId": "3aa579f8-9540-46ef-935e-be887781ecb4", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "2.5 3.5 5.0 6.25\n" + ] + } + ], + "source": [ + "y = 2.5\n", + "print(type(y))\n", + "print(y, y + 1, y * 2, y ** 2)" + ] + }, + { + "cell_type": "markdown", + "id": "0b7beebc-96b7-4b62-86e2-6d946329643d", + "metadata": { + "id": "r2A9ApyaL9fB" + }, + "source": [ + "注意,与许多语言不同,Python 没有单数增量 (`x++`) 或减量 (`x--`) 操作符。\n", + "\n", + "Python 也有长整数和复数的内置类型;如有疑问,可查看[文档细节](https://docs.python.org/zh-cn/3.11/library/stdtypes.html#numeric-types-int-float-long-complex)。\n" + ] + }, + { + "cell_type": "markdown", + "id": "322ab33a-c59e-40a3-976e-322d162d8cdd", + "metadata": { + "id": "EqRS7qhBL9fC" + }, + "source": [ + "#### 布尔型\n", + "\n", + "Python 实现了布尔逻辑的所有常用运算符,但使用英文单词而不是符号 (`&&`, `||`, 等等)。" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "63e4ccf6-d7ce-4f3b-bb01-e945b9418ef6", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "RvoImwgGL9fE", + "outputId": "1517077b-edca-463f-857b-6a8c386cd387", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "t, f = True, False\n", + "print(type(t))" + ] + }, + { + "cell_type": "markdown", + "id": "9616f32f-2069-4dc3-85ee-761351f1a233", + "metadata": { + "id": "YQgmQfOgL9fI" + }, + "source": [ + "现在我们来看看这些操作:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "196b2bd3-d3a4-4328-8245-4b337f6002d8", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 86 + }, + "id": "6zYm7WzCL9fK", + "outputId": "f3cebe76-5af4-473a-8127-88a1fd60560f", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "False\n", + "True\n", + "False\n", + "True\n" + ] + } + ], + "source": [ + "print(t and f) # Logical AND;\n", + "print(t or f) # Logical OR;\n", + "print(not t) # Logical NOT;\n", + "print(t != f) # Logical XOR;" + ] + }, + { + "cell_type": "markdown", + "id": "97598521-e74e-47e3-a4dd-b6ae18a35063", + "metadata": { + "id": "UQnQWFEyL9fP" + }, + "source": [ + "#### 字符串\n", + "\n", + "字符串不区分单引号或双引号:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f5ea11d3-5c0a-4218-b0a2-9323e16712de", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "AijEDtPFL9fP", + "outputId": "2a6b0cd7-58f1-43cf-e6b7-bf940d532549", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hello 5\n" + ] + } + ], + "source": [ + "hello = 'hello' # String literals can use single quotes\n", + "world = \"world\" # or double quotes; it does not matter\n", + "print(hello, len(hello))" + ] + }, + { + "cell_type": "markdown", + "id": "0a58a167-5f3f-4d57-8aac-d2b27a756094", + "metadata": { + "tags": [] + }, + "source": [ + "字符串拼接:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "edbe74f8-8d03-491d-a34f-b19537328dd7", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "saDeaA7hL9fT", + "outputId": "2837d0ab-9ae5-4053-d087-bfa0af81c344", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hello world\n" + ] + } + ], + "source": [ + "hw = hello + ' ' + world # String concatenation\n", + "print(hw)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "651bbbd3-90e6-4dd2-9b87-178ea11db68e", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "Nji1_UjYL9fY", + "outputId": "0149b0ca-425a-4a34-8e24-8dff7080922e", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hello world 12\n" + ] + } + ], + "source": [ + "hw12 = '{} {} {}'.format(hello, world, 12) # string formatting\n", + "print(hw12)" + ] + }, + { + "cell_type": "markdown", + "id": "3a258cf4-aac1-4ccc-b3f5-3889c4180987", + "metadata": {}, + "source": [ + "最新版Python里(3.7以上)推荐使用“f-string”格式化字符串:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a76bcee8-fecd-4247-997c-8c23b19f7d4e", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hello, world 12\n" + ] + } + ], + "source": [ + "print(f'{hello}, {world} {12}')" + ] + }, + { + "cell_type": "markdown", + "id": "517f3f58-3252-4808-bc12-c2023540667d", + "metadata": { + "id": "bUpl35bIL9fc" + }, + "source": [ + "字符串对象有很多有用的方法;例如:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "0d0dcfa1-36b5-48c1-99d9-3406c6f8f81e", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 121 + }, + "id": "VOxGatlsL9fd", + "outputId": "ab009df3-8643-4d3e-f85f-a813b70db9cb", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello\n", + "HELLO\n", + " hello\n", + " hello \n", + "he(ell)(ell)o\n", + "world\n" + ] + } + ], + "source": [ + "s = \"hello\"\n", + "print(s.capitalize()) # Capitalize a string\n", + "print(s.upper()) # Convert a string to uppercase; prints \"HELLO\"\n", + "print(s.rjust(7)) # Right-justify a string, padding with spaces\n", + "print(s.center(7)) # Center a string, padding with spaces\n", + "print(s.replace('l', '(ell)')) # Replace all instances of one substring with another\n", + "print(' world '.strip()) # Strip leading and trailing whitespace" + ] + }, + { + "cell_type": "markdown", + "id": "aae8a237-8bde-4b8a-bcb0-45e9234fd8b2", + "metadata": { + "id": "06cayXLtL9fi" + }, + "source": [ + "你可以在[文档](https://docs.python.org/zh-cn/3.11/library/stdtypes.html#string-methods)中找到所有字符串方法的列表。" + ] + }, + { + "cell_type": "markdown", + "id": "8ce225b4-1283-42f1-a5da-0c07333bc94e", + "metadata": { + "id": "p-6hClFjL9fk", + "tags": [] + }, + "source": [ + "### 容器\n", + "\n", + "Python 包括几种内置的容器类型:列表、字典、集合和图元。" + ] + }, + { + "cell_type": "markdown", + "id": "3c63d495-90bb-4eae-af27-0a39dcdf31d8", + "metadata": { + "id": "UsIWOe0LL9fn" + }, + "source": [ + "#### 列表\n", + "\n", + "列表相当于数组,但是可以调整大小,并且可以包含不同类型的元素。" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "3fb8f690-2e82-4823-8e92-68171a82d44c", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "hk3A8pPcL9fp", + "outputId": "b545939a-580c-4356-db95-7ad3670b46e4", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[3, 1, 2] 2\n", + "2\n" + ] + } + ], + "source": [ + "xs = [3, 1, 2] # Create a list\n", + "print(xs, xs[2])\n", + "print(xs[-1]) # Negative indices count from the end of the list; prints \"2\"" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "97736c21-b355-431c-a9b4-005d9e46e3e2", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "YCjCy_0_L9ft", + "outputId": "417c54ff-170b-4372-9099-0f756f8e48af", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[3, 1, 'foo']\n" + ] + } + ], + "source": [ + "xs[2] = 'foo' # Lists can contain elements of different types\n", + "print(xs)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "ba24ceaf-92b3-4777-b162-6b56b36e8235", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "vJ0x5cF-L9fx", + "outputId": "a97731a3-70e1-4553-d9e0-2aea227cac80", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[3, 1, 'foo', 'bar']\n" + ] + } + ], + "source": [ + "xs.append('bar') # Add a new element to the end of the list\n", + "print(xs) " + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "22a59aee-504c-47be-838c-92514fa79abc", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "cxVCNRTNL9f1", + "outputId": "508fbe59-20aa-48b5-a1b2-f90363e7a104", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "bar [3, 1, 'foo']\n" + ] + } + ], + "source": [ + "x = xs.pop() # Remove and return the last element of the list\n", + "print(x, xs)" + ] + }, + { + "cell_type": "markdown", + "id": "2a8d5172-0e96-4b56-8dbd-0a3b99a2f386", + "metadata": {}, + "source": [ + "像往常一样,你可以在[文档](https://docs.python.org/zh-cn/3.11/tutorial/datastructures.html#more-on-lists)中找到关于列表的所有细节。" + ] + }, + { + "cell_type": "markdown", + "id": "3336e208-433f-42d0-9138-0263175c8b52", + "metadata": { + "id": "ovahhxd_L9f5" + }, + "source": [ + "#### 数据切片\n", + "\n", + "除了每次访问列表元素之外,Python 还提供了简洁的语法来访问子列表;这被称为切片。" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "23f6cd10-738b-441e-b8a4-1b365e21e0c9", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 139 + }, + "id": "ninq666bL9f6", + "outputId": "c3c2ed92-7358-4fdb-bbc0-e90f82e7e941", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 1, 2, 3, 4]\n", + "[2, 3]\n", + "[2, 3, 4]\n", + "[0, 1]\n", + "[0, 1, 2, 3, 4]\n", + "[0, 1, 2, 3]\n", + "[0, 1, 8, 9, 4]\n" + ] + } + ], + "source": [ + "nums = list(range(5)) # range is a built-in function that creates a list of integers\n", + "print(nums) # Prints \"[0, 1, 2, 3, 4]\"\n", + "print(nums[2:4]) # Get a slice from index 2 to 4 (exclusive); prints \"[2, 3]\"\n", + "print(nums[2:]) # Get a slice from index 2 to the end; prints \"[2, 3, 4]\"\n", + "print(nums[:2]) # Get a slice from the start to index 2 (exclusive); prints \"[0, 1]\"\n", + "print(nums[:]) # Get a slice of the whole list; prints [\"0, 1, 2, 3, 4]\"\n", + "print(nums[:-1]) # Slice indices can be negative; prints [\"0, 1, 2, 3]\"\n", + "nums[2:4] = [8, 9] # Assign a new sublist to a slice\n", + "print(nums) # Prints \"[0, 1, 8, 9, 4]\"" + ] + }, + { + "cell_type": "markdown", + "id": "e31c6e89-fd6e-4a4c-95d3-49a92e773d6d", + "metadata": { + "id": "UONpMhF4L9f_" + }, + "source": [ + "#### 遍历\n", + "\n", + "遍历列表元素在Python中也非常简洁:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2aa575f8-3016-453a-86aa-faf3b691f572", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + }, + "id": "4cCOysfWL9gA", + "outputId": "560e46c7-279c-409a-838c-64bea8d321c4", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cat\n", + "dog\n", + "monkey\n" + ] + } + ], + "source": [ + "animals = ['cat', 'dog', 'monkey']\n", + "for animal in animals:\n", + " print(animal)" + ] + }, + { + "cell_type": "markdown", + "id": "07ea5b44-6e3b-4662-b015-d4aae4c497a4", + "metadata": {}, + "source": [ + "如果你想访问一个循环体内每个元素的索引,请使用内置的`enumerate`函数。" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "ea3e8838-3dc8-4636-bf9c-ca5ee3445ae8", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + }, + "id": "JjGnDluWL9gF", + "outputId": "81421905-17ea-4c5a-bcc0-176de19fd9bd", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "#1: cat\n", + "#2: dog\n", + "#3: monkey\n" + ] + } + ], + "source": [ + "animals = ['cat', 'dog', 'monkey']\n", + "for idx, animal in enumerate(animals):\n", + " print('#{}: {}'.format(idx + 1, animal))" + ] + }, + { + "cell_type": "markdown", + "id": "ce76067a-6cb3-4742-892a-51094b1357f2", + "metadata": { + "id": "arrLCcMyL9gK" + }, + "source": [ + "#### 列表理解\n", + "\n", + "在编程时,我们经常想把一种类型的数据转换成另一种类型的数据。作为一个简单的例子,考虑以下计算平方数的代码。" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "5587ed10-64d0-49d2-b299-afbb34b4cab6", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "IVNEwoMXL9gL", + "outputId": "d571445b-055d-45f0-f800-24fd76ceec5a", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 1, 4, 9, 16]\n" + ] + } + ], + "source": [ + "nums = [0, 1, 2, 3, 4]\n", + "squares = []\n", + "for x in nums:\n", + " squares.append(x ** 2)\n", + "print(squares)" + ] + }, + { + "cell_type": "markdown", + "id": "2755e0f7-430f-4f5b-9f77-601ff6573027", + "metadata": { + "id": "7DmKVUFaL9gQ" + }, + "source": [ + "你可以用列表理解法使这段代码更简单。" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "a9d47d92-4383-41bb-be05-4c9d9af25362", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "kZxsUfV6L9gR", + "outputId": "4254a7d4-58ba-4f70-a963-20c46b485b72", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 1, 4, 9, 16]\n" + ] + } + ], + "source": [ + "nums = [0, 1, 2, 3, 4]\n", + "squares = [x ** 2 for x in nums]\n", + "print(squares)" + ] + }, + { + "cell_type": "markdown", + "id": "25f6e74b-15f4-4360-8f77-c9b5d48bc8d0", + "metadata": { + "id": "-D8ARK7tL9gV" + }, + "source": [ + "列表理解也可以包含条件:" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "c0e48840-6751-47ed-b1b1-3f26ba8d155a", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "yUtgOyyYL9gV", + "outputId": "1ae7ab58-8119-44dc-8e57-fda09197d026", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 4, 16]\n" + ] + } + ], + "source": [ + "nums = [0, 1, 2, 3, 4]\n", + "even_squares = [x ** 2 for x in nums if x % 2 == 0]\n", + "print(even_squares)" + ] + }, + { + "cell_type": "markdown", + "id": "7db9e9a4-6638-4d19-82f6-7f6c599e82d7", + "metadata": { + "id": "H8xsUEFpL9gZ", + "tags": [] + }, + "source": [ + "### 字典\n", + "\n", + "`dictionary` 存储对 `(key, value)`,类似于 Java 中的 `Map` 或 Javascript 中的对象。" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "550e661f-e598-4ec8-b6d0-d99bb81e6f53", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "XBYI1MrYL9gb", + "outputId": "8e24c1da-0fc0-4b4c-a3e6-6f758a53b7da", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cute\n", + "True\n" + ] + } + ], + "source": [ + "d = {'cat': 'cute', 'dog': 'furry'} # Create a new dictionary with some data\n", + "print(d['cat']) # Get an entry from a dictionary; prints \"cute\"\n", + "print('cat' in d) # Check if a dictionary has a given key; prints \"True\"" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "089c01a4-49d7-46a0-9fbc-2b54d9e7a051", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "pS7e-G-HL9gf", + "outputId": "feb4bf18-c0a3-42a2-eaf5-3fc390f36dcf", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "wet\n" + ] + } + ], + "source": [ + "d['fish'] = 'wet' # Set an entry in a dictionary\n", + "print(d['fish']) # Prints \"wet\"" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "8bd7abd2-3817-4210-b131-cbe653fe00ab", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 165 + }, + "id": "tFY065ItL9gi", + "outputId": "7e42a5f0-1856-4608-a927-0930ab37a66c", + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Traceback (most recent call last):\n", + " File \"/var/folders/zv/bzxgq5_j5f9gkpm84rg80kph0000gn/T/ipykernel_31837/1743455973.py\", line 3, in \n", + " print(d['monkey']) # KeyError: 'monkey' not a key of d\n", + "KeyError: 'monkey'\n" + ] + } + ], + "source": [ + "import traceback\n", + "try:\n", + " print(d['monkey']) # KeyError: 'monkey' not a key of d\n", + "except Exception as e:\n", + " traceback.print_exc()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "1097ca3d-f41d-484a-9aac-a3836df4e211", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "8TjbEWqML9gl", + "outputId": "ef14d05e-401d-4d23-ed1a-0fe6b4c77d6f", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "N/A\n", + "wet\n" + ] + } + ], + "source": [ + "print(d.get('monkey', 'N/A')) # Get an element with a default; prints \"N/A\"\n", + "print(d.get('fish', 'N/A')) # Get an element with a default; prints \"wet\"" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "6405b789-d95d-4e88-a5de-cf9416b36d3c", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "0EItdNBJL9go", + "outputId": "652a950f-b0c2-4623-98bd-0191b300cd57", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "N/A\n" + ] + } + ], + "source": [ + "del d['fish'] # Remove an element from a dictionary\n", + "print(d.get('fish', 'N/A')) # \"fish\" is no longer a key; prints \"N/A\"" + ] + }, + { + "cell_type": "markdown", + "id": "cbe19427-0004-4670-86af-ce7358f14689", + "metadata": { + "id": "IxwEqHlGL9gr" + }, + "source": [ + "遍历字典中的关键字:" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "4063d23c-ee46-4d50-8e89-d355d39d1d8c", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + }, + "id": "rYfz7ZKNL9gs", + "outputId": "155bdb17-3179-4292-c832-8166e955e942", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "A person has 2 legs\n", + "A cat has 4 legs\n", + "A spider has 8 legs\n" + ] + } + ], + "source": [ + "d = {'person': 2, 'cat': 4, 'spider': 8}\n", + "for animal, legs in d.items():\n", + " print('A {} has {} legs'.format(animal, legs))" + ] + }, + { + "cell_type": "markdown", + "id": "f6cfaebb-9616-4051-a29f-819f80b19ccd", + "metadata": { + "id": "17sxiOpzL9gz" + }, + "source": [ + "字典理解与列表理解类似,但允许你轻松构建字典:" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "5e892ea9-2373-46e7-b3b3-b090d58f6ec7", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "8PB07imLL9gz", + "outputId": "e9ddf886-39ed-4f35-dd80-64a19d2eec9b", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{0: 0, 2: 4, 4: 16}\n" + ] + } + ], + "source": [ + "nums = [0, 1, 2, 3, 4]\n", + "even_num_to_square = {x: x ** 2 for x in nums if x % 2 == 0}\n", + "print(even_num_to_square)" + ] + }, + { + "cell_type": "markdown", + "id": "bc64fa19-0405-47aa-aa0f-bf78e543115b", + "metadata": {}, + "source": [ + "你可以在[文档](https://docs.python.org/3/tutorial/datastructures.html#dictionaries)中找到所有你需要知道的关于字典的信息。" + ] + }, + { + "cell_type": "markdown", + "id": "ac1224a2-0c3d-4f9c-adcc-5c5c3883cc84", + "metadata": { + "id": "V9MHfUdvL9g2" + }, + "source": [ + "#### 集合\n", + "\n", + "集合是一个由不同元素组成的无序集合。" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "587cd6cc-03bf-49cc-875b-4bca003add02", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "MmyaniLsL9g2", + "outputId": "8f152d48-0a07-432a-cf98-8de4fd57ddbb", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "False\n" + ] + } + ], + "source": [ + "animals = {'cat', 'dog'}\n", + "print('cat' in animals) # Check if an element is in a set; prints \"True\"\n", + "print('fish' in animals) # prints \"False\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "646a17ac-74b7-4f25-b152-8642ede7088b", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "ElJEyK86L9g6", + "outputId": "b9d7dab9-5a98-41cd-efbc-786d0c4377f7", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "3\n" + ] + } + ], + "source": [ + "animals.add('fish') # Add an element to a set\n", + "print('fish' in animals)\n", + "print(len(animals)) # Number of elements in a set;" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "981d0004-8861-44da-9347-b0cd06908485", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "5uGmrxdPL9g9", + "outputId": "e644d24c-26c6-4b43-ab15-8aa81fe884d4", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3\n", + "2\n" + ] + } + ], + "source": [ + "animals.add('cat') # Adding an element that is already in the set does nothing\n", + "print(len(animals)) \n", + "animals.remove('cat') # Remove an element from a set\n", + "print(len(animals)) " + ] + }, + { + "cell_type": "markdown", + "id": "3830fb70-27dc-404e-9c49-a90e586f9008", + "metadata": { + "tags": [] + }, + "source": [ + "循环:对集合进行迭代的语法与对列表进行迭代的语法相同;但是由于集合是无序的,你不能对访问集合中的元素的顺序做出假设。" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "da338585-7f7e-4d94-a9a6-628133b50f94", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + }, + "id": "K47KYNGyL9hA", + "outputId": "4477f897-4355-4816-b39b-b93ffbac4bf0", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "#1: fish\n", + "#2: dog\n", + "#3: cat\n" + ] + } + ], + "source": [ + "animals = {'cat', 'dog', 'fish'}\n", + "for idx, animal in enumerate(animals):\n", + " print('#{}: {}'.format(idx + 1, animal))" + ] + }, + { + "cell_type": "markdown", + "id": "7a06bf28-d149-4b45-9df0-34da529fd78c", + "metadata": { + "id": "puq4S8buL9hC" + }, + "source": [ + "集合理解:像列表和字典一样,我们可以用集合理解法轻松地构建集合。" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "fea46a27-7797-46a3-b3b5-473f801481fc", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "iw7k90k3L9hC", + "outputId": "72d6b824-6d31-47b2-f929-4cf434590ee5", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{0, 1, 2, 3, 4, 5}\n" + ] + } + ], + "source": [ + "from math import sqrt\n", + "print({int(sqrt(x)) for x in range(30)})" + ] + }, + { + "cell_type": "markdown", + "id": "da599c6a-e1f7-46cd-8202-246ae074cd96", + "metadata": { + "id": "qPsHSKB1L9hF" + }, + "source": [ + "#### 元组\n", + "\n", + "元组是一个(不可变的)有序的数值列表。元组在许多方面与列表相似;最重要的区别之一是,元组可以作为字典的键和集合的元素,而列表则不能。" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "d7d2eccc-5f3f-4995-b685-398b60a7c027", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + }, + "id": "9wHUyTKxL9hH", + "outputId": "cdc5f620-04fe-4b0b-df7a-55b061d23d88", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "5\n", + "1\n" + ] + } + ], + "source": [ + "d = {(x, x + 1): x for x in range(10)} # Create a dictionary with tuple keys\n", + "t = (5, 6) # Create a tuple\n", + "print(type(t))\n", + "print(d[t]) \n", + "print(d[(1, 2)])" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "4637f799-d63d-4c44-ad59-be6317a23dca", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 165 + }, + "id": "HoO8zYKzL9hJ", + "outputId": "28862bfc-0298-40d7-f8c4-168e109d2d93", + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Traceback (most recent call last):\n", + " File \"/var/folders/zv/bzxgq5_j5f9gkpm84rg80kph0000gn/T/ipykernel_31837/743935956.py\", line 3, in \n", + " t[0] = 1 # TypeError: 'tuple' object does not support item assignment\n", + "TypeError: 'tuple' object does not support item assignment\n" + ] + } + ], + "source": [ + "import traceback\n", + "try:\n", + " t[0] = 1 # TypeError: 'tuple' object does not support item assignment\n", + "except Exception as e:\n", + " traceback.print_exc()" + ] + }, + { + "cell_type": "markdown", + "id": "58fd22de-586a-4214-a608-c915964b01af", + "metadata": { + "id": "AXA4jrEOL9hM", + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### 函数\n", + "\n", + "Python 函数是用 `def` 关键字定义的。" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "d1ff2700-4910-49d3-ae32-51ebd6fb83ba", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + }, + "id": "kiMDUr58L9hN", + "outputId": "9f53bf9a-7b2a-4c51-9def-398e4677cd6c", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "negative\n", + "zero\n", + "positive\n" + ] + } + ], + "source": [ + "def sign(x):\n", + " if x > 0:\n", + " return 'positive'\n", + " elif x < 0:\n", + " return 'negative'\n", + " else:\n", + " return 'zero'\n", + "\n", + "for x in [-1, 0, 1]:\n", + " print(sign(x))" + ] + }, + { + "cell_type": "markdown", + "id": "bfcc59de-5a62-4a95-a75f-4bd161238a91", + "metadata": {}, + "source": [ + "我们可以像这样定义函数来接受可选的关键字参数。" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "06d000b0-59bb-4a69-adaa-c93159f6661f", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "PfsZ3DazL9hR", + "outputId": "6e6af832-67d8-4d8c-949b-335927684ae3", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello, Bob!\n", + "HELLO, FRED\n" + ] + } + ], + "source": [ + "def hello(name, loud=False):\n", + " if loud:\n", + " print('HELLO, {}'.format(name.upper()))\n", + " else:\n", + " print('Hello, {}!'.format(name))\n", + "\n", + "hello('Bob')\n", + "hello('Fred', loud=True)" + ] + }, + { + "cell_type": "markdown", + "id": "b87320a3-5ca9-4c64-be86-1b026bf6b125", + "metadata": { + "id": "ObA9PRtQL9hT", + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### 类\n", + "\n", + "在Python中定义类的语法与其他面向对象语言类似。" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "60d14004-4a28-46d9-8861-f2236fa0c452", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "RWdbaGigL9hU", + "outputId": "4f6615c5-75a7-4ce4-8ea1-1e7f5e4e9fc3", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello, Fred!\n", + "HELLO, FRED\n" + ] + } + ], + "source": [ + "class Greeter:\n", + "\n", + " # Constructor\n", + " def __init__(self, name):\n", + " self.name = name # Create an instance variable\n", + "\n", + " # Instance method\n", + " def greet(self, loud=False):\n", + " if loud:\n", + " print('HELLO, {}'.format(self.name.upper()))\n", + " else:\n", + " print('Hello, {}!'.format(self.name))\n", + "\n", + "g = Greeter('Fred') # Construct an instance of the Greeter class\n", + "g.greet() # Call an instance method; prints \"Hello, Fred\"\n", + "g.greet(loud=True) # Call an instance method; prints \"HELLO, FRED!\"" + ] + }, + { + "cell_type": "markdown", + "id": "d52b89c1-b890-409f-80e5-3e321679d5e0", + "metadata": { + "id": "3cfrOV4dL9hW", + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "## Numpy" + ] + }, + { + "cell_type": "markdown", + "id": "9821dc0b-b68c-4309-958f-46abdae7f2a3", + "metadata": { + "id": "3cfrOV4dL9hW", + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "Numpy是Python中科学计算的核心库。它提供了一个高性能的多维数组对象,以及处理这些数组的工具。\n", + "\n", + "要使用Numpy,我们首先需要导入`numpy`包。" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "545df4bf-976e-42f8-be87-2b225846ca48", + "metadata": { + "id": "58QdX8BLL9hZ", + "tags": [] + }, + "outputs": [], + "source": [ + "import numpy as np" + ] + }, + { + "cell_type": "markdown", + "id": "83fec0fa-5c2d-4490-b045-5570471a986f", + "metadata": { + "id": "DDx6v1EdL9hb", + "tags": [] + }, + "source": [ + "### 数组\n", + "\n", + "numpy数组是一个由数值组成的网格,所有的数值都是相同的类型,并由一个非负整数的元组来索引。维数的数量是数组的等级;数组的形状是一个整数的元组,给出数组在每个维度上的大小。" + ] + }, + { + "cell_type": "markdown", + "id": "0bd6e576-1a11-432c-9b12-fee1a96bc1d4", + "metadata": {}, + "source": [ + "我们可以从嵌套的Python列表中初始化numpy数组,并使用方括号访问元素。" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "id": "6af27792-1ec3-48c6-a50e-2222092a30d0", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "-l3JrGxCL9hc", + "outputId": "8d9dad18-c734-4a8a-ca8c-44060a40fb79", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " (3,) 1 2 3\n", + "[5 2 3]\n" + ] + } + ], + "source": [ + "a = np.array([1, 2, 3]) # Create a rank 1 array\n", + "print(type(a), a.shape, a[0], a[1], a[2])\n", + "a[0] = 5 # Change an element of the array\n", + "print(a) " + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "f559d4a6-3589-4e81-8855-7b59f62a0873", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "ma6mk-kdL9hh", + "outputId": "0b54ff2f-e7f1-4b30-c653-9bf81cb8fbb0", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1 2 3]\n", + " [4 5 6]]\n" + ] + } + ], + "source": [ + "b = np.array([[1,2,3],[4,5,6]]) # Create a rank 2 array\n", + "print(b)" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "aae07be5-11b2-4d38-9b9b-cde518cb7bfa", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "ymfSHAwtL9hj", + "outputId": "5bd292d8-c751-43b9-d480-f357dde52342", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2, 3)\n", + "1 2 4\n" + ] + } + ], + "source": [ + "print(b.shape)\n", + "print(b[0, 0], b[0, 1], b[1, 0])" + ] + }, + { + "cell_type": "markdown", + "id": "686760a7-1358-4e70-9f87-d55c16f479b0", + "metadata": {}, + "source": [ + "Numpy还提供了许多创建数组的函数。" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "56264728-2c1d-4280-9fc4-2643a9ca0a40", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "mVTN_EBqL9hn", + "outputId": "d267c65f-ba90-4043-cedb-f468ab1bcc5d", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[0. 0.]\n", + " [0. 0.]]\n" + ] + } + ], + "source": [ + "a = np.zeros((2,2)) # Create an array of all zeros\n", + "print(a)" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "id": "131e2011-a8f8-4082-b234-e132284b2a81", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "skiKlNmlL9h5", + "outputId": "7d1ec1b5-a1fe-4f44-cbe3-cdeacad425f1", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1. 1.]]\n" + ] + } + ], + "source": [ + "b = np.ones((1,2)) # Create an array of all ones\n", + "print(b)" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "cfd9b07e-e046-4e93-91c3-b185b24304af", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "HtFsr03bL9h7", + "outputId": "2688b157-2fad-4fc6-f20b-8633207f0326", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[7 7]\n", + " [7 7]]\n" + ] + } + ], + "source": [ + "c = np.full((2,2), 7) # Create a constant array\n", + "print(c)" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "03a22b15-abd7-434d-b10f-a5407eba4f1e", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "-QcALHvkL9h9", + "outputId": "5035d6fe-cb7e-4222-c972-55fe23c9d4c0", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1. 0.]\n", + " [0. 1.]]\n" + ] + } + ], + "source": [ + "d = np.eye(2) # Create a 2x2 identity matrix\n", + "print(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "5657a511-97df-4277-b8f5-4e547002387e", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "RCpaYg9qL9iA", + "outputId": "25f0b387-39cf-42f3-8701-de860cc75e2e", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[0.00480263 0.8985954 ]\n", + " [0.19113444 0.57867786]]\n" + ] + } + ], + "source": [ + "e = np.random.random((2,2)) # Create an array filled with random values\n", + "print(e)" + ] + }, + { + "cell_type": "markdown", + "id": "428e2575-e475-4a49-99d9-6da69e0c72af", + "metadata": { + "id": "jI5qcSDfL9iC", + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### 数组索引\n", + "\n", + "切片:与Python列表类似,numpy数组可以被切片。由于数组可能是多维的,你必须为数组的每个维度指定一个分片。" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "id": "c5d07ae7-a2e9-468f-a4e9-e529addcedde", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "wLWA0udwL9iD", + "outputId": "99f08618-c513-4982-8982-b146fc72dab3", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[2 3]\n", + " [6 7]]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "\n", + "# Create the following rank 2 array with shape (3, 4)\n", + "# [[ 1 2 3 4]\n", + "# [ 5 6 7 8]\n", + "# [ 9 10 11 12]]\n", + "a = np.array([[1,2,3,4], [5,6,7,8], [9,10,11,12]])\n", + "\n", + "# Use slicing to pull out the subarray consisting of the first 2 rows\n", + "# and columns 1 and 2; b is the following array of shape (2, 2):\n", + "# [[2 3]\n", + "# [6 7]]\n", + "b = a[:2, 1:3]\n", + "print(b)" + ] + }, + { + "cell_type": "markdown", + "id": "76bb97fc-6a08-4803-8c03-183483453aa5", + "metadata": {}, + "source": [ + "一个数组的片断是对同一数据的视图,所以修改它将修改原始数组。" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "id": "405b781e-b00e-45e0-99bb-5da1984d6d95", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "1kmtaFHuL9iG", + "outputId": "ee3ab60c-4064-4a9e-b04c-453d3955f1d1", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2\n", + "77\n" + ] + } + ], + "source": [ + "print(a[0, 1])\n", + "b[0, 0] = 77 # b[0, 0] is the same piece of data as a[0, 1]\n", + "print(a[0, 1]) " + ] + }, + { + "cell_type": "markdown", + "id": "8f5ec9c8-0475-4758-ab0f-512539135edd", + "metadata": { + "id": "_Zcf3zi-L9iI" + }, + "source": [ + "你也可以把整数索引和片断索引混合起来。但是,这样做会产生一个比原数组秩更低的数组。" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "id": "fdccd1c2-0af6-426c-8e10-e61ea8b24013", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + }, + "id": "G6lfbPuxL9iJ", + "outputId": "a225fe9d-2a29-4e14-a243-2b7d583bd4bc", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 1 2 3 4]\n", + " [ 5 6 7 8]\n", + " [ 9 10 11 12]]\n" + ] + } + ], + "source": [ + "# Create the following rank 2 array with shape (3, 4)\n", + "a = np.array([[1,2,3,4], [5,6,7,8], [9,10,11,12]])\n", + "print(a)" + ] + }, + { + "cell_type": "markdown", + "id": "de488e13-2a40-4ff9-8e5a-58eb79e1b083", + "metadata": { + "id": "NCye3NXhL9iL" + }, + "source": [ + "访问数组中间行的数据的两种方法。将整数索引与分片混合使用会产生一个较低秩的数组,而只使用分片会产生一个与原数组秩相同的数组。" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "92c5d9e0-9b34-4e4d-b95d-2a97b4efe24c", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + }, + "id": "EOiEMsmNL9iL", + "outputId": "ab2ebe48-9002-45a8-9462-fd490b467f40", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[5 6 7 8] (4,)\n", + "[[5 6 7 8]] (1, 4)\n", + "[[5 6 7 8]] (1, 4)\n" + ] + } + ], + "source": [ + "row_r1 = a[1, :] # Rank 1 view of the second row of a \n", + "row_r2 = a[1:2, :] # Rank 2 view of the second row of a\n", + "row_r3 = a[[1], :] # Rank 2 view of the second row of a\n", + "print(row_r1, row_r1.shape)\n", + "print(row_r2, row_r2.shape)\n", + "print(row_r3, row_r3.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "8755f036-6798-4e90-ac7b-39f33f12d2e9", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 104 + }, + "id": "JXu73pfDL9iN", + "outputId": "6c589b85-e9b0-4c13-a39d-4cd9fb2f41ac", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ 2 6 10] (3,)\n", + "\n", + "[[ 2]\n", + " [ 6]\n", + " [10]] (3, 1)\n" + ] + } + ], + "source": [ + "# We can make the same distinction when accessing columns of an array:\n", + "col_r1 = a[:, 1]\n", + "col_r2 = a[:, 1:2]\n", + "print(col_r1, col_r1.shape)\n", + "print()\n", + "print(col_r2, col_r2.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "8a01a977-3298-43a2-b95d-8c20904c9f29", + "metadata": { + "id": "VP3916bOL9iP" + }, + "source": [ + "整数阵列的索引:当你使用切片法对numpy数组进行索引时,产生的数组视图总是原始数组的一个子数组。相比之下,整数数组索引允许你使用另一个数组的数据来构造任意的数组。" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "8f67a0dc-7f00-4618-876e-42d8fef94190", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "TBnWonIDL9iP", + "outputId": "c29fa2cd-234e-4765-c70a-6889acc63573", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[1 4 5]\n", + "[1 4 5]\n" + ] + } + ], + "source": [ + "a = np.array([[1,2], [3, 4], [5, 6]])\n", + "\n", + "# An example of integer array indexing.\n", + "# The returned array will have shape (3,) and \n", + "print(a[[0, 1, 2], [0, 1, 0]])\n", + "\n", + "# The above example of integer array indexing is equivalent to this:\n", + "print(np.array([a[0, 0], a[1, 1], a[2, 0]]))" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "21beba4c-b2ef-4431-8c01-835fd6424abc", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "n7vuati-L9iR", + "outputId": "c3e9ba14-f66e-4202-999e-2e1aed5bd631", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2 2]\n", + "[2 2]\n" + ] + } + ], + "source": [ + "# When using integer array indexing, you can reuse the same\n", + "# element from the source array:\n", + "print(a[[0, 0], [1, 1]])\n", + "\n", + "# Equivalent to the previous integer array indexing example\n", + "print(np.array([a[0, 1], a[0, 1]]))" + ] + }, + { + "cell_type": "markdown", + "id": "51b52118-7e6d-4ee0-bf6d-63b7c33d675d", + "metadata": { + "id": "kaipSLafL9iU" + }, + "source": [ + "整数数组索引的一个有用的技巧是从矩阵的每一行中选择或改变一个元素。\n" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "ce10fe00-c1d9-40d6-bb16-7d4e551f9cb2", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 86 + }, + "id": "ehqsV7TXL9iU", + "outputId": "de509c40-4ee4-4b7c-e75d-1a936a3350e7", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 1 2 3]\n", + " [ 4 5 6]\n", + " [ 7 8 9]\n", + " [10 11 12]]\n" + ] + } + ], + "source": [ + "# Create a new array from which we will select elements\n", + "a = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])\n", + "print(a)" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "d0da6687-a4aa-44fb-a887-5f5e8c8ef313", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "pAPOoqy5L9iV", + "outputId": "f812e29b-9218-4767-d3a8-e9854e754e68", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ 1 6 7 11]\n" + ] + } + ], + "source": [ + "# Create an array of indices\n", + "b = np.array([0, 2, 0, 1])\n", + "\n", + "# Select one element from each row of a using the indices in b\n", + "print(a[np.arange(4), b]) # Prints \"[ 1 6 7 11]\"" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "id": "061bee12-7c9d-4bbc-a446-23cd62b725ed", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 86 + }, + "id": "6v1PdI1DL9ib", + "outputId": "89f50f82-de1b-4417-e55c-edbc0ee07584", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[11 2 3]\n", + " [ 4 5 16]\n", + " [17 8 9]\n", + " [10 21 12]]\n" + ] + } + ], + "source": [ + "# Mutate one element from each row of a using the indices in b\n", + "a[np.arange(4), b] += 10\n", + "print(a)" + ] + }, + { + "cell_type": "markdown", + "id": "590a3bda-0be0-4fa7-808c-cb2d0bfe5baf", + "metadata": { + "id": "kaE8dBGgL9id" + }, + "source": [ + "布尔数组索引:布尔数组索引可以让你挑选出一个数组中的任意元素。这种类型的索引经常被用来选择一个数组中满足某些条件的元素。" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "id": "4b2f3a7d-28c4-4572-aa51-8d56f5087562", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + }, + "id": "32PusjtKL9id", + "outputId": "8782e8ec-b78d-44d7-8141-23e39750b854", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[False False]\n", + " [ True True]\n", + " [ True True]]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "\n", + "a = np.array([[1,2], [3, 4], [5, 6]])\n", + "\n", + "bool_idx = (a > 2) # Find the elements of a that are bigger than 2;\n", + " # this returns a numpy array of Booleans of the same\n", + " # shape as a, where each slot of bool_idx tells\n", + " # whether that element of a is > 2.\n", + "\n", + "print(bool_idx)" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "id": "bb96e409-bb51-4e52-ac33-ae7b9c3ea002", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "cb2IRMXaL9if", + "outputId": "5983f208-3738-472d-d6ab-11fe85b36c95", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[3 4 5 6]\n", + "[3 4 5 6]\n" + ] + } + ], + "source": [ + "# We use boolean array indexing to construct a rank 1 array\n", + "# consisting of the elements of a corresponding to the True values\n", + "# of bool_idx\n", + "print(a[bool_idx])\n", + "\n", + "# We can do all of the above in a single concise statement:\n", + "print(a[a > 2])" + ] + }, + { + "cell_type": "markdown", + "id": "73e45a4e-a384-49ee-ae12-806d743d2c12", + "metadata": { + "id": "CdofMonAL9ih" + }, + "source": [ + "为了简洁起见,我们省略了很多关于numpy数组索引的细节;如果你想知道更多,你应该阅读文档。" + ] + }, + { + "cell_type": "markdown", + "id": "ddc7baf1-cd19-4f61-96b6-aacbd8f2fa81", + "metadata": { + "id": "jTctwqdQL9ih", + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### 数据类型\n", + "\n", + "每个numpy数组都是由相同类型的元素组成的网格。Numpy提供了一大批数字数据类型,你可以用它们来构造数组。当你创建一个数组时,Numpy会尝试猜测一个数据类型,但是构造数组的函数通常也包括一个可选参数来明确指定数据类型。" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "id": "3fb87156-0f42-4797-9328-51213ea21af8", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "4za4O0m5L9ih", + "outputId": "2ea4fb80-a4df-43f9-c162-5665895c13ae", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "int64 float64 int64\n" + ] + } + ], + "source": [ + "x = np.array([1, 2]) # Let numpy choose the datatype\n", + "y = np.array([1.0, 2.0]) # Let numpy choose the datatype\n", + "z = np.array([1, 2], dtype=np.int64) # Force a particular datatype\n", + "\n", + "print(x.dtype, y.dtype, z.dtype)" + ] + }, + { + "cell_type": "markdown", + "id": "82fe5afc-fcc3-49c9-97af-fd2a9372a6cc", + "metadata": { + "id": "RLVIsZQpL9ik" + }, + "source": [ + "你可以在[文档](http://docs.scipy.org/doc/numpy/reference/arrays.dtypes.html)中阅读关于numpy数据类型的所有信息。" + ] + }, + { + "cell_type": "markdown", + "id": "32476fb7-f4d1-4011-9cb9-36c9688e01a1", + "metadata": { + "id": "TuB-fdhIL9ik", + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### 数组的数学运算\n", + "\n", + "基本数学函数对数组进行元素操作,既可以作为运算符重载,也可以作为numpy模块的函数。" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "id": "4d38715d-b81e-4087-961b-5aef0e1253c3", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 86 + }, + "id": "gHKvBrSKL9il", + "outputId": "a8a924b1-9d60-4b68-8fd3-e4657ae3f08b", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 6. 8.]\n", + " [10. 12.]]\n", + "[[ 6. 8.]\n", + " [10. 12.]]\n" + ] + } + ], + "source": [ + "x = np.array([[1,2],[3,4]], dtype=np.float64)\n", + "y = np.array([[5,6],[7,8]], dtype=np.float64)\n", + "\n", + "# Elementwise sum; both produce the array\n", + "print(x + y)\n", + "print(np.add(x, y))" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "id": "f39f9dff-c4a7-4520-8429-5b320ac7824d", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 86 + }, + "id": "1fZtIAMxL9in", + "outputId": "122f1380-6144-4d6c-9d31-f62d839889a2", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[-4. -4.]\n", + " [-4. -4.]]\n", + "[[-4. -4.]\n", + " [-4. -4.]]\n" + ] + } + ], + "source": [ + "# Elementwise difference; both produce the array\n", + "print(x - y)\n", + "print(np.subtract(x, y))" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "id": "faceeb11-2c18-4351-a876-d86f2ab5bc3d", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 86 + }, + "id": "nil4AScML9io", + "outputId": "038c8bb2-122b-4e59-c0a8-a091014fe68e", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 5. 12.]\n", + " [21. 32.]]\n", + "[[ 5. 12.]\n", + " [21. 32.]]\n" + ] + } + ], + "source": [ + "# Elementwise product; both produce the array\n", + "print(x * y)\n", + "print(np.multiply(x, y))" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "id": "7a9ac1a8-23ae-4b45-bd98-fad524121d5e", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 86 + }, + "id": "0JoA4lH6L9ip", + "outputId": "12351a74-7871-4bc2-97ce-a508bf4810da", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[0.2 0.33333333]\n", + " [0.42857143 0.5 ]]\n", + "[[0.2 0.33333333]\n", + " [0.42857143 0.5 ]]\n" + ] + } + ], + "source": [ + "# Elementwise division; both produce the array\n", + "# [[ 0.2 0.33333333]\n", + "# [ 0.42857143 0.5 ]]\n", + "print(x / y)\n", + "print(np.divide(x, y))" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "id": "10c74001-044f-4e48-a398-dff64e1e62e4", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "g0iZuA6bL9ir", + "outputId": "29927dda-4167-4aa8-fbda-9008b09e4356", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1. 1.41421356]\n", + " [1.73205081 2. ]]\n" + ] + } + ], + "source": [ + "# Elementwise square root; produces the array\n", + "# [[ 1. 1.41421356]\n", + "# [ 1.73205081 2. ]]\n", + "print(np.sqrt(x))" + ] + }, + { + "cell_type": "markdown", + "id": "2fe65ae4-0a57-410f-a3f0-e7b202865244", + "metadata": { + "id": "a5d_uujuL9it" + }, + "source": [ + "注意,与MATLAB不同,`*`是元素乘法,而不是矩阵乘法。我们使用`dot`函数来计算向量的内积,用一个向量乘以一个矩阵,以及乘以矩阵。" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "id": "738b4437-fd7b-4061-876d-ae2432d5f59e", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "I3FnmoSeL9iu", + "outputId": "46f4575a-2e5e-4347-a34e-0cc5bd280110", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "219\n", + "219\n" + ] + } + ], + "source": [ + "x = np.array([[1,2],[3,4]])\n", + "y = np.array([[5,6],[7,8]])\n", + "\n", + "v = np.array([9,10])\n", + "w = np.array([11, 12])\n", + "\n", + "# Inner product of vectors; both produce 219\n", + "print(v.dot(w))\n", + "print(np.dot(v, w))" + ] + }, + { + "cell_type": "markdown", + "id": "8403d5cc-786e-4a09-933e-c0801480d831", + "metadata": { + "id": "vmxPbrHASVeA" + }, + "source": [ + "你也可以使用`@`运算符,它等同于numpy的`dot`运算符。" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "id": "7f1584a4-69ae-4d98-8648-bba0c9fce512", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "vyrWA-mXSdtt", + "outputId": "a9aae545-2c93-4649-b220-b097655955f6", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "219\n" + ] + } + ], + "source": [ + "print(v @ w)" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "id": "631c889d-5b47-4050-93ae-2aeccea37a28", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + }, + "id": "zvUODeTxL9iw", + "outputId": "4093fc76-094f-4453-a421-a212b5226968", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[29 67]\n", + "[29 67]\n", + "[29 67]\n" + ] + } + ], + "source": [ + "# Matrix / vector product; both produce the rank 1 array [29 67]\n", + "print(x.dot(v))\n", + "print(np.dot(x, v))\n", + "print(x @ v)" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "id": "bcf85865-0aba-4873-bb12-e01459aed85d", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 121 + }, + "id": "3V_3NzNEL9iy", + "outputId": "af2a89f9-af5d-47a6-9ad2-06a84b521b94", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[19 22]\n", + " [43 50]]\n", + "[[19 22]\n", + " [43 50]]\n", + "[[19 22]\n", + " [43 50]]\n" + ] + } + ], + "source": [ + "# Matrix / matrix product; both produce the rank 2 array\n", + "# [[19 22]\n", + "# [43 50]]\n", + "print(x.dot(y))\n", + "print(np.dot(x, y))\n", + "print(x @ y)" + ] + }, + { + "cell_type": "markdown", + "id": "b2d179c8-4696-4ccb-8ab4-32149ba8c196", + "metadata": { + "id": "FbE-1If_L9i0" + }, + "source": [ + "Numpy提供了许多有用的函数来对数组进行计算,其中最有用的一个是`sum`。" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "id": "b4e13eaf-776d-4d58-9d7a-47bf5287d30d", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + }, + "id": "DZUdZvPrL9i0", + "outputId": "99cad470-d692-4b25-91c9-a57aa25f4c6e", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10\n", + "[4 6]\n", + "[3 7]\n" + ] + } + ], + "source": [ + "x = np.array([[1,2],[3,4]])\n", + "\n", + "print(np.sum(x)) # Compute sum of all elements; prints \"10\"\n", + "print(np.sum(x, axis=0)) # Compute sum of each column; prints \"[4 6]\"\n", + "print(np.sum(x, axis=1)) # Compute sum of each row; prints \"[3 7]\"" + ] + }, + { + "cell_type": "markdown", + "id": "9115fd7c-075e-4938-9597-8f5b5c6ac332", + "metadata": { + "id": "ahdVW4iUL9i3" + }, + "source": [ + "除了使用数组计算数学函数外,我们还经常需要对数组中的数据进行重塑或其他操作。这类操作的最简单例子是转置矩阵;要转置矩阵,只需使用数组对象的`T`属性。" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "id": "9740187f-5fdb-407b-9333-33820eaf00c2", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 104 + }, + "id": "63Yl1f3oL9i3", + "outputId": "c75ac7ba-4351-42f8-a09c-a4e0d966ab50", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1 2]\n", + " [3 4]]\n", + "transpose\n", + " [[1 3]\n", + " [2 4]]\n" + ] + } + ], + "source": [ + "print(x)\n", + "print(\"transpose\\n\", x.T)" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "id": "9bd47a46-d582-414d-a09a-6543fac10786", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 104 + }, + "id": "mkk03eNIL9i4", + "outputId": "499eec5a-55b7-473a-d4aa-9d023d63885a", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1 2 3]]\n", + "transpose\n", + " [[1]\n", + " [2]\n", + " [3]]\n" + ] + } + ], + "source": [ + "v = np.array([[1,2,3]])\n", + "print(v )\n", + "print(\"transpose\\n\", v.T)" + ] + }, + { + "cell_type": "markdown", + "id": "d9358129-29e5-4010-a080-f822609bc7f9", + "metadata": { + "id": "REfLrUTcL9i7", + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### 广播\n", + "\n", + "广播是一种强大的机制,它允许numpy在进行算术运算时与不同形状的数组一起工作。通常我们有一个较小的数组和一个较大的数组,我们想多次使用较小的数组来对较大的数组进行一些操作。\n", + "\n", + "例如,假设我们想给一个矩阵的每一行添加一个常数向量。" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "id": "4c6582f2-457c-45e7-964d-40207fca952b", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 86 + }, + "id": "WEEvkV1ZL9i7", + "outputId": "3896d03c-3ece-4aa8-f675-aef3a220574d", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 2 2 4]\n", + " [ 5 5 7]\n", + " [ 8 8 10]\n", + " [11 11 13]]\n" + ] + } + ], + "source": [ + "# We will add the vector v to each row of the matrix x,\n", + "# storing the result in the matrix y\n", + "x = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])\n", + "v = np.array([1, 0, 1])\n", + "y = np.empty_like(x) # Create an empty matrix with the same shape as x\n", + "\n", + "# Add the vector v to each row of the matrix x with an explicit loop\n", + "for i in range(4):\n", + " y[i, :] = x[i, :] + v\n", + "\n", + "print(y)" + ] + }, + { + "cell_type": "markdown", + "id": "8b4e7921-7cc3-4dc6-95c2-cf6de3060d87", + "metadata": { + "id": "2OlXXupEL9i-" + }, + "source": [ + "这个方法是可行的;但是当矩阵`x`非常大时,在Python中计算一个显式循环可能会很慢。请注意,将向量`v`添加到矩阵`x`的每一行,相当于通过垂直堆叠多个副本形成矩阵`vv`,然后对`x`和`vv`进行元素求和。 我们可以这样实现这个方法。" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "id": "b0c5d546-27b0-4bc9-aaf3-a4a92580e1f2", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 86 + }, + "id": "vS7UwAQQL9i-", + "outputId": "8621e502-c25d-4a18-c973-886dbfd1df36", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[1 0 1]\n", + " [1 0 1]\n", + " [1 0 1]\n", + " [1 0 1]]\n" + ] + } + ], + "source": [ + "vv = np.tile(v, (4, 1)) # Stack 4 copies of v on top of each other\n", + "print(vv) # Prints \"[[1 0 1]\n", + " # [1 0 1]\n", + " # [1 0 1]\n", + " # [1 0 1]]\"" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "id": "55015b0b-2113-4265-a933-94f125cc2b05", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 86 + }, + "id": "N0hJphSIL9jA", + "outputId": "def6a757-170c-43bf-8728-732dfb133273", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 2 2 4]\n", + " [ 5 5 7]\n", + " [ 8 8 10]\n", + " [11 11 13]]\n" + ] + } + ], + "source": [ + "y = x + vv # Add x and vv elementwise\n", + "print(y)" + ] + }, + { + "cell_type": "markdown", + "id": "27c8605f-5c3e-48ce-a7af-fb817860e4b3", + "metadata": { + "id": "zHos6RJnL9jB" + }, + "source": [ + "Numpy广播允许我们在不实际创建`v`的多个副本的情况下进行这种计算。" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "id": "3aa583c2-7776-420a-8a2a-91d721ac6d60", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 86 + }, + "id": "vnYFb-gYL9jC", + "outputId": "df3bea8a-ad72-4a83-90bb-306b55c6fb93", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 2 2 4]\n", + " [ 5 5 7]\n", + " [ 8 8 10]\n", + " [11 11 13]]\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "\n", + "# We will add the vector v to each row of the matrix x,\n", + "# storing the result in the matrix y\n", + "x = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])\n", + "v = np.array([1, 0, 1])\n", + "y = x + v # Add v to each row of x using broadcasting\n", + "print(y)" + ] + }, + { + "cell_type": "markdown", + "id": "65a5ce1e-caac-431d-9bce-bdf2cd8498cf", + "metadata": {}, + "source": [ + "尽管x的形状是(4, 3),而v的形状是(3,),但由于广播的原因,这行代码`y = x + v`是有效的;这行代码的作用就像v实际上有形状(4, 3)一样,其中每一行都是v的一个副本,并且按元素进行求和。" + ] + }, + { + "cell_type": "markdown", + "id": "5d2ffca7-fc4f-4588-a6d0-f5b15a11ea80", + "metadata": {}, + "source": [ + "两个数组一起广播时要遵循这些规则。\n", + "\n", + "1. 如果数组没有相同的秩,则在低秩数组的形状前加上1,直到两个形状具有相同的长度。\n", + "2. 如果两个数组在某一维度上有相同的大小,或者其中一个数组在该维度上有1的大小,则称这两个数组在该维度上兼容。\n", + "3. 如果这两个数组在所有维度上都是兼容的,就可以一起广播。\n", + "4. 在广播之后,每个数组的行为就像它的形状等于两个输入数组形状的元素最大值一样。\n", + "5. 在任何一个维度上,如果一个数组的大小为1,而另一个数组的大小大于1,那么第一个数组的行为就像是沿着该维度复制的一样\n", + "\n", + "尝试阅读[文档](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)中的解释来理解。" + ] + }, + { + "cell_type": "markdown", + "id": "749e267b-bab9-424c-b12c-a56c53450ce6", + "metadata": {}, + "source": [ + "支持广播的函数被称为通用函数。你可以在[文档](http://docs.scipy.org/doc/numpy/reference/ufuncs.html#available-ufuncs)中找到所有通用函数的列表。\n", + "\n", + "下面是广播的一些应用。" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "id": "bd7f1a13-b4cf-4fb5-9a16-7dfeb2f709bd", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 69 + }, + "id": "EmQnwoM9L9jH", + "outputId": "f59e181e-e2d4-416c-d094-c4d003ce8509", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 4 5]\n", + " [ 8 10]\n", + " [12 15]]\n" + ] + } + ], + "source": [ + "# Compute outer product of vectors\n", + "v = np.array([1,2,3]) # v has shape (3,)\n", + "w = np.array([4,5]) # w has shape (2,)\n", + "# To compute an outer product, we first reshape v to be a column\n", + "# vector of shape (3, 1); we can then broadcast it against w to yield\n", + "# an output of shape (3, 2), which is the outer product of v and w:\n", + "\n", + "print(np.reshape(v, (3, 1)) * w)" + ] + }, + { + "cell_type": "code", + "execution_count": 86, + "id": "46c50dd9-8d9b-4049-a3fa-be0544153703", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "PgotmpcnL9jK", + "outputId": "567763d3-073a-4e3c-9ebe-6c7d2b6d3446", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[2 4 6]\n", + " [5 7 9]]\n" + ] + } + ], + "source": [ + "# Add a vector to each row of a matrix\n", + "x = np.array([[1,2,3], [4,5,6]])\n", + "# x has shape (2, 3) and v has shape (3,) so they broadcast to (2, 3),\n", + "# giving the following matrix:\n", + "\n", + "print(x + v)" + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "id": "95d83311-482d-402c-ac36-11d790d2e5bb", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "T5hKS1QaL9jK", + "outputId": "5f14ac5c-7a21-4216-e91d-cfce5720a804", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 5 6 7]\n", + " [ 9 10 11]]\n" + ] + } + ], + "source": [ + "# Add a vector to each column of a matrix\n", + "# x has shape (2, 3) and w has shape (2,).\n", + "# If we transpose x then it has shape (3, 2) and can be broadcast\n", + "# against w to yield a result of shape (3, 2); transposing this result\n", + "# yields the final result of shape (2, 3) which is the matrix x with\n", + "# the vector w added to each column. Gives the following matrix:\n", + "\n", + "print((x.T + w).T)" + ] + }, + { + "cell_type": "code", + "execution_count": 88, + "id": "6ffa86e6-9c42-4e5a-b061-da2a2f44d60a", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "JDUrZUl6L9jN", + "outputId": "53e99a89-c599-406d-9fe3-7aa35ae5fb90", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 5 6 7]\n", + " [ 9 10 11]]\n" + ] + } + ], + "source": [ + "# Another solution is to reshape w to be a row vector of shape (2, 1);\n", + "# we can then broadcast it directly against x to produce the same\n", + "# output.\n", + "print(x + np.reshape(w, (2, 1)))" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "id": "f1cc377a-c728-4313-8174-38d9663939d0", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 52 + }, + "id": "VzrEo4KGL9jP", + "outputId": "53c9d4cc-32d5-46b0-d090-53c7db57fb32", + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 2 4 6]\n", + " [ 8 10 12]]\n" + ] + } + ], + "source": [ + "# Multiply a matrix by a constant:\n", + "# x has shape (2, 3). Numpy treats scalars as arrays of shape ();\n", + "# these can be broadcast together to shape (2, 3), producing the\n", + "# following array:\n", + "print(x * 2)" + ] + }, + { + "cell_type": "markdown", + "id": "79790419-5ae6-4fe1-924e-f1aa0b241da2", + "metadata": {}, + "source": [ + "广播通常会使你的代码更加简洁和快速,所以你应该尽可能地使用它。" + ] + }, + { + "cell_type": "markdown", + "id": "b44df6e5-3098-485d-b498-84372e82176d", + "metadata": { + "id": "tEINf4bEL9jR", + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "## Matplotlib" + ] + }, + { + "cell_type": "markdown", + "id": "e67f32eb-595a-48e8-8b2a-e375d6977a2b", + "metadata": { + "id": "tEINf4bEL9jR", + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "Matplotlib是一个绘图库。在本节中简要介绍`matplotlib.pyplot`模块,它提供了一个类似于MATLAB的绘图系统。" + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "id": "bc9e9400-1931-41c6-8016-726cba8df1ac", + "metadata": { + "id": "cmh_7c6KL9jR", + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "markdown", + "id": "7d0b6894-c2d9-48b9-a201-5ed76192a144", + "metadata": { + "id": "jOsaA5hGL9jS" + }, + "source": [ + "> 注意:通过运行这个特殊的iPython命令,我们将内联显示图画。" + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "id": "2e564fc4-9270-45fe-8767-34baff096633", + "metadata": { + "id": "ijpsmwGnL9jT", + "tags": [] + }, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "id": "fde628ec-c878-440f-8ccb-970af9696791", + "metadata": { + "id": "U5Z_oMoLL9jV" + }, + "source": [ + "### `plot`绘图\n", + "\n", + "`matplotlib`中最重要的函数是`plot`,它允许你绘制2D数据。下面是一个简单的例子。" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "id": "acf23905-201c-4c3c-aa34-8525b857fd3a", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "pua52BGeL9jW", + "outputId": "9ac3ee0f-7ff7-463b-b901-c33d21a2b10c", + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 92, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAGdCAYAAAAfTAk2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABf5klEQVR4nO3deXhTZdo/8G/2dE1b0p3SlrWFspa1LLIWEFTUYRm16AwyMuooMM44uLyv+o7yc2ZUZhxxRxxFxBERVEDKvpW9ZS2lrC3dW9qka9Ik5/dHmkBtKS00PVm+n+vKH6QnJ3cKnNzneZ77fiSCIAggIiIiciNSsQMgIiIiam9McIiIiMjtMMEhIiIit8MEh4iIiNwOExwiIiJyO0xwiIiIyO0wwSEiIiK3wwSHiIiI3I5c7ADEYLFYkJ+fDz8/P0gkErHDISIiolYQBAGVlZWIiIiAVNryGI1HJjj5+fmIiooSOwwiIiK6Dbm5uejcuXOLx3hkguPn5wfA+gvy9/cXORoiIiJqDb1ej6ioKPv3eEs8MsGxTUv5+/szwSEiInIxrVlewkXGRERE5HaY4BAREZHbYYJDREREbocJDhEREbkdJjhERETkdpjgEBERkdthgkNERERuhwkOERERuR0mOEREROR2HJrg7N69G/fccw8iIiIgkUjw/fff3/I1u3btQmJiItRqNbp27YoPPvigyTFr165F7969oVKp0Lt3b6xbt84B0RMREZGrcmiCU11djf79++Pf//53q46/dOkS7r77bowePRrp6el44YUX8Mwzz2Dt2rX2Y9LS0jB79mykpKTg+PHjSElJwaxZs3Dw4EFHfQwiIiJyMRJBEIQOeSOJBOvWrcOMGTNueszzzz+PDRs2IDMz0/7cggULcPz4caSlpQEAZs+eDb1ej02bNtmPmTJlCgIDA7F69epWxaLX66HRaKDT6bgXFRERkYtoy/e3U222mZaWhuTk5EbPTZ48GZ9++inq6+uhUCiQlpaGRYsWNTlm2bJlNz2vwWCAwWCw/1mv17dr3NR6uddqcLmsGhU19aioMaK8ph5VBhN6hvphTA8tQvzVYodIRGRntgg4eKkMl0trYLZYYLYIMFkEWAQB3YJ9MaqHFiq5TOwwqRlOleAUFhYiNDS00XOhoaEwmUwoLS1FeHj4TY8pLCy86XmXLl2KV1991SEx062ZzBZszSzGf9IuY/+FshaPjQvzw109g3FXr2AMj+0EqfTWO8YSEbW3rMJKfJd+FevT81Gor7vpcX5qOZJ7h2F6v3CM7K6FUs7aHWfhVAkO0HQLdNsM2o3PN3dMS1unL1myBIsXL7b/Wa/XIyoqqj3CpRZcqzZi9aEcrDpwBfk66wVCKgG6h/giwFuJQG8FAr2VUMmlOJZTgZN5OpwtrMTZwkp8uPsihsYE4fX7E9Aj1E/kT0JEnuKH4/l4f+cFnCm4PtLvr5ZjSEwQlHIppFIJ5FIJBAE4cLEMxZUGrD12FWuPXYW/Wo7fjemKBXd1g1zGREdsTpXghIWFNRmJKS4uhlwuR6dOnVo85pejOjdSqVRQqVTtHzDd1I6sYixak4GKmnoAQJCPEnOGROHh4dGIDPBq9jVlVQbsPV+K3edKsfFkAQ5dvoa7/7UHC+7qhqfGdYdawWFgInIMfV09Xv7+FNZn5AMAFDIJxvUKwf0DIzEuLqTZ64/FIuDIlXL8dCIfG08VoqTSgH9sOYfUM0V4a9YAdA/x7eiPQTdwukXGP/zwA86cOWN/7ve//z0yMjIaLTKurKzExo0b7cdMnToVAQEBXGTsBMwWAcu2nsO7288DAHqF+uF3Y7piWr/wNiUoV8tr8MqG09iaWQwAiOnkjb/O6ItRPbQOiZuIPNfhy9ew8OsM5FXUQiaV4Kmx3fCbkbEI9FG2+hxmi4D1GXl4ZcNp6OtMUMml+POUOPwmKYZT7e2oLd/fDk1wqqqqcP689Ytu4MCBePvttzFu3DgEBQWhS5cuWLJkCfLy8vCf//wHgLVMPCEhAU888QTmz5+PtLQ0LFiwAKtXr8aDDz4IANi/fz/GjBmD119/Hffddx/Wr1+Pl156CXv37sWwYcNaFRcTHMcorTLg2a/Tse+8dZ1NyvBovDQ9/rYX4AmCgJ9PF+GVDaftc+D/M703fjsqtt1iJiLPVW+24F/bsvHejvOwCECXIG+8M3sAEqMDb/ucBbpaPL/2JHafKwEADIsNwru/HsgCinbiNAnOzp07MW7cuCbPP/roo1i5ciUee+wxXL58GTt37rT/bNeuXVi0aBFOnz6NiIgIPP/881iwYEGj13/77bd46aWXcPHiRXTr1g2vv/46HnjggVbHxQSn/R3LKcfvvzyKIr0BXgoZ/t+DfXHfgMh2OXdlXT3e2HgWqw/lAACenxKH34/t1i7nJiLPVG+24MlVx5B6pggA8KvEznjl3j7wVd35yg1BELDqYA7e2JiJGqMZXYN98PXvhiPEj0nOnXKaBMdZMcFpX6fzdZjz4QFUGkzoFuyDDx5JbPeFwYIgYNnWbPxzWzYA4NkJPbBwYo8WF5cTETXHbBGwaE0GNhzPh1IuxVsz++Oe/hHt/j6XSqvx8McHkK+rQ48QX3z9u+Ho5Mv1oHeiLd/fXOZNdySnrAaPfXYYlQYThsUGYf3ToxxS9SSRSLBoUk/8eUovAMA/t2Xj/20+Cw/Mz4noDlgsAl747iQ2HM+HXCrBB48MckhyAwCxWh+s/t1whPqrkF1chYc/OYjyaqND3ouaYoJDt620yoC5Kw6ipNKAuDA/fPzo4HYZ3m3Jk2O74+XpvQEAH+66iNd+PMMkh4haRRAEvPbjGaw5kgupBPjnnIEYH3fzCtz2EN3JB6vnD0ewnwpnCyvxyKcHoWuoLiXHYoJDt6XKYMJvPjuMy2U16Bzohc9/OxT+akWHvPe8UbH464wEAMBn+y7jq4a1OURELfnHliys3H8ZAPC3X/XHtH7hHfK+XYN9sXr+MGh9lTidr0fKioOoNpg65L09GRMcajOjyYIFXxzFyTwdgnyU+M9vhyK0gysEHhkejb9MjQMAvLrhDE5crejQ9yci17L6UA7e23EBAPB/MxLwq8TOHfr+3UP8sOrx4QjyUeLEVR1e/v5Uh76/J2KCQ2322o+nsfd8KbyVMnz22BB0DRanmdUTY7oiuXcojGYLfv/lMc5tE1Gzsosq8eoPpwEAf5zUEynDo0WJo1eYHz5MSYRMKsF36Xn49uhVUeLwFExwqE22ny3ClwesU0LvPTQI/aMCRItFIpHg7zP7I7qTN/IqarHomwxYLFyPQ0TX1dWb8YfV6airt2B0Dy2eGtdd1HiGxARh0cQeAICXvz+F88WVosbjzpjgUKuVVRnw529PAgB+OzIW4+JCRI4I0Hgp8P7DiVDJpdiZVYJ/7zgvdkhE5ESWbszE2cJKaH2VeGtWf6foKvz7sd0xqrsWtfVmPP1VOurqzWKH5JaY4FCrCIKAv3x3EqVVBvQM9bWXazuD3hH+9kXH72w9Z+8gSkSeLfVMET5PuwIA+MfM/k7TaE8mleDt2f2h9VXibGElXvvxzK1fRG3GBIda5ZsjuUg9UwSFTIJlswc63caXMwdH4ddDoyAIwOJvjkNXyzJMIk9WqKvDn749DgCYPzoWY3uJP+J8oxA/Nd6ZPQASCfDVwRz8eCJf7JDcDhMcuqUrZdV49QfrHcYfk3uhd4Rzdn/+33v6oGuwD0qrDPjHz1lih0NEIrFYBCxck46Kmnr0jdTgT5PjxA6pWaN7BOPJhm1nlqw9iaKGPfeofTDBoRaZzBYsWpOBGqMZQ2ODMH90V7FDuim1Qmafqvry4BVk5FaIGxARieKbI7k4cPEavJUy/OvXA6GUO+9X3aKJPdG/swaVBhPe2JgpdjhuxXn/1skpfJ52BcdyKuCnkuPtWf0hc4IFei1J6qbFAwMjIQjAC9+dhMlsETskIupAupp6/K1hBPePyb0Qq/UROaKWyWVS/N+MBEgkwPqMfBy4WCZ2SG6DCQ7dVFmVAcu2ngMALLk7Hp0DvUWOqHVemBYPjZcCZwr09q6lROQZ3k7NwrVqI3qE+GLuCHH63bRVv84BeGhoFwDA/6w/hXremLULJjh0U2+nnkNlnQnx4f6YPSRK7HBaTeursnc5fjv1HPIrakWOiIg6QmaBHl8csFZNvXpvHyhkrvMV96fJvRDorcC5oip8zhuzduE6f/vUoc4W6rG6YY+n/72nt9NPTf3S7MFRGBwdiBqjGa9sOC12OETkYIIg4H/Xn4ZFAKb1DUdSd63YIbVJgLfSfmO2bGs2Fxy3AyY41IQgCPi/H8/AIgBTE8IwvGsnsUNqM6lUgr/enwC5VIItZ4qQeqZI7JCIyIE2HM/HocvXoFZI8cK0eLHDuS0zE6MwICoAVVxw3C6Y4FATqWeKsO98GZRyKV642zUvFAAQF+aPeaNjAVi7mXLBMZF7qr4hIXh6XHdEBniJHNHtkUol+L/7uOC4vTDBoUYMJjNeb7hQPD4qFlFBrrGw+GaeHtcdgd4KXCytxnfH8sQOh4gc4N3t51GkN6BLkDced+JWFq3Rt7MGDw+zLjh+ZcNp7q93B5jgUCMr913GlbIaBPup8KTIm9K1Bz+1Ak+OtX6Of27LhsHEPV+I3El+RS1W7L0EwLpe0Nm6rN+O55J7wU8tx9nCSmw8VSB2OC6LCQ7ZXas24t3t1s0q/zy5F3xVcpEjah8pI6IR6q9CXkUtVh/METscImpHy3eeh9FswbDYIEyIDxU7nHYR4K3E46OsI1HLtmbDzFGc28IEh+w+2XMRVQYT+kT448FBncUOp92oFTL8YXwPAMC/d1xAjdEkckRE1B7yKmqx5nAuAGDRpJ4iR9O+fjMqBhovBc4XV+GH49yn6nYwwSEAQEWN0d57YeHEnpC6WFn4rcwaHIWoIC+UVhnw+f4rYodDRO1g+Y7zqDcLGNG1k0tWe7bEX63A78ZYR3H+uS2bRRK3gQkOAQBW7L2EaqMZ8eH+mBjvXLvutgelXIpFE613eB/susDdxolc3NXyGnxzxDp6s3BiD5GjcYxHk2IQ6K3ApdJqfJ/BUZy2YoJD0NXW47N9lwEAz4zvDonEvUZvbO4bEIkeIb7Q1dbj0z0XxQ6HiO7AezsuoN4sIKlbJwxzs9EbG1+VHE/cZd1t/F/bsrmFQxsxwSF8vv8yKg0m9Az1xeQ+YWKH4zAyqQR/TLaO4ny69xLKqgwiR0REtyP3Wg3+e8Q919780twR0dD6KpFzrQbfHbsqdjguhQmOh6usq8enDSWWT4/v4XZrb35pcp8w9I3UoNpoxicNn5uIXMvynedhsggY1V2LITFBYofjUN5KORbYR3HOw2jiKE5rMcHxcP9JuwJdbT26BvtgWt9wscNxOIlEgj+Mt/bFWXXgCqoMrKgiciXW0RvrSIa7rr35pUeGRyPYz9rqwrbuiG6NCY4HqzaYro/ejOvuchtq3q6J8aHoqvWBvs5kLzElItfw3g7r6M3oHloMdvPRGxu1QoYnx1pHcT7afZF9cVqJCY4HW3XwCq5VGxHdyRv39o8QO5wOI5VK7O3cV+y9xPJLIhdRUmmwb7ny7ATPGL2xmT0kCgHeCuRcq+Hmwa3EBMdDGUxmfLTbOnrz1LjukMs865/CA4MiofVVIq+iFj+dZCt0Ilfw5YErMJotGBAV4DGjNzbeSrl9j6pP97IKtDU861uN7H44XoDSKgPC/NW4f2Ck2OF0OLVChrkjYgAAH++5CEHgkC+RM6urN+PLA9YmnfNGxYocjTjmjoiBQibB4cvlyMitEDscp9chCc7y5csRGxsLtVqNxMRE7Nmz56bHPvbYY5BIJE0effr0sR+zcuXKZo+pq6vriI/j8gRBwGf7rKM3KSOiofCw0RubR4ZHQ62Q4lSeHmkXysQOh4hasCEjH2XVRkRo1Jia4L7tLFoS6q/GPQ3LCT5lFegtOfybbc2aNVi4cCFefPFFpKenY/To0Zg6dSpycprf9PCf//wnCgoK7I/c3FwEBQVh5syZjY7z9/dvdFxBQQHUarWjP45bOHKlHKfz9VDJpXhoaBexwxFNkI8SswZHAQA+YuM/IqclCAJWNNyUPZoU43FT6jeyjV5tPFmAvIpakaNxbg7/V/L2229j3rx5ePzxxxEfH49ly5YhKioK77//frPHazQahIWF2R9HjhxBeXk5fvOb3zQ6TiKRNDouLMwzM/rbYRu9uX9gJAJ9lCJHI655o2IhlQA7s0qQVVgpdjhE1Ix958twtrAS3koZ5njwTRkA9InQIKlbJ5gtgn3/QGqeQxMco9GIo0ePIjk5udHzycnJ2L9/f6vO8emnn2LixImIjo5u9HxVVRWio6PRuXNnTJ8+Henp6Tc9h8FggF6vb/TwVHkVtfj5tHUF/mMjY8QNxglEd/LBlIbh7o85ikPklGyLamcmdobGSyFyNOJ7fLR1FGf1wRz28mqBQxOc0tJSmM1mhIaGNno+NDQUhYWFt3x9QUEBNm3ahMcff7zR83FxcVi5ciU2bNiA1atXQ61WY+TIkcjOzm72PEuXLoVGo7E/oqKibv9Dubgv0q7AbLHu3xIX5i92OE5hfkPJ+PqMPBTpuY6LyJmcL67CjqwSSCTAb0Z65uLiXxrbMwRdg31QaTDhG/byuqkOmcj85eaNgiC0akPHlStXIiAgADNmzGj0/PDhw/HII4+gf//+GD16NL755hv07NkT7777brPnWbJkCXQ6nf2Rm+uZ/yBqjWasPmRd+/RYUoy4wTiRgV0CMTg6EPVmAV8f8sx/G0TOyjalPiEuFDFaH5GjcQ5SqcS+FmfFvkts/HcTDk1wtFotZDJZk9Ga4uLiJqM6vyQIAlasWIGUlBQolS2vE5FKpRgyZMhNR3BUKhX8/f0bPTzRuvQ86GrrERXkhQnxLf/+PU3KCOsU6NeHc9j4j8hJlFcbsbZhg0lPLQ2/mQcGdkagtwJXy2ux5fStZ0Q8kUMTHKVSicTERKSmpjZ6PjU1FUlJSS2+dteuXTh//jzmzZt3y/cRBAEZGRkID3f/vZRulyAIWLm/oQphRIzHbMvQWlMSwhDko0SBrg47skrEDoeIAHx9OBd19Rb0DvfH8K6e1djvVryUMjzU0Phv1cHmq5I9ncOnqBYvXoxPPvkEK1asQGZmJhYtWoScnBwsWLAAgHX6aO7cuU1e9+mnn2LYsGFISEho8rNXX30VP//8My5evIiMjAzMmzcPGRkZ9nNSU/svlOFcURV8lDLMGuK5a5BuRiWXYWZiZwDWLSyISFwWi3B9Sn1kTKuWNXiaOUO6QCIB9p4vxeXSarHDcToOT3Bmz56NZcuW4bXXXsOAAQOwe/dubNy40V4VVVBQ0KQnjk6nw9q1a286elNRUYHf/e53iI+PR3JyMvLy8rB7924MHTrU0R/HZX227zIA4FeJneGvZhVCc37dUH6661wJcq/ViBwNkWfbf6EMOddq4KeW455+nrNXXltEBXljTI9gANbRLmpMInhgj3q9Xg+NRgOdTucR63EKdXVI+n/bYBGArYvvQvcQX7FDclopnx7EnuxSPDm2G/48JU7scIg81lNfHcNPJwqQMjwa/zej6Ug+WW0+VYgFXx6F1leJ/X+ZAKXcvZsgtuX7271/EwQAWHvsKiwCMDQmiMnNLdg2s/vmSC6MJi42JhJDWZXBvnD21x7e2O9WJsSHINhPhdIqI7ZmcpfxGzHBcXMWi4A1DUOXs7n25pYmxIcipOFiseUMKxOIxLD22FXUmwX076xB7wj3H2W/EwqZFLMbtpz5iouNG2GC4+YOXGqYx1bJcXdfVpndikImxZyGRHDVAV4siDqaIFzvR8XRm9aZPSTKvtj4ShkXG9swwXFzttGbewdEwEspEzka1zB7aBdIJUDaxTKcL64SOxwij3Lw0jVcLK2Gj1Jm3zmbWhYV5I3RXGzcBBMcN1ZRY8SmU9ZpljlDeCfUWpEBXhgfFwIA9jJVIuoYtv9z9w6IhI9KLnI0ruOhhtGu/3L9oB0THDf2fXoejCZrk6yESM5jt8XDw6xtDL49ehV19WaRoyHyDOXV12/KHuL0VJtwsXFTTHDclCAI9qHKOUOj2CSrjcb0DEaERg1dbT0vFkQd5LuGm7I+Ef7o21kjdjguRSGTYtZga7NSjjxbMcFxUyeu6nC2sBIquRT39Y8UOxyXI5NKcP8g6+9t7dGrIkdD5P6si4utX8xcXHx7bJ2N92SXIqeMzUqZ4Lgp2+jN3X3DofFm5+Lb8cAg693Q7uxSFFfWiRwNkXs7eqUc2cVV8FLIcN8ALi6+HVFB3hjVXQsA9k1KPRkTHDdUbTBhQ0YeAPa+uRPdgn0xsEsAzBYB69PzxQ6HyK3ZvpCn9QuHH7eTuW0PNtyYrUvPgwduVNAIExw39NPJAlQbzYjp5I1hsdyB907YLhZrj131+IsFkaPU1Zvx44kCAMADgzilfieS+4TCRylDzrUaHL1SLnY4omKC44a+a7gTmjmYi4vv1D39IqCUS3G2sBKn8/Vih0PklrafLUZlnQkRGjWGx3YSOxyX5q2UY0qCtanr2mN5IkcjLiY4biavohYHLl4DANw/kHdCd0rjrcCk+FAAnNMmcpTvGr6I7xsYCamUN2V36sGGUbCfTuR7dJsLJjhuZkOGda3I8K5BiAjwEjka9/BgovVisSEjH/VmNtAiak/Xqo3YmVUMAHiAN2XtYnjXTojQqKGvM2H72WKxwxENExw3IggC1qVbRxlmDOCFor2M6REMra8KZdVG7MoqETscIrfy44l8mCwCEiL90SPUT+xw3IJUKsF9Dcnidx488swEx41kFlTiXFEVlDIppnJjzXYjl0kxo6FsldNURO3LNj11/8DOIkfiXmyjYTuzSlBWZRA5GnEwwXEj6xtKwyfEh0DjxTLL9vRgovXiuy2zGBU1RpGjIXIPF0uqkJFbAZlUgnu5sWa76hHqh36dNTBZBPxw3DPbXDDBcRNmi4D1DetvZnAeu93Fh/ujd7g/jGaLx14siNrb9+nWm7LRPbQI9lOJHI37sRWarEv3zGoqJjhu4uDFMhTq6+CvlmNsr2Cxw3FLtlGcbz289JKoPQiCgHUZtukp3pQ5wj39IyCXSnD8qg7niyvFDqfDMcFxE7YMfVq/CKjkMpGjcU/39o+AVAIcz63gPi9Ed+jIlXLkXquFr0qO5N5hYofjlrS+KvsN73ceeGPGBMcN1NWbselUIQDeCTlSsJ8KSd2s+7z8cILTVER3wvaFOyUhDF5K3pQ5im3x9vfpebBYPKsbOxMcN7A1swhVBhMiA7wwODpQ7HDc2j39rdVpXIdDdPusWzNY/w+x941jTYgPgZ9ajnxdHY542NYNTHDcwPfptsXFEewC6mCT+4RBIZPgbGElsos8b06bqD3sOleCyjoTwvzVGN6VWzM4klohs08B/uhhI89McFzcjV1A2dzP8QK8lRjTwzqnzVEcotvzU8PGmtP6hfOmrANMbxh53niyEGYPmqZiguPiNp0qgMkioHc4u4B2lHsbmv79cKKAO4wTtVGt0YytmUUAgOn92JC0I4zqrkWAtwKlVQYcvFgmdjgdhgmOi7PdCdm+dMnxJsaHQq2Q4lJpNXcYJ2qjnVnFqDGaERnghQFRAWKH4xEUMimm9LFOU/3Q8J3hCZjguLCSSgMONGTj07g1Q4fxUckxIc66wzinqYja5seGL9jp/cIhkXB6qqNM72e9Cd58qsBjNg1mguPCNp8uhEUA+nfWICrIW+xwPIqtmurHEwUeV3pJdLuqDSZsO2ubnuKoc0ca3jUInXyUKK+px/4LnjFNxQTHhf3UsCJ+GuexO9zYXiHwVcmRV1GL9FzPKr0kul3bzxajrt6C6E7eSIj0FzscjyKXSTG1b0M1lYeMPDPBcVHFlXU4dOkaAOBuTk91OGvppW2aynPmtInuhK1MeVpfTk+JwTZq9vPpQhhN7j9NxQTHRf18qmF6KioAnQM5PSWGexp2P/7xRIFHlV4S3Y7KunrsyCoBwOkpsQyJCUKInwr6OhP2ZJeIHY7DdUiCs3z5csTGxkKtViMxMRF79uy56bE7d+6ERCJp8jh79myj49auXYvevXtDpVKhd+/eWLdunaM/hlOxL9Tj6I1oRnpo6SXR7diWWQyjyYKuwT6ID2dLCzHIpBL7koYfPaCayuEJzpo1a7Bw4UK8+OKLSE9Px+jRozF16lTk5OS0+LqsrCwUFBTYHz169LD/LC0tDbNnz0ZKSgqOHz+OlJQUzJo1CwcPHnT0x3EKxfo6HLpsnZ6yzalSx1PKpZiaYCu99Iw5baLbZZuems7pKVHZRs9SzxShrt4scjSO5fAE5+2338a8efPw+OOPIz4+HsuWLUNUVBTef//9Fl8XEhKCsLAw+0Mmu74Z27JlyzBp0iQsWbIEcXFxWLJkCSZMmIBly5Y5+NM4h82nCyEIwMAunJ4S27S+tjntIpg8pPSSqK10tfXYfa4UADC9P6enxDSoSwAiA7xQZTDZu+C7K4cmOEajEUePHkVycnKj55OTk7F///4WXztw4ECEh4djwoQJ2LFjR6OfpaWlNTnn5MmTb3pOg8EAvV7f6OHKbEOL7H0jvmFdgxDgrcC1aiMOX2Y1FVFzUs8UwWi2oGeoL3qy47qoJJLr01Tu3vTPoQlOaWkpzGYzQkNDGz0fGhqKwsLCZl8THh6Ojz76CGvXrsV3332HXr16YcKECdi9e7f9mMLCwjadc+nSpdBoNPZHVFTUHX4y8RTp63D4MqunnIVCJsWkeOu/xc2n3PtiQXS7rldPcfTGGdi2yNhxttitp6k6ZJHxL+dbBUG46Rxsr169MH/+fAwaNAgjRozA8uXLMW3aNPzjH/+47XMuWbIEOp3O/sjNzb2DTyOuTScLIAjWYcaIAC+xwyFcXwe1+XQhm/4R/YKuth57s63TU+zZ5Rz6RmoQGeCFGqMZu8+5bzWVQxMcrVYLmUzWZGSluLi4yQhMS4YPH47s7Gz7n8PCwtp0TpVKBX9//0YPV7XxpPVzT2OZpdMY2V0LX5UcRXoD0nMrxA6HyKlsP1sEk0VAz1BfdA/xFTscgnWAYHKf6zdm7sqhCY5SqURiYiJSU1MbPZ+amoqkpKRWnyc9PR3h4dcz/xEjRjQ555YtW9p0TldUpK/D4Su26SlWTzkLlVyG8XEhADhNRfRLm09Zv0Btmz2Sc5jSUAG69UyR2+5NJXf0GyxevBgpKSkYPHgwRowYgY8++gg5OTlYsGABAOv0UV5eHv7zn/8AsFZIxcTEoE+fPjAajfjyyy+xdu1arF271n7OZ599FmPGjMGbb76J++67D+vXr8fWrVuxd+9eR38cUW1pqJ4a1CUA4RpOTzmTqQlh2HA8H5tOFeKFu+NZBksEoMZowq6GKZDJCUxwnElidCC0vkqUVhmRdqEMY3oGix1Su3N4gjN79myUlZXhtddeQ0FBARISErBx40ZER0cDAAoKChr1xDEajXjuueeQl5cHLy8v9OnTBz/99BPuvvtu+zFJSUn4+uuv8dJLL+Hll19Gt27dsGbNGgwbNszRH0dUP5+2blI3mXdCTueuXsFQK6S4Wl6L0/l6JERqxA6JSHS7z5Wgrt6CqCAv9A533aUB7kgmlSC5Txi+OpiDzacL3TLBkQiC4HGrIvV6PTQaDXQ6ncusx6moMWLwX7fCZBGw87mxiNH6iB0S/cKCL45i8+lCPDWuG/40OU7scIhEt2hNBtal52H+6Fi8OK232OHQL+w+V4K5Kw5B66vCwRcmQCZ1/pHntnx/cy8qF7Etsxgmi4BeoX5MbpyUrZpq06lCeOB9A1EjRpMFWzOto85TOD3llIZ37QR/tRylVQYcy3G/Pl5McFzEzw0r3TmP7bzGx4VAKZPiYkk1sourxA6HSFRpF8tQWWdCsJ8KA6MCxQ6HmqGUSzGxoY/XppPuV03FBMcF1BrN2N2w8+vkPq0vr6eO5adWYFQPLYDrlSNEnsr2fyC5dyikLjD14also2s/n3a/kWcmOC5gV8NCvc6BXKjn7GylsJuY4JAHM1sEpJ5pKA/nqLNTG9MzGF4KGfIqanEqz7W3MfolJjguYItteqpPGMuPndyk3qGQSSXILNDjSlm12OEQieLolXKUVhnhr5ZjeNdOYodDLVArZBgXZ62g2nzavfp4McFxcvXm6wv1WB7u/AJ9lBjeNQgAR3HIc9nWDE7sHQqFjF8zzm5yH/cskOC/PCd34GIZ9HUmaH2VSIzmQj1XYLtYpJ4pEjkSoo4nCIJ9/Q1vylzDjQUS592oQIIJjpOz3QnZpj7I+dmqEo7llKOk0iByNEQd63S+HnkVtfBSyDCmh/s1j3NHNxZIuNPIMxMcJ2axCNjS0L04mXdCLiMiwAsJkf4QBOtGg0SexDZ6M7ZXMLyUMpGjodayVei608gzExwnlnG1AsWVBviq5EjqxoV6rmRSPKepyDP9fJrTU65ofFwoJBLgZJ4OBbpascNpF0xwnNjPDXdC4+JCoJLzTsiVJDfcDe3JLkWN0SRyNEQd43KptcmlXCrBuF4hYodDbRDsp8KgLtZ1nlszi0WOpn0wwXFSgiDccCfE5n6uJi7MD50DvWAwWbAnu1TscIg6hK3ic2hsEDTeCpGjobayrR90l5FnJjhO6kJJFS6X1UApk2Is74RcjkQiwaTe7nWxILoV27912799ci22v7e0C6WorKsXOZo7xwTHSaWesQ4RjujWCb4qucjR0O2wXSy2ny2G2eI+vSWImlNebcSRK9YNG20jAeRaugX7IFbrg3qzgN3nXH/kmQmOk7IN9U7knZDLGhoTBI2XAteqjTh6xf126iW60c5z1kQ+LswPUUHeYodDt6HxyLPrl4szwXFCZTdsXT8xntNTrkouk2J8nPXvzx0uFkQt4fSUe7CNvm0/W4x6s0XkaO4MExwntP1sMQQBSIj0R7jGS+xw6A4kN1zst5wpcqsW6EQ3MpjM2JVVAoDTU64uMToQQT5K6OtMOHz5mtjh3BEmOE7IPj3FC4XLG9MzGEq5FFfKapDtRi3QiW504OI1VBvNCPFToW+kRuxw6A7IpBL7yPPWM65dLs4Ex8nU1Zvti7uY4Lg+H5UcIxuaNLKaitzV1jPX1wxKuaWMy7OXi2e69uabTHCcTNqFMtTWmxHmr0afCH+xw6F2MKm3taPrFiY45IYEQbCPOk/iTZlbGNNTC6VcitxrtThX5Lojz0xwnEyqvXoqBBIJ74TcgW2h+PHcChTp60SOhqh9nc7Xo0BXB2+lDCO4pYxb8FbKMaq7dfNNVy6QYILjRARBwDauv3E7If5qDIgKAGBdQE7kTmxTr2N6BEOt4JYy7sIdGpUywXEip/L0KNIb4MM7IbdjG8XZ5iZ7vBDZpJ5hzy53NKFhofHxqzqXHXlmguNEbNNTY3oGc3NNNzM+znrx33u+BHX1ZpGjIWofeRW1OFOgh1QCe+UNuYcbR55ta6xcDRMcJ2KrRJjA6Sm3Ex/uhwiNGnX1FqRdKBM7HKJ2YZtSHxwdhCAfpcjRUHuzTVPtcNGpdSY4TuLGO6FxvYLFDofamUQiwXjbNNVZ17wbIvqlrQ1TrhPYcd0t2Ubl9p4vdcmRZyY4TsJ2J5QYHYhOviqRoyFHmNAwTbU9s9ile0sQAUC1wYQDDaORHHV2T3Fhrj3yzATHSdjuhFg95b5GdOsEtUKKfF0dMgsqxQ6H6I7sPV8Ko9mC6E7e6BbsI3Y45AASiQTj4lx35JkJjhNofCfEoV53pVbIMKq7dfpxuwteLIhuZFuXMa4Xe3a5M9t3kiuOPDPBcQL7Gu6EugR5o1uwr9jhkAPZLhZbWS5OLsxiEew9nXhT5t6SumntI89nC11r5JkJjhOwXSjGx/FOyN2Nt/eWqEBJpUHkaIhuz+l8PYorrT27hsYGiR0OOZBaIcPIbtauxq7WqLRDEpzly5cjNjYWarUaiYmJ2LNnz02P/e677zBp0iQEBwfD398fI0aMwM8//9zomJUrV0IikTR51NW5XjMiQRAaJTjk3kL91egbqYEgADuyXOtiQWRjW48xugd7dnkCewWoi/XDcXiCs2bNGixcuBAvvvgi0tPTMXr0aEydOhU5OTnNHr97925MmjQJGzduxNGjRzFu3Djcc889SE9Pb3Scv78/CgoKGj3UarWjP067s90JeStlGNaVd0KewJbIbuc0Fbko3pR5Ftvfc3puBcqqXGfk2eEJzttvv4158+bh8ccfR3x8PJYtW4aoqCi8//77zR6/bNky/PnPf8aQIUPQo0cPvPHGG+jRowd++OGHRsdJJBKEhYU1ergi24ViVHct74Q8hG3Nwp7sEhhMrtdbgjxbcWUdTlzVAQDGxrFnlycI13ihd7g/BAHYmVUidjit5tAEx2g04ujRo0hOTm70fHJyMvbv39+qc1gsFlRWViIoqPHoRlVVFaKjo9G5c2dMnz69yQjPjQwGA/R6faOHs9jGOyGPkxChQbCfCtVGMw5evCZ2OERtsvOs9Quuf2cNQvxcb9Scbo+9msqF1uE4NMEpLS2F2WxGaGjj3i6hoaEoLGzdFuxvvfUWqqurMWvWLPtzcXFxWLlyJTZs2IDVq1dDrVZj5MiRyM7ObvYcS5cuhUajsT+ioqJu/0O1o9IqA05crQAAe68Bcn9SqcS+kZ0rXSyIgOvrb2z7q5FnsN2E7z5XAqPJInI0rdMhi4x/WRkkCEKrqoVWr16NV155BWvWrEFIyPUEYPjw4XjkkUfQv39/jB49Gt988w169uyJd999t9nzLFmyBDqdzv7Izc29sw/UTnZmlUAQgIRIf4T6807Ik9guFlszi1yutwR5LoPJjD3ZpQA46uxp+ncOQCcfJSoNJhy57Bojzw5NcLRaLWQyWZPRmuLi4iajOr+0Zs0azJs3D9988w0mTpzY4rFSqRRDhgy56QiOSqWCv79/o4czsDV7G9+LFwpPM6qHFkq5FFfLa3G+uErscIha5eDFa6gxmhHip0KfCOe4jlLHkEpv7GrsGiPPDk1wlEolEhMTkZqa2uj51NRUJCUl3fR1q1evxmOPPYavvvoK06ZNu+X7CIKAjIwMhIeH33HMHcVosmDPuYY7IW7P4HG8lXIM79oJAMvFyXXcWD0llbJnl6exTa27yu7iDp+iWrx4MT755BOsWLECmZmZWLRoEXJycrBgwQIA1umjuXPn2o9fvXo15s6di7feegvDhw9HYWEhCgsLodPp7Me8+uqr+Pnnn3Hx4kVkZGRg3rx5yMjIsJ/TFRy5fA2VBhO0vkr0i9SIHQ6JwLZrPNfhkCsQBOGG9TccdfZEo3pooZBJcLG0GhdLnH/k2eEJzuzZs7Fs2TK89tprGDBgAHbv3o2NGzciOjoaAFBQUNCoJ86HH34Ik8mEp556CuHh4fbHs88+az+moqICv/vd7xAfH4/k5GTk5eVh9+7dGDp0qKM/Truxfand1ZN3Qp5qXMPU5JHL5dDX1YscDVHLLpRUIfdaLZRyKUZ214odDonAT62wd67e4QLl4vKOeJMnn3wSTz75ZLM/W7lyZaM/79y585bne+edd/DOO++0Q2Ti2Z7FfVw8XYzWB121PrhYWo192aWY2td1pljJ89huyoZ37QQfVYd8dZATGtcrBPvOl2FnVjHmjYoVO5wWcS8qEVwurcbFkmrIpRKM6sE7IU82juXi5CJ2NPS/Gd+Lzf08me2adfDiNVQbTCJH0zImOCKwfZkNjQ2Cv1ohcjQkJts01c5zJbBYWC5Ozqmyrh6HG0qDx7Lq06N11fqgS5A3jGYL9p0vFTucFjHBEYGtamYcLxQeb0hsIHyUMpRUGnA633k6bBPdaN/5UpgsArpqfRCj9RE7HBKRRCKxF0g4+zocJjgdrMZosrfnZ/diUsll9gWbLBcnZ2UbdeboDQHXv7t2ZhU7daNSJjgdbP/5MhjNFkQFeaFbMO+E6IbdxbkOh5yQIAj2O/Vx3FyTYF1orlZIUaCrQ1ZRpdjh3BQTnA524/RUa7arIPdnuys+frUCZVUGkaMhaux0vh4llQZ4K2X2EmHybGqFDEndrCPPznxjxgSnAwmCYN9qnutvyCZMo0bvcH8IArDrnHPPaZPn2dlwUzayuxYquUzkaMhZ2Nbh2HaXd0ZMcDpQdnEV8ipqoZJL7W36iYDrQ//OvmiPPM8O3pRRM2wjz0dzyqGrcc5GpUxwOpBt/44R3TrBS8k7IbrOtg5n97kSmMwWkaMhsiqvNiI9pxwAMJb9b+gGUUHe6BHiC7NFwJ7zznljxgSnA7E8nG5mQFQgArwV0NXWIz23QuxwiAAAu7NLYBGAuDA/RAR4iR0OOZlx9s03meB4NH1dPY5ctt4JMcGhX5JJJbirZ8M0lRMv2iPPYlszyPJwao5tVG/XuWKnbFTKBKeD7MtuaJQV7IMunbzFDoeckC3xdeaqBPIcZotgX2A8jtNT1IwhMUHwVclRWmXEyTyd2OE0wQSng3B6im5lTM9gSCTA2cJKFOrqxA6HPNzxqxUor6mHn1qOxOhAscMhJ6SQSTG6h/M2KmWC0wEaNcpigkM3EeSjxICoAADXS3OJxLKzYSRxTM9gyGX8qqDm2b7TnLEClP9qO8CNjbKGxPJOiG5ubE9bC3Tnu1iQZ+FNGbWGbR3OiasVKHWyRqVMcDoAG2VRa9kuFnvPl6Ke5eIkkuLKOvuaCtvid6LmhPir0SfC2qh0t5M1KmWC0wF22CsReKGglvWN1KCTjxJVBpO96o6oo+1quGb1jdQg2E8lcjTk7Gzfbc428swEx8EaN8riUC+1THpDufjOc1yHQ+LYec42PcWbMro123fb7uwSmJ2oXJwJjoPZGmX1DPVFJBtlUSvc5QJ7vJD7Mpkt2NOQ4NzFmzJqhYFRAfBXy1FRU48MJ2pUygTHweyba8bxQkGtM6ZHMKQSIKuoEvkVtWKHQx4mI7cC+joTArwV9qo+opbIZVKMbhh53uVEFaBMcBzIYhHsi65s1TFEtxJ4Q7k4dxenjmbrZzKmRzBkUonI0ZCrGGufWneeaxYTHAc6madDWbURvio5BsewPJxaz95bgl2NqYPtZFEE3Ya77OXiOpRUOke5OBMcB7JdKEZ27wQFG2VRG9gW7e07XwqjieXi1DGK9XU4na8HYG3wR9RaIX5qJET6A3CecnF+6zqQrQqG1VPUVn0i/KH1VaLaaMaRy9fEDoc8hG16oX9nDbS+LA+ntrE3KmWC496uVRvtq8k51EttZS0Xd66LBbk/W/8bVk/R7bB91+0+VwKTEzQqZYLjIHuySyAIQFyYH8I1LA+ntrvePIvrcMjxTGYL9mRz/Q3dvgFRAdB4KaCrrcfxqxVih8MEx1F22u+EeKGg2zO6hxZSCXCuqAp5LBcnB0tvKA8P9Fagf+cAscMhFyS/YXdxZ+hqzATHAVgeTu0hwFuJQV2s1XccxSFH23HD7uEsD6fbNda+u7j41ywmOA7A8nBqL7apgh3sakwOxvJwag+2rWZO5elRXFknaixMcBzAdqEY1V3L8nC6I7a7of0XSmEwmUWOhtxVsb4OZwr0kEisDf6Iblewnwp9IzUAgN3nSkWNhd++DnC9PJwXCrozvcP9ofVVocZo5u7i5DC2Sr1+nQPQieXhdIfsI88iT1N1SIKzfPlyxMbGQq1WIzExEXv27Gnx+F27diExMRFqtRpdu3bFBx980OSYtWvXonfv3lCpVOjduzfWrVvnqPDb5MbycC4wpjvVaHdxJ5jTJvdk+7c1ls39qB3YEpw9IpeLOzzBWbNmDRYuXIgXX3wR6enpGD16NKZOnYqcnJxmj7906RLuvvtujB49Gunp6XjhhRfwzDPPYO3atfZj0tLSMHv2bKSkpOD48eNISUnBrFmzcPDgQUd/nFtieTi1t+vl4lyHQ+3PWh5unUrgqDO1hwFRgXhpWjy+/X2SqAvWJYIgCI58g2HDhmHQoEF4//337c/Fx8djxowZWLp0aZPjn3/+eWzYsAGZmZn25xYsWIDjx48jLS0NADB79mzo9Xps2rTJfsyUKVMQGBiI1atX3zImvV4PjUYDnU4Hf3//O/l4TSxak4F16Xl44q6uWDI1vl3PTZ5JV1OPgf+3BRYB2PeX8YgMYOJM7efQpWuY9WEaAr0VOPLSJFZQkVNry/e3Q0dwjEYjjh49iuTk5EbPJycnY//+/c2+Ji0trcnxkydPxpEjR1BfX9/iMTc7p8FggF6vb/RwhBvLw8exEyi1E423guXi5DC2f1MsDyd349AEp7S0FGazGaGhoY2eDw0NRWFhYbOvKSwsbPZ4k8mE0tLSFo+52TmXLl0KjUZjf0RFRd3uR2qRrTzcTyVHYjTLw6n9cJqKHIXl4eSuOmSRsUTS+K5AEIQmz93q+F8+35ZzLlmyBDqdzv7Izc1tU/ytFRnohdfu64MFY7uxPJza1Y27i7NcnNpLEcvDyY3JHXlyrVYLmUzWZGSluLi4yQiMTVhYWLPHy+VydOrUqcVjbnZOlUoFlcrxpY9aXxXmjohx+PuQ57GVi5dWGXDkcjlGdteKHRK5AdvmmiwPJ3fk0GEGpVKJxMREpKamNno+NTUVSUlJzb5mxIgRTY7fsmULBg8eDIVC0eIxNzsnkatjuTg5gr1nF8vDyQ05fB5l8eLF+OSTT7BixQpkZmZi0aJFyMnJwYIFCwBYp4/mzp1rP37BggW4cuUKFi9ejMzMTKxYsQKffvopnnvuOfsxzz77LLZs2YI333wTZ8+exZtvvomtW7di4cKFjv44RKLhOhxqTywPJ3fn0CkqwFrSXVZWhtdeew0FBQVISEjAxo0bER0dDQAoKCho1BMnNjYWGzduxKJFi/Dee+8hIiIC//rXv/Dggw/aj0lKSsLXX3+Nl156CS+//DK6deuGNWvWYNiwYY7+OESise0unl1s3V2c5eJ0J47lVKCyYffwftw9nNyQw/vgOCNH9sEhcqQH39+Po1fK8fr9CXh4WLTY4ZAL+9vms1i+8wLuGxCBf84ZKHY4RK3iNH1wiKh9jeM0FbUTloeTu2OCQ+RC7LuLny+F0STeHi/k2lgeTp6ACQ6RC7GVi1cbzTh8+ZrY4ZCLYnk4eQImOEQuhOXi1B5YHk6egAkOkYsZF8d1OHT7WB5OnoIJDpGLGd092F4ufrW8RuxwyMWwPJw8BRMcIhej8VbYN3PlKA61FXcPJ0/BBIfIBdmqqbgOh9qK5eHkKZjgELkg25fTvvNl3F2cWq1Qd708/K6eIWKHQ+RQTHCIXFDvcH+E+KlQW2/GoUssF6fW2dVQPdW/cwCCfJQiR0PkWExwiFyQRCLh5pvUZjvOWv+tjOvF0Rtyf0xwiFyU7UtqB9fhUCvUmy3Ye95aHm5rNUDkzpjgELmokT20kEsluFhSjZwylotTy45cLkeVwQStrxIJERqxwyFyOCY4RC7KX31Dufg5juJQy24sD5eyPJw8ABMcIhdmKxffcZYJDrXMNpXJ9TfkKZjgELkw21qKtItlqKtnuTg1L6+iFueKqiDl7uHkQZjgELmwXqF+CNeoUVdvwYGLZWKHQ07KNj01qEsgNN4KkaMh6hhMcIhcGMvFqTXs5eFxnJ4iz8EEh8jFcdsGaonBZMa+89w9nDwPExwiFzeyuxYKmQSXy2pwqbRa7HDIyRy6dA219WaE+KnQO9xf7HCIOgwTHCIX56uSY0hMEABWU1FTN26uKZGwPJw8BxMcIjcwPo5djal5LA8nT8UEh8gN2NbhHLx4DdUGk8jRkLO4UlaNiyXVkEslGNlDK3Y4RB2KCQ6RG+gW7IMuQd4wmi32BaVEtumpwTGB8FezPJw8CxMcIjcgkUg4TUVNbG9YkzWW01PkgZjgELkJW4+THWdLIAiCyNGQ2GqMJqQ1NH8cz/435IGY4BC5iWGxQfBSyFCor8OZAr3Y4ZDI9p8vg9FkQWSAF3qE+IodDlGHY4JD5CbUChlGdu8EgF2NCdjeMFU5IT6E5eHkkZjgELkR2zTVdvbD8WiCINh7InF7BvJUTHCI3IhtMWl6TjnKq40iR0NiySyoRIGuDmqFFCO6dhI7HCJRMMEhciORAV6IC/ODRQB2neM0laeyVdKN7KaFWiETORoicTg0wSkvL0dKSgo0Gg00Gg1SUlJQUVFx0+Pr6+vx/PPPo2/fvvDx8UFERATmzp2L/Pz8RseNHTsWEomk0WPOnDmO/ChELmMcy8U93nZOTxE5NsF56KGHkJGRgc2bN2Pz5s3IyMhASkrKTY+vqanBsWPH8PLLL+PYsWP47rvvcO7cOdx7771Njp0/fz4KCgrsjw8//NCRH4XIZdha8u86VwKzheXinqa82oj0nHIATHDIs8kddeLMzExs3rwZBw4cwLBhwwAAH3/8MUaMGIGsrCz06tWryWs0Gg1SU1MbPffuu+9i6NChyMnJQZcuXezPe3t7IywszFHhE7msQV0CoPFSoKKmHuk55RjcsBEneYZd50pgEYC4MD9EBniJHQ6RaBw2gpOWlgaNRmNPbgBg+PDh0Gg02L9/f6vPo9PpIJFIEBAQ0Oj5VatWQavVok+fPnjuuedQWVl503MYDAbo9fpGDyJ3JZdJMaZnMABWU3mibZyeIgLgwASnsLAQISFN/4OFhISgsLCwVeeoq6vDX/7yFzz00EPw9/e3P//www9j9erV2LlzJ15++WWsXbsWDzzwwE3Ps3TpUvs6II1Gg6ioqLZ/ICIXMq6XNcHZwX44HsVktmCXrf8NExzycG1OcF555ZUmC3x/+Thy5AgANNtcShCEVjWdqq+vx5w5c2CxWLB8+fJGP5s/fz4mTpyIhIQEzJkzB99++y22bt2KY8eONXuuJUuWQKfT2R+5ublt/dhELmVsrxBIJEBmgR75FbVih0Md5FhOBfR1JgR4KzCwS6DY4RCJqs1rcJ5++ulbVizFxMTgxIkTKCoqavKzkpIShIaGtvj6+vp6zJo1C5cuXcL27dsbjd40Z9CgQVAoFMjOzsagQYOa/FylUkGlUrV4DiJ3EuSjxMCoABzLqcC2s8VIGR4tdkjUAWxTknf1DIZMyu7F5NnanOBotVpotdpbHjdixAjodDocOnQIQ4cOBQAcPHgQOp0OSUlJN32dLbnJzs7Gjh070KnTrZtUnT59GvX19QgPD2/9ByFycxPiQ3EspwLbM4uY4HgIW/dibq5J5MA1OPHx8ZgyZQrmz5+PAwcO4MCBA5g/fz6mT5/eqIIqLi4O69atAwCYTCb86le/wpEjR7Bq1SqYzWYUFhaisLAQRqO1K+uFCxfw2muv4ciRI7h8+TI2btyImTNnYuDAgRg5cqSjPg6Ry5kYbx0p3XehDDVGk8jRkKNdLa9BVlElpBLrCA6Rp3NoH5xVq1ahb9++SE5ORnJyMvr164cvvvii0TFZWVnQ6XQAgKtXr2LDhg24evUqBgwYgPDwcPvDVnmlVCqxbds2TJ48Gb169cIzzzyD5ORkbN26FTIZO3YS2fQM9UXnQC8YTRbszS4VOxxyMNvozaAugQjwVoocDZH4HNYHBwCCgoLw5ZdftniMIFxvRBYTE9Poz82JiorCrl272iU+IncmkUgwMT4UK/dfxrbMYiT3Yd8od5aaads9vOU1jkSegntREbkx21qM7VnFsLCrsduqMphw4EIZAGBSb66/IQKY4BC5tWFdg+CjlKGk0oCTeTqxwyEH2ZtdAqPZguhO3ugW7Ct2OEROgQkOkRtTyWX2rsbbMpu2bSD3kHrG1twvtFV9xog8ARMcIjdnW5Oxjds2uCWzRbDvHD+R01NEdkxwiNzc2F7BkEiA0/l6FOjY1djdZOSW41q1EX5qOYZwY1UiOyY4RG5O66vCwKgAAMC2TI7iuBvb9NTYXiFQyHhJJ7Lh/wYiD2CbpuLu4u7HtrZqYjynp4huxASHyAPYuxqfL0Wt0SxyNNRerpRVI7u4CjKpBGN7MsEhuhETHCIP0DPUF5EBXjCYLNh7nl2N3cXWhinHoTFB0HgrRI6GyLkwwSHyANauxtY7fJaLuw/b3+UETk8RNcEEh8hD3Fguzq7Grk9XW49Dl64BuD4FSUTXMcEh8hDDu3aCn0qOkkoDMq5WiB0O3aFd50pgsgjoHuKLGK2P2OEQOR0mOEQeQimXYmzD3lRbTnOaytVxeoqoZUxwiDzIpN7WqYzUM4UiR0J3wmS2YEdDyT+np4iaxwSHyIOM7RUMhUyCCyXVOF9cJXY4dJsOXboGfZ0Jgd4KDOoSKHY4RE6JCQ6RB/FXKzCimxYAkHqG01SuassZW3O/UMik3FyTqDlMcIg8DKepXJsgCNhy2vp3N7lPmMjREDkvJjhEHmZSw5qN9NwKFOvrRI6G2upUnh75ujp4K2UY1UMrdjhETosJDpGHCdOo0T8qAIJwvRMuuY6fG0Zv7uoZDLVCJnI0RM6LCQ6RB0rmNJXL2tLwd5bch9VTRC1hgkPkgWwJzr7zZagymESOhlrrUmk1zhVVQS6VYHwvJjhELWGCQ+SBuof4IlbrA6PZgl1ZJWKHQ61km54a3rUTN9ckugUmOEQeSCKRsJrKBV2vnuLoDdGtMMEh8lC2aaptZ4tRb7aIHA3dSrG+DsdyKgAAk3qzPJzoVpjgEHmogV0CofVVorLOhIMXr4kdDt1CasPeU/2jAhCmUYscDZHzY4JD5KFkUgkmxFlHcbZwmsrp/dywQSqnp4hahwkOkQebnGD9stx8qhAWiyByNHQz+rp6pF0oBQAkc3qKqFWY4BB5sJHdtfBTyVFcacCxnHKxw6Gb2HG2GPVmAd2CfdA9xFfscIhcAhMcIg+mksswsWGx8aZTnKZyVrbNNZO59xRRqzHBIfJwUxKsX5qbThZAEDhN5Wzq6s3Yeda6pYat8o2Ibo0JDpGHu6tnMLyVMuTr6nD8qk7scOgXdp0rQbXRjAiNGgOiAsQOh8hlODTBKS8vR0pKCjQaDTQaDVJSUlBRUdHiax577DFIJJJGj+HDhzc6xmAw4A9/+AO0Wi18fHxw77334urVqw78JETuS62QYXxcCABg06kCkaOhX9p40vp3MrVvOCQSicjRELkOhyY4Dz30EDIyMrB582Zs3rwZGRkZSElJueXrpkyZgoKCAvtj48aNjX6+cOFCrFu3Dl9//TX27t2LqqoqTJ8+HWaz2VEfhcitTU0IBwBsOlnIaSonUldvxraGHd/v7hsucjRErkXuqBNnZmZi8+bNOHDgAIYNGwYA+PjjjzFixAhkZWWhV69eN32tSqVCWFjzi+l0Oh0+/fRTfPHFF5g4cSIA4Msvv0RUVBS2bt2KyZMnt/+HIXJzY3sFQ62QIudaDU7n65EQqRE7JAKw+1wJqgwmhGvUGMjpKaI2cdgITlpaGjQajT25AYDhw4dDo9Fg//79Lb52586dCAkJQc+ePTF//nwUFxfbf3b06FHU19cjOTnZ/lxERAQSEhJuel6DwQC9Xt/oQUTX+ajkGNvTOk21mdVUTsM+PZUQDqmU01NEbeGwBKewsBAhISFNng8JCUFh4c0voFOnTsWqVauwfft2vPXWWzh8+DDGjx8Pg8FgP69SqURgYGCj14WGht70vEuXLrWvA9JoNIiKirqDT0bknqb2tY6abjzFaipnUFdvxtaG6alp/VgeTtRWbU5wXnnllSaLgH/5OHLkCAA0uyBOEIQWF8rNnj0b06ZNQ0JCAu655x5s2rQJ586dw08//dRiXC2dd8mSJdDpdPZHbm5uGz4xkWcYHxcCpUyKiyXVyC6uEjscj7c3uxRVBhPC/NUYGBV46xcQUSNtXoPz9NNPY86cOS0eExMTgxMnTqCoqKjJz0pKShAa2vpeDuHh4YiOjkZ2djYAICwsDEajEeXl5Y1GcYqLi5GUlNTsOVQqFVQqVavfk8gT+akVGNNTi62Zxdh4sgA9Q/3EDsmjXa+eCuP0FNFtaPMIjlarRVxcXIsPtVqNESNGQKfT4dChQ/bXHjx4EDqd7qaJSHPKysqQm5uL8HBrBUFiYiIUCgVSU1PtxxQUFODUqVNtOi8RNXVjNRWJx2AyI7WhezGrp4huj8PW4MTHx2PKlCmYP38+Dhw4gAMHDmD+/PmYPn16owqquLg4rFu3DgBQVVWF5557Dmlpabh8+TJ27tyJe+65B1qtFvfffz8AQKPRYN68efjjH/+Ibdu2IT09HY888gj69u1rr6oiotszMT4UcqkEWUWVuFDCaSqx7M0uRaXBhFB/FRK7cHqK6HY4tA/OqlWr0LdvXyQnJyM5ORn9+vXDF1980eiYrKws6HTW7qkymQwnT57Efffdh549e+LRRx9Fz549kZaWBj+/68Pl77zzDmbMmIFZs2Zh5MiR8Pb2xg8//ACZTObIj0Pk9jTeCozsrgUAbDzBpn9i+YnVU0R3TCJ4YLmEXq+HRqOBTqeDv7+/2OEQOZX/HsnFn749ge4hvkhdNIbdczuYwWTG4L9uRWWdCd88MQJDY4PEDonIabTl+5t7URFRI5MTwqCUS3G+uAqZBZVih+Nx9p8vQ2WdCSF+KgyO5vQU0e1igkNEjfirFRjXKxgAsOF4vsjReJ7r01OsniK6E0xwiKiJe/tHAgB+OJ7Ppn8dqK7ejJ8bOklP6xchcjREro0JDhE1MSE+BD5KGfIqanEsp1zscDzG9rPFqDSYEBngxekpojvEBIeImlArZEjuY90eYEMGp6k6yvqMPADAPf0jOD1FdIeY4BBRs+4dYJ0i+elkAUxmi8jRuD9dTT12nC0BAMwYyOkpojvFBIeImjWquxaB3gqUVhmx/0KZ2OG4vU2nCmA0WxAX5oe4MLavILpTTHCIqFkKmdS+TQCrqRxvfcNUoG3kjIjuDBMcIrqpe/tbv2x/PlWIunqzyNG4r0JdHQ5cso6S2X7nRHRnmOAQ0U0NiQlCuEaNSoMJO7NKxA7HbW04ngdBAIbGBKFzoLfY4RC5BSY4RHRTUqkE0/tZp6l+4DSVw3yfzukpovbGBIeIWmRr+rc1swhVBpPI0bif7KJKnCnQQy6VYFrDmiciunNMcIioRQmR/uiq9YHBZMGmk9xhvL3ZFheP7RWMQB+lyNEQuQ8mOETUIolEggcTOwMA/nv0qsjRuBdBELD+uLW5370DIkWOhsi9MMEholu6f2AkJBLg0KVruFJWLXY4buNYTgVyr9XCRynDpPhQscMhcitMcIjoliICvDCquxYAsPZYnsjRuI916dYRseQ+YfBSykSOhsi9MMEholb5VcM01dqjV2GxcIfxO1VXb7avv7H9bomo/TDBIaJWmdwnDH4qOfIqanHgIrduuFObTxWiss6EzoFeGNG1k9jhELkdJjhE1CpqhQzTG7rsfsvFxndszeFcAMDMxCjuHE7kAExwiKjVZg62TqVsPFWAyrp6kaNxXTllNUi7WAaJBPjVYE5PETkCExwiarWBUQHoGuyDunoLNp0sFDscl/XtUevozajuWkQGeIkcDZF7YoJDRK0mkUjsC2I5TXV7zBbB3k9o1uAokaMhcl9McIioTR4Y2BlSCXDo8jVcLmVPnLbae74UBbo6BHgrkNyHvW+IHIUJDhG1SZhGjdE9ggEAa49xFKetvmlYXDxjQCRUcva+IXIUJjhE1GY39sQxsydOq12rNmLLGevaJU5PETkWExwiarNJvUMR4K1Avq4OO7OKxQ7HZazPyEO9WUBCpD96R/iLHQ6RW2OCQ0RtplbIMLNhFOfLA1dEjsY1CIJg733D0Rsix2OCQ0S35eFh0QCAnedKkFNWI3I0zu9kng5nCyuhlEtxX3/uHE7kaExwiOi2xGh9MKZnMAQBWHWIozi38p806+/o7oQwaLwVIkdD5P6Y4BDRbUsZbh3F+eZwLurqzSJH47zKqgzYcNy6sebcpBhxgyHyEExwiOi2jY8LQWSAF8pr6rHxZIHY4TitNUdyYTRZ0K+zBgOjAsQOh8gjODTBKS8vR0pKCjQaDTQaDVJSUlBRUdHiayQSSbOPv//97/Zjxo4d2+Tnc+bMceRHIaJmyKQSPDSsCwDgCy42bpbJbMGXDdNTc0fEQCLhxppEHcGhCc5DDz2EjIwMbN68GZs3b0ZGRgZSUlJafE1BQUGjx4oVKyCRSPDggw82Om7+/PmNjvvwww8d+VGI6CZmDY6CQiZBek4FTuXpxA7H6WzNLEK+rg5BPkpM7xcudjhEHkPuqBNnZmZi8+bNOHDgAIYNGwYA+PjjjzFixAhkZWWhV69ezb4uLCys0Z/Xr1+PcePGoWvXro2e9/b2bnIsEXW8YD8VpiSE44fj+Vh18AqWPtBP7JCcyuf7raM3vx4aBbWCnYuJOorDRnDS0tKg0WjsyQ0ADB8+HBqNBvv372/VOYqKivDTTz9h3rx5TX62atUqaLVa9OnTB8899xwqKytveh6DwQC9Xt/oQUTtx7bY+Pv0fOhq60WOxnlkFVYi7WIZZFKJvayeiDqGwxKcwsJChISENHk+JCQEhYWFrTrH559/Dj8/PzzwwAONnn/44YexevVq7Ny5Ey+//DLWrl3b5JgbLV261L4OSKPRICqKTbaI2tOQmED0CvVDbb0Z33F/KrvP0y4DAJJ7hyIiwEvcYIg8TJsTnFdeeeWmC4FtjyNHjgBAs4vpBEFo9SK7FStW4OGHH4ZarW70/Pz58zFx4kQkJCRgzpw5+Pbbb7F161YcO3as2fMsWbIEOp3O/sjNzW3jpyailkgkEjwywjpC8Z+0K9yfCoCuph7rjuUBAB5laThRh2vzGpynn376lhVLMTExOHHiBIqKipr8rKSkBKGhobd8nz179iArKwtr1qy55bGDBg2CQqFAdnY2Bg0a1OTnKpUKKpXqluchotv3wMBI/OPnLFwqrcaW04WY2tezF9T+92guauvN6BXqh2GxQWKHQ+Rx2pzgaLVaaLXaWx43YsQI6HQ6HDp0CEOHDgUAHDx4EDqdDklJSbd8/aefforExET079//lseePn0a9fX1CA/37AsqkZh8VHLMHRGNd7efxwe7LmBKQpjHlkRbLIK9bP7RJJaGE4nBYWtw4uPjMWXKFMyfPx8HDhzAgQMHMH/+fEyfPr1RBVVcXBzWrVvX6LV6vR7//e9/8fjjjzc574ULF/Daa6/hyJEjuHz5MjZu3IiZM2di4MCBGDlypKM+DhG1wqNJMVDJpTh+VYe0i2VihyOazacLcaWsBv5qOWYMjBA7HCKP5NA+OKtWrULfvn2RnJyM5ORk9OvXD1988UWjY7KysqDTNe6d8fXXX0MQBPz6179uck6lUolt27Zh8uTJ6NWrF5555hkkJydj69atkMlYgkkkJq2vyr5T9ge7LoocjTgEQcB7O84DAB5LioG30mHdOIioBRJBEDxuNaBer4dGo4FOp4O/v7/Y4RC5lZyyGoz9xw5YBOCnZ0ahT4RG7JA61I6sYvzms8PwVsqw7/nxCPRRih0Skdtoy/c396IionbVpZM3pvWzTst86GGjOIIg4L3t1tGbh4d1YXJDJCImOETU7p4YY+08/uOJfOReqxE5mo5z8NI1HLlSDqVMisdHd731C4jIYZjgEFG7S4jUYHQPLSwC8PEezxnFsa29mTm4M0L91bc4mogciQkOETnEgru6AQC+OZKLsiqDyNE43vHcCuzJLoVMKrF/diISDxMcInKIpG6d0DdSg7p6Cz7bd1nscBzONnpz34AIRAV5ixwNETHBISKHkEgkeGqcdSRjxb5LKKl031GcrMJKbDlTBIkEeHIsR2+InAETHCJymMl9wtC/swY1RjPe3Z4tdjgOs3yndfRmSp8wdA/xEzkaIgKY4BCRA0kkEjw/NQ4A8NXBHFwpqxY5ovZ3Ol+HDcfzAQBPjesucjREZMMEh4gcKqmbFmN6BsNkEfDWlnNih9OuBEHAGxszIQjA9H7hSIj0rKaGRM6MCQ4ROdyfJ1v3n9twPB+n8nS3ONp17Mgqxr7zZVDKpHh+SpzY4RDRDZjgEJHDJURqcG9/a3fjv/2cJXI07cNktuCNjWcBAL8ZGcPKKSInwwSHiDrEH5N7Qi6VYPe5Euy/UCp2OHfs68O5OF9chUBvBZ7k2hsip8MEh4g6RHQnHzw0rAsA4M3NWXDlfX4r6+rxTqp1PdGzE3pA46UQOSIi+iUmOETUYf4wvge8lTIcz62wVx65ovd3XkBZtRFdtT54eHi02OEQUTOY4BBRhwn2U9m3Mfi/H8+gosYockRtl1dRi0/3XgIA/GVqHBQyXkaJnBH/ZxJRh3rirq7oHuKL0iojXv8pU+xw2uyNjZkwmCwYFhuESb1DxQ6HiG6CCQ4RdSiVXIY3H+wLiQT479Gr2JvtOguON54swE8nCiCTSvDy9N6QSCRih0REN8EEh4g6XGJ0EFIa1q68sO4kao1mkSO6tdIqA176/hQA635TbOpH5NyY4BCRKP40uRfCNWrkXKvBsq3O3eFYEAS8tO4UrlUbERfmhz+M7yF2SER0C0xwiEgUfmoF/jojAQDw8Z6LTt3heMPxfGw+XQi5VIK3ZvWHUs5LJ5Gz4/9SIhLNhPhQTOsXDosAPL/2BOrNFrFDaqJYX4f/WX8agLXMvU8Ep6aIXAETHCIS1Sv39IHGS4HT+Xr89cczYofTiCAIeGHdSehq69E3UoMnx3UTOyQiaiUmOEQkqmA/Ff4xsz8A4PO0K/jmcK7IEV236mAOtmYWQymT4h8z+7PnDZEL4f9WIhLdpN6hWDSxJwDgpe9P4VhOucgRAbvPleB/N1inphYn90SvMD+RIyKitmCCQ0RO4Q/ju2Nyn1AYzRYs+OIoivV1osVytlCPJ1cdg9ki4IGBkXhiTFfRYiGi28MEh4icglQqwVuzBqBHiC+KKw1Y8OVRGEwd3x+nWF+H3352GFUGE4bFBmHpg33Z0I/IBTHBISKn4auS4+O5g+GvluNYTgVeXHcKFkvH7TpeYzRh3udHkK+rQ9dgH3yYkgiVXNZh709E7YcJDhE5lRitD/7164GQSIBvj17Fom8yYDQ5vnzcZLbgmdUZOJmnQ5CPEp89NgQB3kqHvy8ROQYTHCJyOmN7heCdWQMgl0qwPiMfj//nCKoNJoe9X0WNEY99dhhbM4uglEvx8dxERHfycdj7EZHjMcEhIqc0Y2AkPnl0MLwUMuw+V4KHPjmIa9XGdn+f88WVmPHePuw9XwpvpQzvPzwIidFB7f4+RNSxmOAQkdMa2ysEX80fhkBvBY7nVuBXH+xH7rWadjv/jrPFuP+9/bhcVoPIAC+s/X0SJsSHttv5iUg8Dk1wXn/9dSQlJcHb2xsBAQGteo0gCHjllVcQEREBLy8vjB07FqdPn250jMFgwB/+8AdotVr4+Pjg3nvvxdWrVx3wCYhIbAO7BOK/C5IQGeCFiyXVSH5nN5ZtPXdHO5DX1Zvx3o7z+O3nh1FpMGFITCDWPz0S8eH+7Rg5EYnJoQmO0WjEzJkz8fvf/77Vr/nb3/6Gt99+G//+979x+PBhhIWFYdKkSaisrLQfs3DhQqxbtw5ff/019u7di6qqKkyfPh1mc8eXlBKR43UP8cXa3ydhcHQgauvNWLY1G+P+sRPfHbvapiqrGqMJn+y5iDF/24G//5wFQQBmD47CqseHQ+urcuAnIKKOJhEEweE1mCtXrsTChQtRUVHR4nGCICAiIgILFy7E888/D8A6WhMaGoo333wTTzzxBHQ6HYKDg/HFF19g9uzZAID8/HxERUVh48aNmDx58i3j0ev10Gg00Ol08PfnHRuRqxAEAT+dLMD/23QWV8trAQD9Omtw/8BIDOwSiN7h/k12+jZbBORX1OL79Dys2HcJ5TX1AIBwjRoLJ/bArMFR7HND5CLa8v0t76CYWuXSpUsoLCxEcnKy/TmVSoW77roL+/fvxxNPPIGjR4+ivr6+0TERERFISEjA/v37m01wDAYDDAaD/c96vd6xH4SIHEIikWB6vwhMjA/Fin2XsHzHBZy4qsOJqzoAgFIuRUKEP7oG+6JIX4ecazXIK6+F6YZRnphO3vj92G64f2DnJskQEbkPp0pwCgsLAQChoY0X+YWGhuLKlSv2Y5RKJQIDA5scY3v9Ly1duhSvvvqqAyImIjGoFTI8ObY7ZiZG4ZsjuTh6pRzpOeUor6nHsZwKHMupaHS8UiZFXLgf5o2KxbS+4ZBz00wit9fmBOeVV165ZbJw+PBhDB48+LaD+uVwsSAItxxCbumYJUuWYPHixfY/6/V6REVF3XZ8ROQcgv1UeGpcdwDWa8CVshqk55Yj91otwjVqRAV5o0uQN0L91ZBJOQ1F5EnanOA8/fTTmDNnTovHxMTE3FYwYWFhAKyjNOHh4fbni4uL7aM6YWFhMBqNKC8vbzSKU1xcjKSkpGbPq1KpoFJxASGRO5NIJIjR+iBGywZ9RHQbCY5Wq4VWq3VELIiNjUVYWBhSU1MxcOBAANZKrF27duHNN98EACQmJkKhUCA1NRWzZs0CABQUFODUqVP429/+5pC4iIiIyLU4dA1OTk4Orl27hpycHJjNZmRkZAAAunfvDl9fXwBAXFwcli5divvvvx8SiQQLFy7EG2+8gR49eqBHjx5444034O3tjYceeggAoNFoMG/ePPzxj39Ep06dEBQUhOeeew59+/bFxIkTHflxiIiIyEU4NMH5n//5H3z++ef2P9tGZXbs2IGxY8cCALKysqDT6ezH/PnPf0ZtbS2efPJJlJeXY9iwYdiyZQv8/Pzsx7zzzjuQy+WYNWsWamtrMWHCBKxcuRIyGXf9JSIiog7qg+Ns2AeHiIjI9bTl+5u1kkREROR2mOAQERGR22GCQ0RERG6HCQ4RERG5HSY4RERE5HaY4BAREZHbYYJDREREbocJDhEREbkdJjhERETkdhy6VYOzsjVv1uv1IkdCRERErWX73m7NJgwemeBUVlYCAKKiokSOhIiIiNqqsrISGo2mxWM8ci8qi8WC/Px8+Pn5QSKRtOu59Xo9oqKikJuby32uRMDfv7j4+xcXf//i4u/f8QRBQGVlJSIiIiCVtrzKxiNHcKRSKTp37uzQ9/D39+c/cBHx9y8u/v7Fxd+/uPj7d6xbjdzYcJExERERuR0mOEREROR2mOC0M5VKhf/93/+FSqUSOxSPxN+/uPj7Fxd//+Li79+5eOQiYyIiInJvHMEhIiIit8MEh4iIiNwOExwiIiJyO0xwiIiIyO0wwWlHy5cvR2xsLNRqNRITE7Fnzx6xQ/IIS5cuxZAhQ+Dn54eQkBDMmDEDWVlZYoflsZYuXQqJRIKFCxeKHYrHyMvLwyOPPIJOnTrB29sbAwYMwNGjR8UOyyOYTCa89NJLiI2NhZeXF7p27YrXXnsNFotF7NA8HhOcdrJmzRosXLgQL774ItLT0zF69GhMnToVOTk5Yofm9nbt2oWnnnoKBw4cQGpqKkwmE5KTk1FdXS12aB7n8OHD+Oijj9CvXz+xQ/EY5eXlGDlyJBQKBTZt2oQzZ87grbfeQkBAgNiheYQ333wTH3zwAf79738jMzMTf/vb3/D3v/8d7777rtiheTyWibeTYcOGYdCgQXj//fftz8XHx2PGjBlYunSpiJF5npKSEoSEhGDXrl0YM2aM2OF4jKqqKgwaNAjLly/HX//6VwwYMADLli0TOyy395e//AX79u3jiLFIpk+fjtDQUHz66af25x588EF4e3vjiy++EDEy4ghOOzAajTh69CiSk5MbPZ+cnIz9+/eLFJXn0ul0AICgoCCRI/EsTz31FKZNm4aJEyeKHYpH2bBhAwYPHoyZM2ciJCQEAwcOxMcffyx2WB5j1KhR2LZtG86dOwcAOH78OPbu3Yu7775b5MjIIzfbbG+lpaUwm80IDQ1t9HxoaCgKCwtFisozCYKAxYsXY9SoUUhISBA7HI/x9ddf49ixYzh8+LDYoXicixcv4v3338fixYvxwgsv4NChQ3jmmWegUqkwd+5cscNze88//zx0Oh3i4uIgk8lgNpvx+uuv49e//rXYoXk8JjjtSCKRNPqzIAhNniPHevrpp3HixAns3btX7FA8Rm5uLp599lls2bIFarVa7HA8jsViweDBg/HGG28AAAYOHIjTp0/j/fffZ4LTAdasWYMvv/wSX331Ffr06YOMjAwsXLgQERERePTRR8UOz6MxwWkHWq0WMpmsyWhNcXFxk1Edcpw//OEP2LBhA3bv3o3OnTuLHY7HOHr0KIqLi5GYmGh/zmw2Y/fu3fj3v/8Ng8EAmUwmYoTuLTw8HL179270XHx8PNauXStSRJ7lT3/6E/7yl79gzpw5AIC+ffviypUrWLp0KRMckXENTjtQKpVITExEampqo+dTU1ORlJQkUlSeQxAEPP300/juu++wfft2xMbGih2SR5kwYQJOnjyJjIwM+2Pw4MF4+OGHkZGRweTGwUaOHNmkLcK5c+cQHR0tUkSepaamBlJp469SmUzGMnEnwBGcdrJ48WKkpKRg8ODBGDFiBD766CPk5ORgwYIFYofm9p566il89dVXWL9+Pfz8/OwjaRqNBl5eXiJH5/78/PyarHfy8fFBp06duA6qAyxatAhJSUl44403MGvWLBw6dAgfffQRPvroI7FD8wj33HMPXn/9dXTp0gV9+vRBeno63n77bfz2t78VOzQSqN289957QnR0tKBUKoVBgwYJu3btEjskjwCg2cdnn30mdmge66677hKeffZZscPwGD/88IOQkJAgqFQqIS4uTvjoo4/EDslj6PV64dlnnxW6dOkiqNVqoWvXrsKLL74oGAwGsUPzeOyDQ0RERG6Ha3CIiIjI7TDBISIiIrfDBIeIiIjcDhMcIiIicjtMcIiIiMjtMMEhIiIit8MEh4iIiNwOExwiIiJyO0xwiIiIyO0wwSEiIiK3wwSHiIiI3A4THCIiInI7/x/4ytlcVtec1AAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Compute the x and y coordinates for points on a sine curve\n", + "x = np.arange(0, 3 * np.pi, 0.1)\n", + "y = np.sin(x)\n", + "\n", + "# Plot the points using matplotlib\n", + "plt.plot(x, y)" + ] + }, + { + "cell_type": "markdown", + "id": "92fed022-83ba-4151-80ad-f3b44047bf51", + "metadata": { + "id": "9W2VAcLiL9jX" + }, + "source": [ + "只需做一点额外的工作,我们就可以很容易地一次绘制多条线,并添加一个标题、图例和轴标签。" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "id": "f5e9b070-b853-417e-b588-ca7deeabe6e8", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 312 + }, + "id": "TfCQHJ5AL9jY", + "outputId": "fdb9c033-0f06-4041-a69d-a0f3a54c7206", + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 93, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAksAAAHFCAYAAADi7703AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAACrlElEQVR4nOzdd3wUdfrA8c/upldIQgqQhAAp9N57CyLYUMGG5x2i3p39vPP4efY7PT1PudOznOWwIioWVER6k95bEkIPkJCEkF53d35/TGaTAMkmkGS2PO/Xa1872czOPpsy+8y3PF+DoigKQgghhBDikox6ByCEEEII4cgkWRJCCCGEaIAkS0IIIYQQDZBkSQghhBCiAZIsCSGEEEI0QJIlIYQQQogGSLIkhBBCCNEASZaEEEIIIRogyZIQQgghRAMkWRJCNGjLli3ccMMNxMTE4O3tTUREBMOGDeMPf/hDnf3Gjh3L2LFj9QmylcyfPx+DwcDx48cbtf/69euZMWMGHTp0wMvLi+DgYIYPH85bb71FSUlJi8RoMBh45plnWuTYQrgrD70DEEI4rh9//JFrr72WsWPH8vLLLxMVFUVmZibbt2/n888/55///Kdt3zfffFPHSB3P008/zXPPPcfw4cN5/vnn6dKlC6WlpWzcuJFnnnmGQ4cO8dprrzX7627atImOHTs2+3GFcGcGWRtOCFGfMWPGcPr0aVJTU/HwqHttZbVaMRrdq3F6/vz5/PrXv+bYsWN06tSp3v2+/PJLZsyYwezZs3n33XcxGAx1vl9UVMSmTZtITk5u4YiFEM3Bvc50QogmOXfuHGFhYRclSsBFidKF3XDHjx/HYDDwyiuv8OqrrxIXF0dAQADDhg1j8+bNFx1v+/btXHvttYSEhODj40O/fv344osvGhXns88+y5AhQwgJCSEoKIj+/fvz/vvvc+G1YKdOnZg2bRpLly6lf//++Pr6kpSUxAcffHDRMTdv3syIESPw8fGhffv2zJ07l6qqqkbF89xzz9G2bVv+/e9/X5QoAQQGBtZJlMrLy5k7dy5xcXF4eXnRoUMHfv/735Ofn1/neatWrWLs2LGEhobi6+tLTEwMN954I6WlpbZ9LuyG07oOV69ezW9/+1vCwsIIDQ1l+vTpnDlz5qLYFi5cyLBhw/D39ycgIIDJkyeza9euRr1vIVyVJEtCiHoNGzaMLVu28OCDD7Jly5ZGJwu1/ec//2H58uXMmzePTz/9lJKSEq6++moKCgps+6xevZoRI0aQn5/P22+/zXfffUffvn2ZOXMm8+fPt/sax48f59577+WLL77g66+/Zvr06TzwwAM8//zzF+27Z88e/vCHP/DII4/w3Xff0bt3b2bPns26dets+xw8eJAJEyaQn5/P/Pnzefvtt9m1axd//etf7caSmZnJ/v37SU5Oxs/Pz+7+iqJw/fXX88orrzBr1ix+/PFHHn30UT788EPGjx9PRUWF7T1OnToVLy8vPvjgA5YuXcrf//53/P39qaystPs6d999N56ennz22We8/PLLrFmzhjvuuKPOPi+88AK33nor3bt354svvuDjjz+mqKiIUaNGcfDgQbuvIYTLUoQQoh65ubnKyJEjFUABFE9PT2X48OHKiy++qBQVFdXZd8yYMcqYMWNsXx87dkwBlF69eilms9n2+NatWxVAWbBgge2xpKQkpV+/fkpVVVWdY06bNk2JiopSLBZLo2O2WCxKVVWV8txzzymhoaGK1Wq1fS82Nlbx8fFRTpw4YXusrKxMCQkJUe69917bYzNnzlR8fX2VrKws22Nms1lJSkpSAOXYsWP1vv7mzZsVQPnzn//cqHiXLl2qAMrLL79c5/GFCxcqgPLf//5XURRF+eqrrxRA2b17d4PHA5Snn37a9vX//vc/BVB+97vf1dnv5ZdfVgAlMzNTURRFOXnypOLh4aE88MADdfYrKipSIiMjlRkzZjTq/QjhiqRlSQhRr9DQUNavX8+2bdv4+9//znXXXcehQ4eYO3cuvXr1Ijc31+4xpk6dislksn3du3dvAE6cOAHA4cOHSU1N5fbbbwfAbDbbbldffTWZmZmkpaU1+BqrVq1i4sSJBAcHYzKZ8PT05KmnnuLcuXNkZ2fX2bdv377ExMTYvvbx8SEhIcEWD6gtXRMmTCAiIsL2mMlkYubMmXbfb1OtWrUKgLvuuqvO4zfffDP+/v6sXLnSFreXlxf33HMPH374IUePHm3S61x77bV1vr7w9/Dzzz9jNpu588476/wOfHx8GDNmDGvWrLmMdyeEa5BkSQhh18CBA3n88cf58ssvOXPmDI888gjHjx/n5Zdftvvc0NDQOl97e3sDUFZWBsDZs2cBeOyxx/D09Kxz+93vfgfQYFK2detW2/ifd999l19++YVt27bxxBNP1Hmd+uLRYqq937lz54iMjLxov0s9diEtETt27JjdfbXX8vDwoF27dnUeNxgMREZGcu7cOQC6dOnCihUrCA8P5/e//z1dunShS5cu/Otf/2rU6zT29zBo0KCLfg8LFy5sVGIshKuS0gFCiCbx9PTk6aef5rXXXmP//v1XfLywsDAA5s6dy/Tp0y+5T2JiYr3P//zzz/H09OSHH37Ax8fH9vi333572TGFhoaSlZV10eOXeuxCUVFR9OrVi2XLllFaWmp33FJoaChms5mcnJw6CZOiKGRlZTFo0CDbY6NGjWLUqFFYLBa2b9/O66+/zsMPP0xERAS33HJLE97hxbTfw1dffUVsbOwVHUsIVyMtS0KIemVmZl7y8ZSUFADat29/xa+RmJhIfHw8e/bsYeDAgZe8BQYG1vt8g8GAh4dHna6+srIyPv7448uOady4caxcudLW2gJgsVhYuHBho57/5JNPcv78eR588MGLZuQBFBcXs2zZMgAmTJgAwCeffFJnn0WLFlFSUmL7fm0mk4khQ4bwn//8B4CdO3c27o01YPLkyXh4eHDkyJF6fw9CuCtpWRJC1Gvy5Ml07NiRa665hqSkJKxWK7t37+af//wnAQEBPPTQQ83yOu+88w5Tpkxh8uTJ3HXXXXTo0IG8vDxSUlLYuXMnX375Zb3PnTp1Kq+++iq33XYb99xzD+fOneOVV16xdTNdjr/85S8sXryY8ePH89RTT+Hn58d//vOfRlfdvvnmm3nyySd5/vnnSU1NZfbs2bailFu2bOGdd95h5syZJCcnM2nSJCZPnszjjz9OYWEhI0aMYO/evTz99NP069ePWbNmAfD222+zatUqpk6dSkxMDOXl5baSBxMnTrzs96rp1KkTzz33HE888QRHjx7lqquuom3btpw9e5atW7fi7+/Ps88+e8WvI4RT0nuEuRDCcS1cuFC57bbblPj4eCUgIEDx9PRUYmJilFmzZikHDx6ss299s+H+8Y9/XHRcLpixpSiKsmfPHmXGjBlKeHi44unpqURGRirjx49X3n77bbtxfvDBB0piYqLi7e2tdO7cWXnxxReV999//6KZa7GxscrUqVMvev6FsSuKovzyyy/K0KFDFW9vbyUyMlL54x//qPz3v/+1OxuutrVr1yo33XSTEhUVpXh6eipBQUHKsGHDlH/84x9KYWGhbb+ysjLl8ccfV2JjYxVPT08lKipK+e1vf6ucP3/ets+mTZuUG264QYmNjVW8vb2V0NBQZcyYMcrixYvrvOaFP1ttNty2bdvq7Ld69WoFUFavXl3n8W+//VYZN26cEhQUpHh7eyuxsbHKTTfdpKxYsaJR71kIVyQVvIUQQgghGiBjloQQQgghGiDJkhBCCCFEAyRZEkIIIYRogCRLQgghhBANkGRJCCGEEKIBkiwJIYQQQjRAilI2A6vVypkzZwgMDMRgMOgdjhBCCCEaQVEUioqKaN++PUZj/e1Hkiw1gzNnzhAdHa13GEIIIYS4DBkZGXTs2LHe70uy1Ay0dasyMjIICgrSORohhBBCNEZhYSHR0dENrj8Jkiw1C63rLSgoSJIlIYQQwsnYG0IjA7yFEEIIIRogyZIQQgghRAMkWRJCCCGEaIAkS0IIIYQQDZBkSQghhBCiAZIsCSGEEEI0QJIlIYQQQogGSLIkhBBCCNEASZaEEEIIIRogyZIQQgghRAOcKllat24d11xzDe3bt8dgMPDtt9/afc7atWsZMGAAPj4+dO7cmbfffvuifRYtWkT37t3x9vame/fufPPNNy0QvRBCCCGckVMlSyUlJfTp04c33nijUfsfO3aMq6++mlGjRrFr1y7+7//+jwcffJBFixbZ9tm0aRMzZ85k1qxZ7Nmzh1mzZjFjxgy2bNnSUm9DCCGEEE7EoCiKoncQl8NgMPDNN99w/fXX17vP448/zuLFi0lJSbE9dt9997Fnzx42bdoEwMyZMyksLOSnn36y7XPVVVfRtm1bFixY0KhYCgsLCQ4OpqCgoHkX0i04DZZKaBMLRqfKa3VjsSpYFQVPk/y8hNCFokB5Pvi0ATuLkwqht8Z+fnu0YkytbtOmTSQnJ9d5bPLkybz//vtUVVXh6enJpk2beOSRRy7aZ968efUet6KigoqKCtvXhYWFzRq3zZa3YeO/wSsA2iVBRHcI7wEdBkDHgW59IiqpMPP1rtOsTs3mXEkl+aWV5JdWUVhehafJyOBOIYxOCGN0QjsSIwLtrigthLgMpXmw+zPIPggFGVBwqvoirwJ8QyBmqHqLHgrt+4KHt94R6yols5Bvdp1mZcpZSiosWBQFi1XBbLHi5WFidHwYU3tHMTI+DG8Pk97hilpcOlnKysoiIiKizmMRERGYzWZyc3OJioqqd5+srKx6j/viiy/y7LPPtkjMdVSVgskLKovh9Hb1pokZDhOehNjhLR+HAzmSU8zHm06waMcpiirMl9yn0mxlw+FcNhzO5YUlqUQEeTO1V3semhBPsJ9nK0cshAsqOAWb3oQd86Gq5NL7lOVB2hL1BuDhC0Pvg1F/AO/AVgtVb2cLy/lu92m+3nma1KyiBvZULwC/3nWaQB8PkrtHcn2/9ozsGiYXew7ApZMl4KI/Mq3Xsfbjl9qnoT/OuXPn8uijj9q+LiwsJDo6ujnCrWvqP+Gqv8O5I+qVW/ZBOHsQjqyEkxvhf1Og6yQY/xf1qs2FHckp5tnvD7LuUI7tsbgwf24ZFE1cmD9t/b1o4+tJGz8v8ksrWZeey/r0HDYfPcfZwgo++OUYi/ec4alrunNN7yg5+QhxOXLTYf2rsO8LsFZfrET0hO7XQ5sYCO6o3vzbQXYKnNxUfdsMpbmw4TXYvQAmPQu9Zrj08AKrVeGDX47x8tI0Ki1WALxMRsYnhXN9v/Z0bOuH0WDAw2TAZDSQW1TBT/uzWLIvk+yiChbtPMWinaeY0jOSv17fk9AA926V05tLJ0uRkZEXtRBlZ2fj4eFBaGhog/tc2NpUm7e3N97erfSHa/KE8CT1xnT1scIzsO4fsPMjOLxcvfWaAdNeA++A1omrFf2w9wyPf7WXkkoLBgNMSArnzmGdGNk1DKPx4qSnXaA38RGBzB4ZR3mVhV8O5/LCkhSO5JTw4IJdfLXjFH+9ricxoX46vBshnNTeL2Hx/WAuV7/uNApGPAxdJ1x6SEDHAept+P3qOKa0n+Dn/4Pzx+Cbe2Hbe3DVS+o+LuZsYTl/+GIPGw7nAtA3ug03D+zI1F5RtPHzuuRzurQLYEjnUJ6a1p3tJ86zeM9pPt+awU/7s9h2PI8Xp/dmUvf6P5dEy3L5Ad7ff/89Bw8etD3229/+lt27d9cZ4F1UVMSSJUts+0yZMoU2bdroP8DbnryjsObvsPcLQIGoPnDbFxAY2XoxtKBKs5UXlqQwf+NxAIZ2DuHlG/tcVpJTYbbwztqjvLH6MJVmK94eRp6Y2o07h3Vq3qCFcDVWC6x4Gja+rn4dNwYmPKWOm2wqcwVsfhPWvaIOL8AAU1+BQXc3a8h6Wro/iz9/vZf80ip8PI08Oa07tw2OuazW7H2nCnj0i92kZxcDcNOAjjx1TXeCfGQ4QXNp7Oe3UyVLxcXFHD58GIB+/frx6quvMm7cOEJCQoiJiWHu3LmcPn2ajz76CFBLB/Ts2ZN7772XOXPmsGnTJu677z4WLFjAjTfeCMDGjRsZPXo0f/vb37juuuv47rvv+Mtf/sKGDRsYMmRIo+LSLVnSZGyFBbeqzdxBHeH2L9XB4E7sTH4Zv/t0J7sz8gH43dguPDopAY8rnOV2NKeYv3y7n41HzgHwh0kJPDAh/krDFcI1lZ2Hr34DR1apX498VO32N17h4OOiLLWVaX91GZdxf4HRjzn1pBWLVeHpxfv5ZPNJAHq0D+Jft/Sja/iVtfaXV1l4bfkh/rv+KIoCMSF+LLhnKB3a+DZH2G7PJZOlNWvWMG7cuIse/9WvfsX8+fO56667OH78OGvWrLF9b+3atTzyyCMcOHCA9u3b8/jjj3PffffVef5XX33FX/7yF44ePUqXLl3429/+xvTp0xsdl+7JEkDeMfj0ZjiXDt5BMOMj6HLxz8oZpGYVctu7W8grqSTIx4PXZvZlQrfma35WFIU3Vh3mn8sPAfD7cV14LDlRxjEJUVtOGiy4RW3B9vSD6/4DPRt/XrRLUWD1C7DuZfXrob+D5L855Tgmq1Vh7tf7WLg9A4MB7hndmT9MSsTLo/ney7bjeTz8+W5O55cRG+rHwnuGERns02zHd1cumSw5KodIlkCdxrvwDjjxCxg94No3oO+t+sVzGU6dL+XGtzZytrCCHu2DePuOAUSHtMzYovfWH+WvP6o1uGaPjOMvU7tJwiQEwPkT8P4kKD4LwTFwy6cQ1btlXmvzW7D0z+p2n1vh2tfVsZpOQlEUnv3+IPM3HsdogH/d0o9r+rRvkdc6k1/GzP9uIiOvjM5h/nx+z1DCgyRhuhKN/fx2vhRe1M8vBGZ9ow72tprhu9/B4ZV6R9VoeSWV3PnBVs4WVpAQEcBndw9tsUQJ4O5RnXn+uh4AvL/hGE9+tx+rVa4dhJsrO6+2UhefVeu63bO65RIlgKG/hRveAYMJ9iyAL+4Ey6XLgjiiV5al2cZVvnRj7xZLlADat/Hls7vVLrijuSXc9t4Wcooq7D9RXDFJllyNhzdM/y/0uQ0UK3z5a8g5pHdUdpVWmvn1/G0czSmhfbAPH/5mcKvURJo1rBMv39gbgwE+2XySvy1Jsf8kIVyVuQIWzoLcNAhsr45/9A9r+dftc4vaeuXho9ZlWvF0y79mM/jP6sP8Z/URAJ6/rgc3D2yBEjIXiA7xY8GcoUQF+3A4u5g73lOHLIiWJcmSKzIY4Jp5atXcigJYMFPtonNQVRYrv/1kJ3sy8mnj58lHswcTFdx6gxdnDIrm1Rl9ALWF6Ye9Z1rttYVwGIoC3/0ejq8Hr0C4/QsI7tB6r584RW1hAtj0Buz7qvVe+zJ8vOk4//g5DYC5U5KY1Yoza2NC/fhszlDCA71JO1vEr+dvo9JsbbXXd0eSLLkqD2+Y+Yk63iDvKHz5K7BU6R3VRRRF4fFFe1l7KAcfTyMf3DWIruGtX933hn4d+e3YLgA8/tVeDldP1RXCbax6HvZ9qXaHzfgQInu1fgw9roeR1ctPfXc/ZO1r/RgaYXdGPs9+r5akeXBCPPeO6dLqMcSF+fPZnKEE+3qyJyOff/yc2uoxuBNJllxZQDu47XN1bblj6+CnP6lXjw7kyx2n+HrnaUxGA2/e3p/+MW11i+UPkxIY1jmUkkoLv/1kByX1LKcihMvZ8zms/6e6fc2/1EKTehn/JHSZAOYy+Px2h2sVLyqv4sEFuzBbFab2juKRifqVHukaHsA/blLHk727/hgrU87qFourk2TJ1UX0gBvfAwyw/QP15iBOnivl2cUHAHh0UgLjk/StTuthMvLvW/sRHuhNenYxc7/eh0wWFS7v/An48TF1e9Rj0H+WvvEYTeo5q20nyD+h1nmyWvSNqZqiKPzl2/2czCulQxtfXrihl+4zaJN7RHLX8E4A/OHLPWQWlOkaj6uSZMkdJE6BidUDJn9+Qu2W05nFqvDoF7spqbQwqFNb7tOhGftS2gV685/b+2MyGli85wwfbz6hd0hCtByrVR2nVFkE0UNg3P/pHZHKLwRmfqrWdzq6Glb9Ve+IAPh652m+230Gk9HAv2/tR7CvY5Q4mHt1Ej07BJFfWsVDC3Zjtsj4peYmyZK7GP6QupaTuQwWP6ieJHX09tojbD9xngBvD16d0RfTJdZ408ugTiHMnZIEwPM/HGTvqXx9AxKipWx7Vx3Q7ekH17915ZW5m1NkT7juDXX7l3lweoeu4RzNKebJ7/YDakv4gFj9hgxcyNvDxBu39ifA24Otx/P418p0vUNyOZIsuQujEa79N3j4qifHnfN1C2XfqQJeq66e/fQ13Vu0ltLlmj0yjik9I6myKPx50T65UhOuJ/cwLK9ucZ70HIQ6RutuHT1vhN4z1TIoix/SbZJKpdnKg5/vorTSwrDOoQ7TEl5bpzB/XpiuDsp/Y/VhfqlexFc0D0mW3ElIZ3UBTIBlT0F+RquHUFZp4eGF6uDIKT0juWlAx1aPoTEMBgPPX9+TYF9PDmYW2orOCeESLGb49j61pbnzWBg4W++I6jf5BfANgbP71JICOpi34hD7TxfS1s+T12Y6Vkt4bdf2ac8tg6JRFJj79T7KqxxjrJcrkGTJ3Qy5Vx2bUFkEPzzc6rPjXv45lSM5JYQHejvE4MiGhAV48+fq7rhXlx/iTL4MnBQuYuO/4dQ2dR3J6/7j2Oux+YepCRPAmr+3+pjLIznFvLtefc0Xp/d2+PXY/jKtOxFB3pzMK+WdtfqPT3UVDvwfIlqE0aSuGWfyhsMr1OUFWklKZiEfVrfQvHxTb9r6e7Xaa1+umQOjGRjbltJKC89+f0DvcIS4ctkp6gK2AFNegmDHbN2to88taguYuRy+f7jVLvK0dd+qLArjk8K5qmdkq7zulQjw9uAvU7sD8Oaaw2TkleockWuQZMkdtUuAcXPV7aV/hsLMFn9JRVF4/oeDWBW4ulckYxPDW/w1m4PRaOCvN/TEw2jg5wNnWXFQ6pgIJ6YosHQuWKsg4Sp14VpnYDDAtNfU5VCOrW21i7zlB8+y7lAOXiYjT03r3iqv2Rym9Y5ieJdQKsxWW/FMcWUkWXJXwx6AqL5QXgArn2vxl1t28Cwbj5zDy8PI3CndWvz1mlNSZBCzR8UB8PTiA5RWSrFK4aQOr1Cn4pu84KoX1STEWYR0hrF/Vrd//j8oadkBzOVVFp7/UU005oyOo1OYf4u+XnMyGAw8e20PPIwGVqScZVWqXORdKUmW3JXJA6a+qm7vWdCiywpUmC28UL1A7ZxRcQ45+82ehybE06GNL6fzy/jXCpmWK5yQxazWWQN17GJIZ33juRzD7oeIXlB2vmYmXwt5Z+1RMvLKiAr24ffjurboa7WE+IhAZo9UL/KeWXxQBntfIUmW3FnHAdBjOqC06Innf78c58S5UsIDvfndWOc76QD4eXnw/PU9AHhvwzEOZxfpHJEQTbTzQ8hNU2eWjXpM72guj8lT7Y4D2POZOv6qBWTklfLmmsMA/N/V3fDz8miR12lpD0yIl8HezUSSJXc34UkwesKRlXBkVbMfPqeogjdWqSedP12VhL+3c550AMYnRTCxWwQWq8Kr1XWihHAK5QU1g7rHzgXfNrqGc0WiB0G3a9TaSyufb5GX+NuPKVSYrQztHMK03lEt8hqtQQZ7Nx9JltxdSGcYVF1jZflTzV7Z+5Wf0yiuMNOnYzDT+3Vo1mPr4Y+TEzEYYMm+LPadKtA7HCEaZ/2rUJoLofEw8Nd6R3Plxj8FBiOk/QgZW5v10BsP57L0QBYmo4Fnr+3p0OVNGqP2YO/X5CLvskmyJGD0H9V6K1n7YN+XzXbY/acL+GKHWvjyqWu6Y3TQQm5NkRgZyHV92gPwyrI0naMRohHOn4DNb6rbyX9Vu7KcXbsE6Hu7ur3imWYrJaAoiu3/+o4hMSRGBjbLcfVkMBhs9eK+3X2aw9nFOkfknCRZEmrRt5EPq9urnoeq8mY57Ms/p6EoalXZAbEhzXJMR/DwxAQ8jAbWHsph67E8vcMRomErnwVLJcSNgYTJekfTfMb+Wa0Xd+IXOLyyWQ65Lj2XnSfz8fYwOuWg7vr07tiGSd0jsCrwb1k37rJIsiRUQ34Lge2hIAO2/veKD7c7I591h3IwGQ08lpzYDAE6jk5h/tw8MBpQuxmVVq6CLkSjndkN+xcBBpj8N+cqFWBPcEcYPEfdXvnMFQ8hUBTF1k11x9BYwoMcu1J3Uz08MR6A7/ee4dBZmaDSVJIsCZWXH4yvnla8/hV1au4VeL366uWGfh2ICXW+UgH2PDihK14eRrYez2NduixYKRzU+lfU+143Q2QvfWNpCaP+UDOE4MDXV3SotYdy2J2Rj4+nkXvHOGFZBTt6tA9mSs9IFEVd6040jSRLokafWyG8uzpzZsvlty7tP13AytRsjAZcqim7tqhgX2YNjQWkdUk4qOwUSPle3R71B31jaSl+ITD8QXV71V/BXHlZh1EUhdeq66fdMSSW8EDXalXSPDwxwTZB5eCZQr3DcSqSLIkaRlPNSXXL21BZclmH0frEr+3TnjgnqnrbVL8b2wV/LxP7Thfw84EsvcMRoq711UVnu10L4Un6xtKShv4W/MPh/DHY/ellHWJNWg57bK1KXZo5QMeRGBnItN7qBBVpXWoaSZZEXd2vh7ZxUJYHOz5s8tNTMgtZdvAsBgPcP941W5U0oQHe/Ka6Qu4ryw5htUrrknAQ547A/q/U7dFOWoCysbwDYNSj6vbGf4O1aZWqFUWxJQ53DutEu0Dv5o7QoTw0IR6jQV2CSsqfNJ4kS6IukweMeEjd3vRGk5u1tQKUU3tF0TXc+afd2nP3qM4E+nhwOLuYFSmy/pJwEBteU4s2xidDVB+9o2l5/e8E37aQd7Sm67GRVqdls+dUAb6eJu4Z7XpjlS7UNTyA6/qqNe9ek9alRpNkSVysz60QEAGFp2HfF41+WvrZIpbszwRcv1VJE+zrye1D1LFL766X5QSEA8jPgD2fq9uj/6hvLK3Fyx8GVc+M++Vfja67pLYqqcMG7hwWS1iAa7cqaR6cEI/JaGBVara0LjWSJEviYp4+MOz36vaGeY1u1n5j9WEUBa7qEUlSZFDLxedgfj2iE54mA9uOn2fnySubRSjEFdv4b7BWQadRED1Y72haz5B7wcMHzuyE4xsa9ZR16bnsdaNWJU1cmD/XVhfXfW+DXOQ1hiRL4tIG/gZ8guFcOqT+aHf3oznFfL/nDAAPTHCPViVNRJCPrVn73XVy4hE6KjpbM9bQXVqVNP5h0O8OdfuXeY16yvsbjgFwy+BoQt2kVUkzu3q85Y97MzmTX6ZzNI5PkiVxad6BMPgedXvDq3abtd/bcAyrAhO7hdOjfXArBOhYtKvSpQeyOHHu8mYRCnHFNr0BlgroOBjiRusdTesbdr+6ZtzhFZC1v8Fd088Wse5QDgYD/Hp4XCsF6Dh6dghmWOdQzFaFDzcd1zschyfJkqjfkPvAwxfO7IKja+rdLb+0kq93ngLUAc/uKCEikLGJ7VAUeG/9Mb3DEe6ovAC2f6Buj37Mtap1N1ZInDqjF9TuyAZ88Iv6f5rcPcIlC+c2xt2j1CTxsy0nKakw6xyNY3O6ZOnNN98kLi4OHx8fBgwYwPr16+vd96677sJgMFx069Gjh22f+fPnX3Kf8vLmWR/NqfmHqbNMQG1dqsfn2zIor7LSLSqIIXGuswZcU2mtS1/uyCCv5PKK4wlx2XZ/BpXF0C5JnQXnrkZUF6nc9xXkn7zkLnkllXy98zQAs0e65wUewLjEcDqH+VNUbubL7Rl6h+PQnCpZWrhwIQ8//DBPPPEEu3btYtSoUUyZMoWTJy/9D/Gvf/2LzMxM2y0jI4OQkBBuvvnmOvsFBQXV2S8zMxMfH9es4Npkwx8AowccW6cuKXABs8XKx5tOAOpAZ4M7Xs1WG9Y5lJ4dgiivsvLJ5hN6hyPcidVas6bj4Hvcs1VJ076fumiwYoFNb15yl083n6DCbKVXh2AGdWrbygE6DqPRYKsV98Evx7FIrbh6OVWy9OqrrzJ79mzuvvtuunXrxrx584iOjuatt9665P7BwcFERkbabtu3b+f8+fP8+te/rrOfwWCos19kZGRrvB3n0CYaul2jbm9996JvLz94ltP5ZYT4e9lmV7grg8HAnOpuyA83Hqe8qmnF8YS4bEdWqjWGvIOh90y9o9GfVitu54dQmlfnWxVmCx9VX8zMHhnn1hd4ADf270gbP09O5pWy/KDUiquP0yRLlZWV7Nixg+Tkus3LycnJbNy4sVHHeP/995k4cSKxsbF1Hi8uLiY2NpaOHTsybdo0du3a1eBxKioqKCwsrHNzadpA731fQll+nW/975fjANw2OAYfT1PrxuWApvaKokMbX87VauYXosVteVu973eHWtHa3XUZry4cXFUKuz6p860f9mSSU1RBRJA3V/eK0ilAx+HrZeKO6lpx70sZgXo5TbKUm5uLxWIhIiKizuMRERFkZdlflyszM5OffvqJu+++u87jSUlJzJ8/n8WLF7NgwQJ8fHwYMWIE6enp9R7rxRdfJDg42HaLjo6+vDflLGKGQXgP9cSz+zPbw/tPF7D1eB4eRgOzhsU2cAD34WEy2pq1399wVBbYFS0v97A6+wsDDL7b7u5uwWCoKVK5/QO1mxK1CKVWLuDOYZ3w8nCaj8AWdeewWFutuN0Z+XqH45Cc7i/lwiZTRVEa1Yw6f/582rRpw/XXX1/n8aFDh3LHHXfQp08fRo0axRdffEFCQgKvv/56vceaO3cuBQUFtltGhosPjDPUOglve9d24tFala7uFUVEkIzx0swY2BE/LxNHckrYcizP/hOEuBLbqrvHEyZDiPsOVr5Ir5vUbsnzx+DoKgA2H83jYGYhPp5Gbh8So3OAjiM8yIdr+6i14rRkUtTlNMlSWFgYJpPpolak7Ozsi1qbLqQoCh988AGzZs3Cy8urwX2NRiODBg1qsGXJ29uboKCgOjeX12sGeAep4yKOriKnqMJWhPLXIzrpG5uDCfTxtBWp/HTLpScfCNEsKopg16fqttZdLlRe/tD3VnV7m1pSQUsE1HE6DX8WuButSOWSfZlkFchs8As5TbLk5eXFgAEDWL58eZ3Hly9fzvDhwxt87tq1azl8+DCzZ8+2+zqKorB7926ioqQvuw7vAOh7u7q99T0WbD1JpcVK3+g29Itx39kk9dGuWpfuzyS3uELnaITL2r0AKosgNB46j9M7Gscz8Dfq/aGfyDxxiJWp6gDmX49wvyKU9nRvH8TgTiFYrIqUEbgEp0mWAB599FHee+89PvjgA1JSUnjkkUc4efIk9913H6B2j915550XPe/9999nyJAh9OzZ86LvPfvss/z8888cPXqU3bt3M3v2bHbv3m07pqhlkNoVpxxayspNWwFpVapPzw7B9I1uQ5VF4Qs58YiWULtcwJB7wehUp/PW0S5RXSNPsXJi+VsoCozoGkrXcBkEfym3DlHH336+LUPKCFzAqf67Zs6cybx583juuefo27cv69atY8mSJbbZbZmZmRfVXCooKGDRokX1tirl5+dzzz330K1bN5KTkzl9+jTr1q1j8GA3WoCyscK6QudxGFCYUv4T4YHeTOkpLXD10VqXPttyEquceERzO7paXbvRKxD63KJ3NI5rkHrujz/1NZ6YuXWwjFWqz5SeUQT7enI6v4x16Tl6h+NQDIpM17lihYWFBAcHU1BQ4Prjl1J/hM9vI08J4KNhP/HwVb31jshhlVVaGPLCCgrLzcz/9SDGJobrHZJwJZ/dAod+UpclmvKS3tE4LksVFf/ohnd5Do8bHuW5J/6Ct4eUOanPs98f4H+/HCe5ewT/vXOg3uG0uMZ+fjtVy5LQX0bYaE4rYYQYipkVuFPvcByar5eJGwd0BGSgt2hmhZmQ/rO6PUjKBTTI5MnP3pMB+F3AGkmU7LituuVtZWo2ZwtloLdGkiXRJF/uyuQT80QAQg98qHM0jk/riluZcpbMgjKdoxEuY89noFjVGmhh8XpH49AyC8p4KXsIFsVAbNFOyE7VOySHFh8RyKBObbFYFb7YJuMtNZIsiUbTZkkstIzFavCEMzvh7EG9w3JoXcMDGRIXglWBz7fKiUc0A0WpqUrdb5a+sTiBL7ad4rQSyk6foeoD2z/QNyAnoI3rkoHeNSRZEo22Lj2HzIJyrH6hKAlqsza7P9U3KCdw+1B1AsLn205itlh1jkY4vRO/qPXOvAKg+3V6R+PQLFaFhdvULvDK/tVlBPYsgMoSHaNyfFf3koHeF5JkSTTawuqWkRv6dcDUv/qKds/nYKnSMSrHN7lHBKH+XpwtrGBlarbe4Qhnp7Uq9Zwu68DZsS49hzMF5QT7ejJg3A3QJhYqCiHlB71Dc2g+niam91cL6y6Q8ZaAJEuikXKKKliRohZ0mzkoGrpOhIAIKM2FQz/rHJ1j8/YwcdNAdaD351vlxCOuQHkBHPhW3e53cU05UZf2QT+9fwd8vDyh723qN6RF3K5bZaB3HZIsiUb5eucpzFaFvtFtSIoMApMH9J6pflNOPHbNGKgWe1uXnkt2kZx4xGXavwjMZdAuCTq6/rTuK5FdWG5rybXVVtLqUR1bB/kyhrAhCRGBDIyVgd4aSZaEXYqisLD6n+WWQdE13+h3h3p/6GcoOqtDZM6jS7sA+ka3wWJV+G7XGb3DEc5q58fqfb871AWuRb2+3HEKi1VhQGxbEiIC1QfbdlIreqPA3s/1DM8p3FY9m3fh9gy3L6wryZKwa/uJ8xzNLcHPy8S0Pu1rvtEuEToOAsUCexfqF6CT0GouLdp5SudIhFM6e0CdgWr0gN5SsbshilKzzFCdCzyo1RX3mTqzUNRrSs8o/L1MnDpfxvYT5/UOR1eSLAm7tCnv03pHEeDtUfeb2uK6uz+VE48d1/SOwstkJDWriANnCvQORzgbbWB34hQIaKdvLA5u58nznDhXip+Xiam9L1iSqdu14OmvzijM2KJPgE7C18vE1b3Un9/Xbn6RJ8mSaFBReRU/7lO7jWYOusSaSj2ng4cv5KTC6R2tHJ1zaePnxcTu6pIni3ac1jka4VTMFerMU5CB3Y3w9U71/+uqHpH4eV1wgecdAD2uV7dlvKVdN1TPivtxXyblVRado9GPJEuiQT8fOEt5lZXOYf70j2lz8Q4+wdD9WnVbu/IV9bqxv9oV993u01RJzSXRWGlLoCwPAttD1wl6R+PQKswWftibCdR80F9E64rb/w1UlrZSZM5paFwo7YN9KCo322ZEuyNJlkSDvt2lXqHd0K8DhvoGlGpdcfsXyYnHjtEJ7QgL8OJcSSVr06TYm2ik3QvU+z63gFHWNmvI6tQcCsqqiAjyZniXsEvvFDNcrblUWQSpUnOpIUajwZZ0frPTfVvEJVkS9TpbWM4vR3IBuK5vPVdooM4uaROjFnuTE0+DPE1G289SBnqLRik5B0dWqtt9ZGC3Pd/sUv+vruvbAZOxngs8o1FqLjXBDf3UFvE1h3LILa7QORp9SLIk6vX9njMoCgyIbUtMqF/9OxqNtQZ6f9Y6wTkxrStuZUo2+aWVOkcjHN7Bb8Bqhsje6gxUUa/80kpWVddWuqFfAxd4UJN4Hl0LBXLh0pCu4QH06RiMxaqweLd7lj6RZEnU65vqLrjr7Z10AHrPUO+PrYViWdKjId3bB9EtKohKi5Xv97jniUc0wd4v1XutCKyo1w97M6myKCRFBtItKqjhndt2gtiRgFIzeF7Ua3r1RZ72ueBuJFkSl3TobBEHzhTiYTQwrVeU/SeEdIYOA0GxwoFvWj5AJ3dj9RiAr9x4DIBohPMnIGMzYICeN+odjcPTPsin1zew+0JaV9yeBVL6xI5r+rTHw2hg3+kC0s8W6R1Oq5NkSVySNrB7bGI4bf29GvekXjer9/u+bKGoXIc2nmJPRj6Hs4v1Dkc4Ku1/KW40BDXiosWNnThXwo4T5zEa7IyxrK37tWrpk3OHIXN3i8bn7EL8vRibqJY++doNW5ckWRIXsVoVvqvul7bb719bjxvAYIRT2yDvWAtF5xraBXozNkEtLPitG554RCMoSk2ypHVzi3pprUojuoYREeTTuCd5B0LCZHV7/6IWisx1aC3i3+46jcXNlj+RZElcZNvxPE7nlxHo7cGEbuGNf2JghHoFDLD/q5YJzoVc21ddOuaHvWdQpAtAXChrn1rs1eQN3a7ROxqHpihK07vgNL1uUu/3fw1WqX3WkPHdwgny8SCzoJzNR8/pHU6rkmRJXOTb3epJZ0qvSHw8m1jTxdYV95WMAbBjYrcIfDyNHD9Xyv7ThXqHIxzNvi/U+8Sr1OKvol47T+bbljeZ3COyaU/uOgm8g6DwdPX4MFEfb4+a9UHdbaC3JEuijgqzhR+rq99e39h+/9q6XaNeCeekqgt/inr5e3swoVsEAN/vlVlxoharBfZVdwv1ki44e7Su7Esub2KPp09Ny90+aRG357rqZOnnA1lUmN1n+RNJlkQdq1NzKCw3Exnkw5DOoU0/gE8wJCSr2zLQ265reld3xe05g9XNxgCIBpz4BYrOqP9P8ZP0jsahmS1WftqvXuBpXdtN1nO6en/wW7BUNU9gLmpQpxAigrwpKjez7lCu3uG0GkmWRB3aFdp1fdvXX/3WHq0rbv8iGQNgx9jEdgR6e3CmoJydJ8/rHY5wFHuru+C6Xw8e3rqG4ui2HMsjt7iStn6ejOhaz/Im9sSNBb8wKD2nFqkU9TIaDVxdXU7mBzdqEZdkSdgUV5hZlaYWlLzsKzSA+GR1DEBBBmRsaaboXJOPp4nk6jEWUqBSAFBVDgcXq9syC84ubdHcq3pG4mm6zI80kwf0uF7dlskpdl1T3RW34uBZyqvcoytOkiVhszLlLJVmK53D/Olur/ptQzx9a40BkK44e67po16l/bgvE7NFWuLc3uHlUFEAQR3VBV9FvaosVpZWd8FN7XUFF3gAPatnxaX8oCasol79otvQoY0vJZUWVqe6x4oNkiwJG+0KbWrvKAyGy+yC02jTcQ98I2MA7BjRNYy2fp7kFley+Wie3uEIvWkV8Htcr667KOq18cg5zpdWEervxdDOIVd2sOghaoJaWQTpy5onQBdlMBiY1lvrisvUOZrWIf+JAoCi8irWHsoB1GTpinUaDf7hUJYHR1Zf+fFcmKfJyJTqMQDSFefmqsogbam63WO6vrE4gR+rx8xM6RWJx+V2wWmMxpqB3tIVZ9e06skpK1PPUlJh1jmalifJkgBgZUo2lWYrXdr5kxgReOUHNHmoFb1B1oprBG1W3E/7M6k0S1ec2zq8AqpKIDgGOvTXOxqHVmm2snR/FtAMXXAarUX80M9QLrXPGtKzQxCxoX6UV1lZ6QZdcZIsCaB2F1z7K++C02gDJtN+BHNl8xzTRQ2OCyE80JvCcjPr03P0DkfoRbuw6H4tNNf/oYvacFgtc9Iu0JvBcVfYBaeJ7A2h8WAuh7QlzXNMF1W7K84dWsQlWRIUllexTuuC69WMi3VGD4GACCgvgGMyHbchJqPB1v3pDicecQl1uuBu0DcWJ/DDHm1gd9Tllzm5kMFQd/kT0SCtK25tWg6F5a49NtXpkqU333yTuLg4fHx8GDBgAOvXr6933zVr1mAwGC66paam1tlv0aJFdO/eHW9vb7p3784337hXt9GKg2eptFjpGh5AQkRA8x3YaKqZFXfw2+Y7rovSpuMuO3iWskr3mI4rarF1wUVDhwF6R+PQyqssLD94FmimMZa1db9evT+6Wr3QE/VKigykSzt/Ki1Wlh84q3c4LcqpkqWFCxfy8MMP88QTT7Br1y5GjRrFlClTOHnyZIPPS0tLIzMz03aLj4+3fW/Tpk3MnDmTWbNmsWfPHmbNmsWMGTPYssV96gNpy5tM7dUMs+AupJ14Un+UWXF2aNNxSystrD3k+mMAxAUOfKved79OuuDsWHcoh6IKdaWBATFtm/fg4UkQlgiWSnXskqiX2hVXsyC4K3OqZOnVV19l9uzZ3H333XTr1o158+YRHR3NW2+91eDzwsPDiYyMtN1MpprFYefNm8ekSZOYO3cuSUlJzJ07lwkTJjBv3rwWfjeOoaCsivXpasn6Zr9CA4gdrlbGLTsPx9Y1//FdiMFgYEpPtUDlT9UDV4WbqCqDQ9VdcNoFhqhX7TInxubqgqut+7Xq/cHvmv/YLkarE7c+PZf8Utcdm+o0yVJlZSU7duwgOTm5zuPJycls3Lixwef269ePqKgoJkyYwOrVdaexb9q06aJjTp48ucFjVlRUUFhYWOfmrLQuuISIABKaYxbchep0xcmJx54pvdRkaVVKtlstUun2Dq+EymK1zk/HgXpH49DKqyysSGmhLjhN9+vU+8MroKK4ZV7DRXQNDyQpMhCzVeHnA657kec0yVJubi4Wi4WIiIg6j0dERJCVdelfUFRUFP/9739ZtGgRX3/9NYmJiUyYMIF162paOLKyspp0TIAXX3yR4OBg2y06OvoK3pm+ftzXTNVvG6LNikv9ASyuX4/jSvSLbqsuUllh5pfD7rNIpdurXYhSuuAatCYtm9JKCx3a+NIvuk3LvEhET2gbp86KO7y8ZV7DhWgTg5a6cIu40yRLmgvH1CiKUu84m8TERObMmUP//v0ZNmwYb775JlOnTuWVV1657GMCzJ07l4KCAtstIyPjMt+NvgpKq2zT1Kf2jmy5F4odCb4h6iKVJza03Ou4AKPRwOTqteJ+2ue6Jx5Ri3TBNYnWRX11r8jmH2OpMRhqWpekRdyuq6qHD/xy+JzLzopzmmQpLCwMk8l0UYtPdnb2RS1DDRk6dCjp6em2ryMjI5t8TG9vb4KCgurcnNGyg1lUWRQSIwLpGt4CXXAakwd0m6Zuy4nHLu3EszzlLFWyVpzrky64RqswW1iVok5+uKpnC3XBabRxS4eWqQmtqFfX8AA6V8+Kc9W14pwmWfLy8mLAgAEsX163SXT58uUMH974xSZ37dpFVFTNP9mwYcMuOuayZcuadExnVXOF1sInHai5Yk75HqwyFqchgzuFEOLvRX5pFVtkrTjXp5XVkFlwdm08co6iCjPhgd4t1wWnad9fLeNQVaImtKJeBoOBq6pbxF113JLTJEsAjz76KO+99x4ffPABKSkpPPLII5w8eZL77rsPULvH7rzzTtv+8+bN49tvvyU9PZ0DBw4wd+5cFi1axP3332/b56GHHmLZsmW89NJLpKam8tJLL7FixQoefvjh1n57raq4wsyG6llw2qDiFhU3GnzaQEkOnGh4QL678zAZSe6utmz+tN89Fql0W1XltQpRXq9rKM7g5+oLvMk9IltmFlxtBgN0q25dSlncsq/lAqZUt/StTs1xyTpxTpUszZw5k3nz5vHcc8/Rt29f1q1bx5IlS4iNjQUgMzOzTs2lyspKHnvsMXr37s2oUaPYsGEDP/74I9On1yxQOXz4cD7//HP+97//0bt3b+bPn8/ChQsZMmRIq7+/1rQmLZtKi5W4MH/iw5uxEGV9TJ6QJF1xjaV1xf184CwWq6JzNKLFHF2trnIf2B46SBdcQyxWxVaIUhvX1+K0rri0n8Bc0Tqv6aR6dgiiQxtfyqosrHPBJZucKlkC+N3vfsfx48epqKhgx44djB492va9+fPns2bNGtvXf/rTnzh8+DBlZWXk5eWxfv16rr766ouOedNNN5GamkplZSUpKSl1kilX9XN1tdXkHhEtN0jyQtqVc8pi6YqzY3iXMAJ9PMgtrmDnyfN6hyNaSuoP6n23aeqq96Je24/nca6kkmBfT4Z0bqa14OzpOBgCIqGiEI7Kkk0NMRhqJqf87IKz4uS/0w1VmC22QXitdoUGEDcGvIOh+CxkuE+F9Mvh5WFkUrfqrjiZFeeaLGa1xQJqWl1FvZZWj4WZ2C0CT1MrfXQZjVInrgm0FvEVKWepNLvW5BRJltzQxsPnKK4wExHkTd+ObVrvhT28IPEqdTv1x9Z7XSdV0xWXhaJIV5zLydisltPwaaNWuhf1UhTF1lqh/V+0Gq2EQJos2WTPgNi2hAV4U1huZvPRc3qH06wkWXJD2myF5O6tMEjyQklT1fvUH0ASgAaNTmiHn5eJ0/ll7D0lC3q6nJTqLrjEKeqYPlGvfacLOFNQjp+XiVHxYa374rWXbDpe/8LtAkxGA8k9tMkprtUiLsmSm9FlkGRtXSeChw+cPw7ZB1v/9Z2Ij6eJcUnhgOudeNyeotS0rkoXnF1aZehxieH4eJrs7N3MjKaaizwtwRX10koILD+Y5VKTUyRZcjM7Tpxv/UGStXn5Q+dx6rZ0xdmlLay7dH+mdMW5kqy9UHASPHyhy3i9o3FoiqLYkqXJrd0Fp9HGLaX9BFbXGovT3IZ2DiXIx4Pc4kp2nHCdySmSLLkZrQtuQlJ46w2SvFDtrjjRoHGJ4Xh5GDl+rpT0bFnQ02VoLRRdJ4CXn76xOLjD2cUczS3By2RkXGI7fYKIGw1eAVB0BjJ36RODk/DyMDKxenKKK60VJ8mSG3GIKzRQx2gYjJC5B/Kdc1291uLv7cHIruoYDa37VLgA7UJBuuDs0s5ZI7qGEuij09guD291CAFA6hJ9YnAirjg5RZIlN3LgTCGn88vw8TQyOl6nKzQA/zCIHqpup8mJx55J1dW8l7noMgJu59wRdbyewQQJk/WOxuH9fFCnWXAXsrWIy/ABe0YntMPXU52csv90od7hNAtJltyI9mE7JqEdvl6tPEjyQtIV12gTuoVjMMCeUwWcLSzXOxxxpbQP204jwU+HcYNOJCOvlP2nCzEasHXt6CZ+Ehg9ICdFTXhFvXw8TYxJUC/Ilx90jYs8SZbciFa1W5dZcBdKqq6kfvwXKJXFYhsSHuhD3+pFQ6UrzgVIF1yjrUhR/94HdgohNMBb32B820LsCHVbWsTtsrWIu8g5S5IlN3Est4S0s0V4GA1MSNL5Cg0gpDOE9wDFAunL9I7G4WknHkmWnFzRWcjYqm5rrauiXlqypC0srTstwZVxS3aNSwrHaIDUrCIy8kr1DueKSbLkJrSm0GFdQgn2c5ACeNIV12jah8WmI2r1deGk0pYACrTvD8Ed9I7GoRWUVbHlqNrqPMlRkqXEKep9xmYodr3FYptTiL8XAzup3cxa0uvMJFlyEysOqmvBOcxJB9TFQwEOr4SqMn1jcXBd2gXQOcyfSouVtWlyknZati44aVWyZ01aNmarQnx4ALGh/nqHo2oTDVF9QLHCoaV6R+PwtIs8SZaEU8grqWT7CfUKbYLegyRri+wNwdFQVQpHVusdjUMzGAy1uuJcY8Ck2ymvtXK9VuRQ1GtFinqBN9GRLvAAEqsTXRm3ZJc2KH/L0TwKypx7XT1JltzA6tRsrAp0jwqiQxtfvcOpYTDIdNwm0JKlVanZVFmkirDTObIKrFUQ0gXCEvSOxqFVmq2sSXXA1nCoOWcdWQWVJfrG4uA6hfkTHx6A2aqwJi1b73CuiCRLbkBrAp3YLVznSC4hqdZVmtWibywOrl9MW0L9vSgsN7PtmMwgdDpat03iFPVCQdRr67E8iirMhAV40bdjG73DqSuiB7SJAXO5tIg3wkQXmZwiyZKLqzBbWHdIHePicM3ZADHDwacNlOXVzBISl2QyGphQnfC6ynRct2G1wKGf1W1tkLCol3aBNyEpAqPRwRJLg6HWrDhpEbdHaxlcm5ZDpdl5W8QlWXJxm4/mUVJpISLIm57tg/UO52ImD7XYG8Chn/SNxQlM6q6t6H3WZZYRcAsZW9ULAp82NdXrxSUpimJrhXC4LjhNYnWduENLwSKzUxvSt2MbwgK8Kaows+XYOb3DuWySLLm4FdUnnQndHPAKTZNwlXqfJrNL7BnZNQwfTyOn88s4mOkaywi4BW0wcPwk9QJB1Csls8i2LNOI6nURHU7MMLVIZVmeWkZA1MtoNNiGgDhzV5wkSy5MURRbc/YkR5oFd6GuE9VlBHLTIO+o3tE4NF8vE6PitWUEnPfE43Zqj1cSDdLOWSO7OsCyTPUxeUB89bp+UkLALq2FcIUTt4hLsuTCDpwpJLOgHF9PE8O6hOodTv1826hXaiCtS42Q7CIDJt3GuSOQe0i9INBWrhf10v6uHaZqd320RZDlnGXXiK5h+HqaOFNQzoEzztkiLsmSC9Ou0EbFh+Hj6aBXaBrtilvGLdk1LkldWPfAmUKyCmRhXYeXVv03HTsCfBxw3KADySwoY9/pAgwG9e/coXWdoCbA59JlYV07fDxNjIpXu1Sd9SJPkiUXZisZ4OhXaFAzbunERigv0DcWBxcW4E2/6oV1V6U6d+0StyBdcI2mFaLsF92GdoE6L5xrj09wzcK60hVn10Qnr+YtyZKLyiwoY//pQgwGGO/oV2gAodWF+qxmOLxC72gcnlaJfVWqc5543EbZefUCAGouCES9VthmwUXqHEkjaQlwmrSI2zOhVov4mXznW95KkiUXtbL6Cq1/TFvCAhz8Ck0js+IaTUuANxzOpbxKink6rPQVoFigXTcIidM7GodWUmFm0xF1arlDFtC9FG3c0slNUJavayiOLjTAm/4xbQFY7YTVvCVZclE1VbudoAtOo12lpS+T2iV2JEUG0qGNL+VVVjYeydU7HFEfrWSAdMHZteFwLpUWK7GhfnQND9A7nMYJ6QxhiWqL+JGVekfj8LSLvFUpkiwJB1BSYWbjYfUKbVJ3J7lCA+g4WK1dUp4PGVv0jsahGQwG24lnhROeeNyCuRIOV3+ASrJkl/YBOj4pHIMzLQeTKC3ijaWtQLDhcC5llc7VIi7Jkgtan15zhdalnZNcoUF17ZJkdVtmxdk1vlvNVZqz1i5xaSc3QkUB+LeDDgP0jsahWa0Kq6q7ZiYkOVFrONQMHzi8XFrE7UiMUFvEK8xWNh11rhZxSZZc0OpUJ71CAxm31ATDOofi62kiq7Bcqnk7Iu1vOH4yGB28dIfO9p8pIKeoAn8vE4PjQvQOp2m0FvGy83BK1rdsSO0W8ZVO1iIuyZKLqX2F5hSz4C4ktUsazcfTxMjq2iXOduJxeYpS0zqaKLPg7NH+fkfFt8PLw8k+lmq3iMusOLtsLeKpztUi7mR/lcKeA2cKnfcKDdTaJZ1Gqtty4rFrgnaVJvWWHEtuOpw/DiYv6DxO72gcnlYvbLyzzIK7kDYr7tDP+sbhBLQW8cyCclIyi/QOp9GcLll68803iYuLw8fHhwEDBrB+/fp69/3666+ZNGkS7dq1IygoiGHDhvHzz3X/mOfPn4/BYLjoVl7unJWRV1bX3RkZH4a3h5M2/Sdo1bylK84erfVwT0Y+2UXO+TfrktKrzzOxI8DbicYN6iC7sJx9p9VCtOMSnTRZkvUtG83H02RbINmZ6sQ5VbK0cOFCHn74YZ544gl27drFqFGjmDJlCidPnrzk/uvWrWPSpEksWbKEHTt2MG7cOK655hp27dpVZ7+goCAyMzPr3Hx8fFrjLTW72uOVnJZ2lXZio9QusSM8yIfeHdUlNNak5ugcjbDRWhikEKVdWs2dPs5Qtbs+PsEQO1zdlvGWdmmz4pypRdypkqVXX32V2bNnc/fdd9OtWzfmzZtHdHQ0b7311iX3nzdvHn/6058YNGgQ8fHxvPDCC8THx/P999/X2c9gMBAZGVnn5oyyi8rZc8rJr9BALd4XlqAW8zuySu9oHJ5twKQTXaW5tPICtUghQEKyvrE4AW280gRnvsCDmsRYWsTt0j6fdmfkk1tcoXM0jeM0yVJlZSU7duwgObnuySc5OZmNGzc26hhWq5WioiJCQuqO5SkuLiY2NpaOHTsybdq0i1qeLlRRUUFhYWGdmyNYk6a2LPTqEEx4kHO2jNloAybTl+kbhxPQplqvT5dq3g7hyCq1SGFovFq0UNSrvMrChsPqFHKnbg2HWutb/iLrW9oRGexDzw5BKErN55ajc5pkKTc3F4vFQkRE3RocERERZGVlNeoY//znPykpKWHGjBm2x5KSkpg/fz6LFy9mwYIF+Pj4MGLECNLT0+s9zosvvkhwcLDtFh0dfXlvqpnVLurm9LSuuPTlYLXqG4uD69khiIggb0orLWw5lqd3OOJQdYKv/Q2Lem05lkdppYWIIG96tA/SO5wrE9pFTZCtZji6Ru9oHN74JOda39JpkiXNhXWDFEVpVC2hBQsW8Mwzz7Bw4ULCw2uSiaFDh3LHHXfQp08fRo0axRdffEFCQgKvv/56vceaO3cuBQUFtltGRsblv6FmUmm2us4VGkDMMPAOgtJcOLNT72gcWt3aJc5x4nFZVmtNa6gkS3atqv57dcqacJdiK6orLeL2aN2u6w/lUml2/Atip0mWwsLCMJlMF7UiZWdnX9TadKGFCxcye/ZsvvjiCyZOnNjgvkajkUGDBjXYsuTt7U1QUFCdm962Hc+juMJMWIA3vToE6x3OlTN5QpfqKdcyHdcubQzA6jTnql3ics7sVBN87yA14Rf1UpTaNeGcrGp3fRJqDR+QFvEG9eoQTFiAN0UVZrYfd/wWcadJlry8vBgwYADLly+v8/jy5csZPnx4vc9bsGABd911F5999hlTp061+zqKorB7926ioqKuOObWpA2SHJfYDqPRBa7QQK18DDXTsEW9RnQNw8tkJCOvjCM5JXqH4760xL7LODXhF/U6nF1MRl4ZXh5GRnQN1Tuc5hEzHLwCoCQbsvboHY1DMxoNjE9qBzjHrDiPxuy0ePHiRh/w2muvvexg7Hn00UeZNWsWAwcOZNiwYfz3v//l5MmT3HfffYDaPXb69Gk++ugjQE2U7rzzTv71r38xdOhQW6uUr68vwcFq68uzzz7L0KFDiY+Pp7CwkH//+9/s3r2b//znPy32PlqCNv12grMWdbuU+EnqfeYeKMqCQOecpdga/L09GNI5hPXpuaxOzXaeVdtdTbqUDGgs7QNyeJdQ/Lwa9VHk+Dy8oPNYSP1B7Ypr30/viBza+KRwvth+ilWp2Tw5rbve4TSoUX+h119/faMOZjAYsFhabjbOzJkzOXfuHM899xyZmZn07NmTJUuWEBsbC0BmZmadmkvvvPMOZrOZ3//+9/z+97+3Pf6rX/2K+fPnA5Cfn88999xDVlYWwcHB9OvXj3Xr1jF48OAWex/N7WhOMcdyS/A0GRgZ307vcJpPQDi07692baQvh/6z9I7IoY1LDFeTpbRs5oyWWVitrjBTTewxQNdJekfj8Fa5SsmACyVMVpOl9J9h7ON6R+PQRsa3w9Nk4FhuCcdyS4gL89c7pHo1KlmyOlDf6+9+9zt+97vfXfJ7WgKkWbNmjd3jvfbaa7z22mvNEJl+tKUCBseFEODtIldomoTJ1cnSz5Is2TE+KZznfjjI1mN5FJVXEegj3UCtShvY3aE/BLjQRUsLKCitYsfJ8wCMdeaacJeiJcqnd0JxjvwtNCDA24PBcSH8cvgca9KyiQuL0zukel3RmCVnXRLE1djWVXKVQZK1abNLjqwBc6WuoTi6TmH+xIX5Y7Yq/FI9M1K0Ii1ZipdZcPasP5yDxaoQHx5AdIif3uE0r6AoiOwNKHB4hd7RODxtcsoqBx+31ORkyWKx8Pzzz9OhQwcCAgI4elRdB+fJJ5/k/fffb/YARcOKyqvYVj2TwCVKBlwoqi/4h0NlEZxsXPFRd+YsJx6XY66AI6vVbSkZYNfq6qV5xrniOQtq1YmTySn2aC2LW47mUVpp1jma+jU5Wfrb3/7G/Pnzefnll/Hy8rI93qtXL957771mDU7Y98vhc1RZFOKqWxVcjtEotUuaYFz17JLVaTlSQqA1Hd8AVSUQEAlRffSOxqFZrQprD6nJ/NhEF+2i0s5Zh1eBxXETAEfQpZ0/0SG+VFqsbDx8Tu9w6tXkZOmjjz7iv//9L7fffjsmU82q9r179yY1NbVZgxP2rUlz8ZMO1KpdIldp9gyOC8HPy0ROUQUHzjjGMjxuIb26pEn8JHCF4ootaN/pAnKLKwnw9mBgbIj9JzijDgPALxQqCiBji97RODSDwVDTIp7muC3iTU6WTp8+TdeuXS963Gq1UlVV1SxBicZRFMVWMsCpF861p/M4MHrCucNw7oje0Tg0bw8TI7qGAbBauuJaj1TtbjTtnDUqPgwvD6cp9dc0RhN0rS6ALBd5dmndsWtSHbeobpP/Unv06MH69esvevzLL7+kXz+pKdGaUjKLOFtYga+nicFxLnqFBuATBLHV1ZBlYV27tLFrqx34Ks2lnDsCeUfUhD5ujN7ROLzV1QunuvQFHtRaDHx5w/sJhnUOxdvDyJmCcg6dLdY7nEtq8jzzp59+mlmzZnH69GmsVitff/01aWlpfPTRR/zwww8tEaOoh/ZhOKJrKD6eJjt7O7n4yXBsnVoheehv9Y7GoWldsrsy8skrqSTE38vOM8QV0T4MY4epib2oV25xBXtP5QMwxpWHDgB0GQ8GI2QfhPwMaOMYC647Ih9PE8O7hLI6LYfVadkkRgbqHdJFmtyydM0117Bw4UKWLFmCwWDgqaeeIiUlhe+//55Jk6QQW2uqGa/k4ldoUHOVduIXqJTlPBoSFexLt6ggFAXWHcrROxzXd7g6WZJClHatO5SDokCP9kFEBPnoHU7L8guB6CHqtrSI26V1xTnqTN7L6jCePHkya9eupbi4mNLSUjZs2EBycnJzxyYaUFBaxY4TWlE3F79CAwiLhzaxYKlUW5hEg8ZV/0046onHZVSWwrHqYQnxcg60x2264DTakk2SLNml/U3sOHGegjLHG/982aPrtm/fzscff8wnn3zCjh07mjMm0Qjr0nOwKpAQEUDHti5W1O1SDIZaJx4ZA2CPNm5p7SG1+J9oIcfXg6UCgmOgXaLe0Tg0s8Vqa+nUSly4PK1A6dG1UCVFnBsSHeJH1/AALFaFDemOV1S3ycnSqVOnGDVqFIMHD+ahhx7iwQcfZNCgQYwcOZKMjIyWiFFcglvMgrtQ7QGTDjpjwlH0jW5DsK8nBWVV7KpeVkK0AFvVbikZYM+ujHwKyqpo4+dJ3+i2eofTOiJ6QGB7MJfBiQ16R+PwHLlFvMnJ0m9+8xuqqqpISUkhLy+PvLw8UlJSUBSF2bNnt0SM4gJWq8La6uZstxivpOk0CkzeUHASctL0jsaheZiMjEnQClQ63onHJShKrWRJuuDs0UpZjEloh8noJomlwQDxWgkBWfrEHu3if+2hbKwO1iLe5GRp/fr1vPXWWyQm1jQ5JyYm8vrrr1+ypIBofvtOF3CupLqoWyc3uUID8PKDTiPV7cPSFWePNpZtTZoM8m4RuemQf1JN4ONG6R2Nw3O78UoaW4u4jFuyZ2CnEPy9TOQWV7L/TIHe4dTR5GQpJibmksUnzWYzHTp0aJagRMNqF3XzNLloUbf6yImn0UYntMNggANnCskulPESzU77G+w0ArxccKmhZpRVUE5KZiEGg/p36VbixoDRQ63FJUV1G+TlYWRkvFpU19G64pr8Sfvyyy/zwAMPsH37dlulze3bt/PQQw/xyiuvNHuA4mJue4UGNYO8T2yCiiJ9Y3FwYQHe9O4QDMAaKSHQ/KQLrtG0Mid9o9u4X90vnyCIqS6qe1i64uzRPtccrUW8UclS27ZtCQkJISQkhF//+tfs3r2bIUOG4OPjg7e3N0OGDGHnzp385je/ael43Z5bFXW7lNAuENIZrFXqDBPRoDHaGAAHO/E4vYoiOLFR3ZZkyS6tNXxsghte4IHM5G0CbRzunlP5nCuu0DmaGo2q4D1v3rwWDkM0llsVdatPfDJseVu9su82Te9oHNq4xHb8e2U669JzMFuseLhbt21LObpWTdhDOqsJvKhXpdnKL9WrybtNyYALxSfD8qfUUhNVZeDpq3dEDisy2IekyEBSs4pYn57L9f0cY3hPo5KlX/3qVy0dh2gkt+6C08RPUpOlwyvUGUkyZbtevTu2oa2fJ+dLq9h5Mt+11xBsTdoEA2lVsmv7iTyKK8yEBXjRs32w3uHoo10SBHWEwlNwfENNS5O4pLGJ4aRmFbEmLdthkqUruswsKyujsLCwzk20nNpF3dyiand9YkeChy8UnlbXXRL1MhkNUkKguSlKTXeKfOjZpXUBj05oh9FdSgZcqE5RXZmcYo9Wb2ldeq7DFNVtcrJUUlLC/fffT3h4OAEBAbRt27bOTbScPafUom5BPh70jW6jdzj68fSBuNHqtpx47BrroAMmnVb2QTVR9/BVE3fRoDXuWBPuUmonS1JUt0H9Y9sS6O1BXkmlbYyu3pqcLP3pT39i1apVvPnmm3h7e/Pee+/x7LPP0r59ez766KOWiFFUW51ac4Xm9mNPbCcemV1ij1ZCICWzkKwCKSFwxbQEPW6UmriLep3JLyPtbBFGA4yunhLutuLGgNETzh+XEgJ2eJpqSgg4ykVekz9xv//+e958801uuukmPDw8GDVqFH/5y1944YUX+PTTT1siRlFtzaHqGSXufoUGNcnSyU1Q7ljFyxxNiL8XfTq2AdTKuOIKaQm6jFeyS/ug6xfTljZ+blYy4ELeARA7XN2WFnG7bCUEHKTsSZOTpby8POLi4gAICgoiLy8PgJEjR7JunawG31Kyi8rZf1odEzbG3Yq6XUrbThCWAIoFjqzWOxqHJ9W8m0l5IWRsVre7TtQ3FiewxlYyQM5ZQE2CLSsQ2KWVxtnrICUEmpwsde7cmePHjwPQvXt3vvjiC0BtcWrTpk1zxiZqWXdIXYW5Z4cg2gV66xyNg+ha3bokJx67tKu0Dem5VFmsOkfjxI6uAasZQrtCSJze0Tg0tWSAet6S1vBqWov48Q1QWaJvLA4uIsiH7lFBKAqsS9f/Iq/JydKvf/1r9uzZA8DcuXNtY5ceeeQR/vjHPzZ7gEK1xt2Lul1K7QUqZcBkg3p1CCbU34uiCjM7TpzXOxznJSUDGm378TxKKi2EBXjRo32Q3uE4hrAEaBMDlko4Jmup2uNILeJNTpYeeeQRHnzwQQDGjRtHamoqCxYsYOfOnTz00EPNHqCoWzLAbYu6XUrsCPD0g+IsyNqndzQOzVirhIAjnHickqLUjFeSLji7tLEmbl0y4EIGQ02LuIxbsktrkVx7KEf3EgJXPKUqJiaG6dOn06dPn+aIR1zC7ox8CsvNBPt60jdayjPYeHirM0xAuuIaYYztKk0GeV+Wsweg6IyaoMeO0Dsah6f9nbl1Ad1Lia81fEBaxBvUP6YNgT4e5JdWsUfnEgKNquD973//u9EH1FqdRPPRWgJGxYdhkiu0uuInwqGf1Cv+UX/QOxqHNjq+HUYDpGYVkVlQRlSwLLnQJFpC3klKBthzOr+MQ2eLMRrU85aoJW40mLwg/yTkpkO7BL0jclgeJiOj49vx475M1qTl0D9Gv8aCRiVLr732WqMOZjAYJFlqAVIyoAFak3bGFijLB982ekbj0Nr6e9E3ug07T+azJi2HWwfH6B2Sc7GVDJCq3fZorUpSMuASvPzVlsmjq9WuOEmWGjQmUU2W1qZl8+gk/X5WjUqWjh071tJxiHpIyQA72saqgyZzD6kzlXpcr3dEDm1sYnh1spQtyVJTlBdIyYAmsFXtlnPWpcVPUpOlw8th+P16R+PQtL+hPacKyC2uICxAn9ngbl4G2vFpJQN6dQiWkgH1kRICjabNLvnl8DkqzVJCoNFsJQPipWSAHRVmCxurSwaMS5LW8EvSzlknNkJFsb6xOLjwIB/bbMp1OhaodLpk6c033yQuLg4fHx8GDBjA+vUNT79cu3YtAwYMwMfHh86dO/P2229ftM+iRYvo3r073t7edO/enW+++aalwm8ybfFTt1441x4pIdBoPdsHExbgRbGUEGgaWTi30bYfP19dMsCb7lFSMuCSwuJrSggclxIC9oxLDKd7VBBeHvqlLE6VLC1cuJCHH36YJ554gl27djFq1CimTJnCyZMnL7n/sWPHuPrqqxk1ahS7du3i//7v/3jwwQdZtGiRbZ9NmzYxc+ZMZs2axZ49e5g1axYzZsxgy5YtrfW26mW2WFl/SFuEUpKlekkJgUYzGg2Mjq+eFSdLnzSOosDhleq2dMHZpY1XGiMlA+pXp4SAtIjb8+ikBJY8NIppvdvrFoNTJUuvvvoqs2fP5u6776Zbt27MmzeP6Oho3nrrrUvu//bbbxMTE8O8efPo1q0bd999N7/5zW945ZVXbPvMmzePSZMmMXfuXJKSkpg7dy4TJkxg3rx5rfSu6iclAxrJw1udYQLSFdcIWgmBtVJvqXGkZECT2MYryQVew6SEQKM5QtLtNMlSZWUlO3bsIDm5buXc5ORkNm7ceMnnbNq06aL9J0+ezPbt26mqqmpwn/qOCVBRUUFhYWGdW0vQTjqjE9pJyQB7utbqihMNurCEgLBDSgY02un8MtKzpWRAo1xYQkA4tCYnS0uXLmXDhg22r//zn//Qt29fbrvtNs6fb7kxELm5uVgsFiIiIuo8HhERQVZW1iWfk5WVdcn9zWYzubm5De5T3zEBXnzxRYKDg2236Ojoy3lLduWXVeJpMsiMksaIv6CEgKiXVkIApJp3o0jJgEbTuuD6S8kA+7QSAiAt4k6gycnSH//4R1tLyr59+/jDH/7A1VdfzdGjR3n00UebPcALGQx1W1gURbnoMXv7X/h4U485d+5cCgoKbLeMjIxGx98Uf72+F7ueSubqXlEtcnyX0raTOlNJsagzl0SDtJpdUs3bjtolAyRZsmt1qnTBNUm8LH3iLJqcLB07dozu3bsD6iyyadOm8cILL/Dmm2/y008/NXuAmrCwMEwm00UtPtnZ2Re1DGkiIyMvub+HhwehoaEN7lPfMQG8vb0JCgqqc2spAd4e+HqZWuz4LiVeSgg0lpQQaKTaJQPadtI7GodWYbaw8YjaYi8FdBtJSgg4jSYnS15eXpSWlgKwYsUK23ifkJCQFhu7o73ugAEDWL687gfh8uXLGT58+CWfM2zYsIv2X7ZsGQMHDsTT07PBfeo7pnBgtqs0KSFgj5QQaCQpGdBo24+fp1RKBjSNlBBwGk1OlkaOHMmjjz7K888/z9atW5k6dSoAhw4domPHjs0eYG2PPvoo7733Hh988AEpKSk88sgjnDx5kvvuuw9Qu8fuvPNO2/733XcfJ06c4NFHHyUlJYUPPviA999/n8cee8y2z0MPPcSyZct46aWXSE1N5aWXXmLFihU8/PDDLfpeRAuQEgKNJiUEGkFR4LCMV2osKRlwGaSEgNNocrL0xhtv4OHhwVdffcVbb71Fhw4dAPjpp5+46qqrmj3A2mbOnMm8efN47rnn6Nu3L+vWrWPJkiXExsYCkJmZWafmUlxcHEuWLGHNmjX07duX559/nn//+9/ceOONtn2GDx/O559/zv/+9z969+7N/PnzWbhwIUOGDGnR9yJagJQQaBIpIWDH2f1QlCklAxpJSgZcJikh4BQMiiK/nStVWFhIcHAwBQUFLTp+STTC1ndhyWMQMxx+03Jj6FzB+ZJKBvx1OVYFNs0dT1Swr94hOZb1r8LKZyHhKrhtod7ROLRT50sZ+dJqjAbY9WQywX6eeofkPCpL4KVOalfc77fJwrqtrLGf341qWao9FunC+kKtUW9IiEaTEgKNVruEgLQuXYLWBSdVu+3SWpX6x7SVRKmpapcQkFlxDqtRyVLbtm3Jzlb7o9u0aUPbtm0vummPC6Grtp0gLKG6hMBqvaNxeNqspdVSQqCu8gI4KSUDGku64K6QzOR1eB6N2WnVqlWEhITYthuqQSSE7uKTIfeQOiuuxw16R+PQxia249Xlh2wlBPRcqNKhHFmtJtxhCVIywA4pGdAMuk6Cn/+vpoSAd4DeEYkLNCpZGjNmjG177NixLRWLEM2j60TY9IZ6lWa1glESgPpoJQRyiyvZceI8w7qE6h2SY9Cu8LtKq5I9UjKgGYTFQ5tYyD8Bx9ZB0tV6RyQu0ORPkSeffBKLxXLR4wUFBdx6663NEpQQVyR2OHj6Q/FZOCslBBpiNBoYXb2cjlTzrqYocHiluh0v45XskZIBzcBgkK44B9fkZOmjjz5ixIgRHDlyxPbYmjVr6NWrF8ePH2/O2IS4PB7e0Lm6NVRql9hVs/SJDPIGpGRAE8l4pWYSX72ge7qUEHBETU6W9u7dS6dOnejbty/vvvsuf/zjH0lOTuauu+6qs8CuELrSZjBJsmTX6PgwjAZIO1vEmfwyvcPRnzYjKW6MmniLep06X0p6djFGA7Yip+IydRoFJm8oyICcNL2jERdo1Jil2oKDg/n888954oknuPfee/Hw8OCnn35iwoQJLRGfEJdHa9I+tRXKzoOvzNSsTxs/L/rFtGXHifOsScvhtiExeoekr3Stard0wdkjJQOakZcfdBoJR1aqCXt4kt4RiVoua+Tr66+/zmuvvcatt95K586defDBB9mzZ09zxybE5WsTA+2SQLGqM5tEg8ZVd6G4fQmBsny1RhfI4O5G0MYrjUuSWXDNQsYtOawmJ0tTpkzh2Wef5aOPPuLTTz9l165djB49mqFDh/Lyyy+3RIxCXB7pims0bdzSxsO5VJgvnsDhNo6uqS4ZkAhtY/WOxqFVmC38cvgcIOOVmo02bunEJqgo0jcWUUeTkyWz2czevXu56aabAPD19eWtt97iq6++4rXXXmv2AIW4bNqJ5/AKtYSAqFeP9kG0C/SmpNLC9uPn9Q5HP1piLYUo7dp6LI+yKgvhgVIyoNmEdoG2cWCtgqNr9Y5G1NLkZGn58uW0b9/+osenTp3Kvn0yTVs4kJhh4BUAJdmQtVfvaByawWBgbHUJgdWpbtoVpyiyxEkTrE5VxyuNSwyXQsXNSbriHFKzVusLCwtrzsMJcWU8vKDzWHVbuuLsspUQOOSmJQSy9kJxllqjK3a43tE4PG28knTBNTMpIeCQmpwsWSwWXnnlFQYPHkxkZCQhISF1bkI4FK2FQK7S7BoZH4bJaOBwdjEZeaV6h9P6tJIBncdKyQA7Tpwr4WhuCR5GAyPi5SK5WXUaCR4+UHgaslP0jkZUa3Ky9Oyzz/Lqq68yY8YMCgoKePTRR5k+fTpGo5FnnnmmBUIU4grYSghsg9I8fWNxcMG+ngyIUUssuGXrkoxXajStZMDATm0J8pGSAc3K01etuQQ1CbzQXZOTpU8//ZR3332Xxx57DA8PD2699Vbee+89nnrqKTZv3twSMQpx+YI7QniP6hICq/SOxuGNTape+sTdxi2V5qkJNUiy1AhaiYlxsnBuy7CNW1qhbxzCpsnJUlZWFr169QIgICCAgoICAKZNm8aPP/7YvNEJ0Ry0E49cpdk1NqG6hMCRc5RXuVEJgSOr1IQ6vIeaYIt6lVVa2HRELRkg9ZVaiDZ84OQmKC/UNxYBXEay1LFjRzIzMwHo2rUry5apH0Dbtm3D21v6+YUDqj1g0upGCcBl6BYVSESQN2VVFrYec6NuSy2RTkjWNw4nsPnoOSrMVjq08SU+PEDvcFxTaBcI6QJWMxyVorqOoMnJ0g033MDKleqK3A899BBPPvkk8fHx3HnnnfzmN79p9gCFuGLRQ8A7GMry4PROvaNxaGoJAbW1wG2qeVsttcYrSbJkj/Z3MSaxnZQMaEkJk9V7aRF3CE1eG+7vf/+7bfumm24iOjqaX375ha5du3Lttdc2a3BCNAuTB3QdDwe+UU880YP0jsihjUtqx8LtGaxJy+Hpa/SOphWc3qkm0t7B0HGw3tE4NEVRbIO7ZbxSC4ufBJvfrG4Rt4KxWSv9iCa64p/+kCFDePTRRyVREo7N1hX3s75xOIERXcPwNBk4llvCsdwSvcNpedqVe9fxamIt6nU0t4STeaV4mYwM7xKqdziuLXaEWvOr+KwU1XUAkqoK96ANmMzcA0VZ+sbi4AJ9PBnUSa2Z5hbVvLVkSbrg7NL+HoZ0DsHfWxLLFuXhXauornTF6U2SJeEeAsKhfX91W6bj2jU+yU3GLRWdhczd6rYscWKX1gU3VrrgWoc24UCSJd1JsiTch9ZycEi64uzRPgy3HM2jpMKsczQtSEuc2/dTE2pRr5IKs22GpCxx0kq0c9ap7VCSq28sbk6SJeE+tBPPkdVgqdI3FgfXpZ0/MSF+VFqsbDjswidp6YJrtA2Hc6m0WIkN9aNzmL/e4biHoPYQ0QtQ4PBKvaNxa01Olu666y7WrVvXErEI0bLa9wO/MKgsgpNSbb4hBoPB1hW3xlW74ixVauIMkiw1wqqUmqrdUjKgFSXI5BRH0ORkqaioiOTkZOLj43nhhRc4ffp0S8QlRPMzGmtV85YTjz1adebVqTkorrj6ecYWqCgAv1A1kRb1UhTFNn5tvFTtbl1aIn94JVhcuEvcwTU5WVq0aBGnT5/m/vvv58svv6RTp05MmTKFr776iqoq6doQDs6WLC3XNw4nMCQuBF9PE1mF5RzMdMElF2wlAyaC0aRvLA7uwJlCsosq8PMyMaRziN7huJeOg8C3LZTn16xfKFrdZY1ZCg0N5aGHHmLXrl1s3bqVrl27MmvWLNq3b88jjzxCenp6c8cpRPPoMh4MJshJhfMn9I7Gofl4mhjRVa2l45IlBA7JeKXGWlX9+x/RNQxvD0ksW5XRBF0mqNsyK043VzTAOzMzk2XLlrFs2TJMJhNXX301Bw4coHv37rz22mvNFaMQzce3rbr8CciJpxFsXXHVU8ZdxvkTkJOiJs5dxusdjcPTkqUJ0gWnD1n6RHdNTpaqqqpYtGgR06ZNIzY2li+//JJHHnmEzMxMPvzwQ5YtW8bHH3/Mc8891xLxCnHlbF1xcuKxRyshsOvkec6XVOocTTPSfvfRQ8BPupUacq64gj2n8oGa5Fm0sq4TAQOc3Q8FMk5YD01OlqKiopgzZw6xsbFs3bqV7du3c9999xEYGGjbZ/LkybRp06Y54+T8+fPMmjWL4OBggoODmTVrFvn5+fXuX1VVxeOPP06vXr3w9/enffv23HnnnZw5c6bOfmPHjsVgMNS53XLLLc0au3Aw2lXasXVQWapvLA6uQxtfkiIDsSqw9pALtS4dWqrea38Lol5r0nJQFOjRPoiIIB+9w3FPfiHq2CWQizydNDlZeu211zhz5gz/+c9/6Nu37yX3adu2LceOHbvS2Oq47bbb2L17N0uXLmXp0qXs3r2bWbNm1bt/aWkpO3fu5Mknn2Tnzp18/fXXHDp06JJr2M2ZM4fMzEzb7Z133mnW2IWDCe8OwdFgLofj6/WOxuFprQmrXGXcUmUJHKv+vSdcpW8sTmCVzIJzDLYSAjI5RQ9NXtynoQSlpaSkpLB06VI2b97MkCHqeJN3332XYcOGkZaWRmJi4kXPCQ4OZvnyun9Ur7/+OoMHD+bkyZPExMTYHvfz8yMyMrJl34RwHAaD2qKw7T21hUFaFxo0Pimct9YcYe2hHCxWBZPRyWvsHF0LlgpoEwvtLj53iBpVFivrqserSReczuKTYdVf4egaqCoHT2nla01OUcF706ZNBAcH2xIlgKFDhxIcHMzGjRsbfZyCggIMBsNFXYSffvopYWFh9OjRg8cee4yioqIGj1NRUUFhYWGdm3AyWovCoZ/BFWsINaN+0W0I9vWkoKyKXSfP6x3OlavdBSfFFRu0/fh5iirMhPh70adjG73DcW+RvSEwCqpK4MQGvaNxO06RLGVlZREefvFVTXh4OFlZjVtBvry8nD//+c/cdtttBAUF2R6//fbbWbBgAWvWrOHJJ59k0aJFTJ8+vcFjvfjii7axU8HBwURHRzftDQn9dRoFnn5QeBqy9ukdjUPzMBkZnaCuBeb0XXGKUjPmQ1oU7dIKUY5NaOf8LYrOTmsRB0hbqm8sbkjXZOmZZ565aHD1hbft27cDXLK8vqIojSq7X1VVxS233ILVauXNN9+s8705c+YwceJEevbsyS233MJXX33FihUr2LlzZ73Hmzt3LgUFBbZbRkZGE9+50J2nD3Qeq27Lwrp2TXCVcUtZe6EoEzz9IXak3tE4PO33LV1wDiJhinovLeKtrsljlprT/fffb3fmWadOndi7dy9nz5696Hs5OTlEREQ0+PyqqipmzJjBsWPHWLVqVZ1WpUvp378/np6epKen079//0vu4+3tjbe3d4PHEU4gYTKkLVG7Zcb8Ue9oHNqYhHYYDZCaVcSp86V0bOund0iXR0uMu4yTMR92ZOSVcji7GJPRYGtZFDqLGw0ePlBwErIPQkQPvSNyG7omS2FhYYSFhdndb9iwYRQUFLB161YGDx4MwJYtWygoKGD48OH1Pk9LlNLT01m9ejWhoaF2X+vAgQNUVVURFRXV+DcinJNWufn0DijOhgC5eq5PW38vBsaGsPV4HitTsvnV8E56h3R5tPFKUrXbLq1VaWBsW4J9PXWORgDg5QdxY9S1LQ8tlWSpFTnFmKVu3bpx1VVXMWfOHDZv3szmzZuZM2cO06ZNqzMTLikpiW+++QYAs9nMTTfdxPbt2/n000+xWCxkZWWRlZVFZaVaXO/IkSM899xzbN++nePHj7NkyRJuvvlm+vXrx4gRI3R5r6IVBbWHqD6AItNxG2FCNzWZXJFycSuvUyjOhtPV3euSLNmlJUtSMsDBJFZPTpFxS63KKZIlUGes9erVi+TkZJKTk+nduzcff/xxnX3S0tIoKCgA4NSpUyxevJhTp07Rt29foqKibDdtBp2XlxcrV65k8uTJJCYm8uCDD5KcnMyKFSswmWT9I7dgmxUnJx57JnRTu7y3HM2juMIJVz9PXw4oENUXgqTluCElFWY2HTkH1CTJwkHEVw/yPrUNSnL1jcWN6NoN1xQhISF88sknDe6j1Brw1qlTpzpfX0p0dDRr165tlviEk0q4Cta+BEdWgbkSPLz0jshhdWnnT2yoHyfOlbIhPYerejpZwmErGSCFKO1Zn55DpcVKbKgfXdoF6B2OqC24g1pGIGuvOrOz7216R+QWnKZlSYgWEdUXAiKgshhO/KJ3NA7NYDAwIUltXVqR4mSz4syVcGS1up0gXXD2aL/fid0iGjXjWLQyaRFvdZIsCfdmNNaMX5ESAnZNrO6SWZ2ajcXqRFOXT26EyiLwD4eofnpH49AsVsU2Xkm64ByUliwdrm4RFy1OkiUhbFdpP0ntEjsGxYUQ6O3BuZJK20r0TkFLhOOT1QRZ1Gt3xnnySioJ9PFgUKcQvcMRl9K+n5r4VxZJi3grkbOGEJ3HgskLzh+H3HS9o3FoniYjoxPVmjsrnWVWnKJA6o/qduIUfWNxAssPVheiTAzH0yQfEQ7JaKzpTpauuFYh/wlCeAeoy5+A2rokGqR1xa10lnFL2SmQf0It5tdlnN7RODwtCZYuOAenVfNOkxbx1iDJkhBQ0xUntUvsGpsQXqeat8NLW6Ledx4LXv66huLoTpwrIT27GA+jgbEJkiw5NK1FPP8E5KTpHY3Lk2RJCKgp9JaxGUrO6RuLg9OqeYOTrBWXVt1aKF1wdmmz4AZ1CiHYT6p2OzTvAHX5E5CuuFYgyZIQAG1iILIXKFZ1KQHRoPG2at4OniwVnYXT6mLcUl/JPumCczJSQqDVSLIkhCZxqnqvDQYW9dLGLW0+cs6xq3lrHyIdBkBgpL6xOLiCsiq2HssDYFL3hhcoFw5CS5Yytkg17xYmyZIQmqSr1fsjq6CqTN9YHFyXdgHEhvpRabGyIT1H73Dqp41Xki44u9YeysFsVegaHkBsqIztcgptomtaxKV1qUVJsiSEJrI3BHWEqlI4tk7vaByawWBgYvVaccsOOmgJgcoSOLpG3U68WtdQnMGKg9IF55SSpqn3qUv0jcPFSbIkhMZgqGmBkK44u7SumlWp2ZgtVp2juYSja8Bcro5HC++udzQOrcpiZU2aOv5sUjfpgnMqibVaxCudYHaqk5JkSYjatK64Q0vB6oAJgAMZGNuWtn6e5JdWse34eb3DuZitC26qmgiLem0/fp7CcjMh/l70i2mrdziiKSJ7QXAMmMvg6Gq9o3FZkiwJUVvsSPAOguKzcGan3tE4NA+TkQm2rrgsnaO5gNVSUzNLxivZtaJ6Fty4xHBMRkksnUqdFnHpimspkiwJUZuHF3SdqG5LV5xdydVdccsPnkVxpCrCp7ZDaS54B0PscL2jcWiKovDzATXZndRdxis5paTqmbyHflIvFESzk2RJiAtpJ540uUqzZ1R8O3w8jZw6X0ZKZpHe4dTQfnfxk8AkxRUbkpJZxKnzZXh7GBmd0E7vcMTliB0OPsFQek4tIyCanSRLQlyo60QwekBOKpw7onc0Ds3Xy8SoePUD1qG64qRqd6NprUqjE9rh5+WhczTispg8IX6yui0t4i1CkiUhLuTbBmJHqNtpsrCuPbW74hzCuSOQm6YmvFqXqqiXVvohWQpROjdtckraEllYtwVIsiTEpUhXXKNN6BaB0QAHzhQ6xsK62pV17Ag18RX1ysgrJSWzEKMBW90s4aS6TlQX1s07KgvrtgBJloS4FK375uQmWVjXjhB/LwZ2UhfWdYjWpZTv1ftu1+gbhxPQuuAGx4XQ1t9L52jEFfEOhLgx6nbqD/rG4oIkWRLiUtrEQIQsrNtYWhfOsgM6J0uFmXBqq7qtVTYW9dJ+X5N7yLp5LqF2V5xoVpIsCVGfbtUftlpLhahXcnf1w3br8TzySyv1CyStuguu42AIitIvDieQW1zBthOycK5LSahuET+9Q71wEM1GkiUh6qN14xxeCRXF+sbi4GJC/UiKDMRiVViVmq1fILYuOGlVsmdlylkUBXp2CKJjWz+9wxHNISgKOgxUtw/J5JTmJMmSEPUJ7w4hncFSAYeX6x2Nw9O9K640D46tV7elC86un7UuuO7SBedStK64FBm31JwkWRKiPgYDdLtW3T64WN9YnMCk6g/ddek5lFfpUEX40FJQLBDRE0K7tP7rO5HiCjMbDucCkCzjlVyLds46thbKHHDNRiclyZIQDdFOPOnLoKpc31gcXM8OQbQP9qG00sL69NzWD0C7kpZZcHatTcuh0mylU6gfCREBeocjmlNYPLTrBlaz1IlrRpIsCdGQDv0hqANUFsuK3nYYDAYm91RbKX7a18qDSyuK4chKdVu64OzSqq0n94jEYJCFc11Od2kRb26SLAnREIOhpqVCZsXZdXUvdQba8pSzVJqtrffCh1eAuRzaxkFEj9Z7XSdUabbaBuFP7iGz4FxS9+vU+yOroMKB1mx0YpIsCWGPliyl/giWKn1jcXADYtoSHuhNUbmZX460Yldcaq0uOGkpadCmo+coKjcTFuBNv+i2eocjWkJ4dwjpok5OOSR14pqDJEtC2BMzDPzCoDwfjm/QOxqHZjQauKq1u+LMtT4QZLySXdrvJblHBEajJJYuyWCo6YpLka645iDJkhD2GE01a8VJV5xdWrK07OBZqiyt0BV3bB1UFEJAZE2NGXFJVRarbYmTqb2kaKdLs01OWQ6VDrBmo5NzmmTp/PnzzJo1i+DgYIKDg5k1axb5+fkNPueuu+7CYDDUuQ0dOrTOPhUVFTzwwAOEhYXh7+/Ptddey6lTp1rwnQinpJ14Un8AayuOxXFCgzuFEOrvRX5pFVuO5rX8C2pXzt2mgdFpTmm62Hz0HOdLqwjx92JIXIje4YiW1L4fBMdAVWnN5Adx2ZzmzHLbbbexe/duli5dytKlS9m9ezezZs2y+7yrrrqKzMxM223Jkrpr5jz88MN88803fP7552zYsIHi4mKmTZuGxaJDnRjhuOJGg3cwFJ+tWXtMXJKHyWir3bNkfwt3xVktkFr9Py1dcHYt2ae2Kk3uEYGHyWlO/+Jy1O6Kk1lxV8wp/ltSUlJYunQp7733HsOGDWPYsGG8++67/PDDD6SlpTX4XG9vbyIjI223kJCaq6mCggLef/99/vnPfzJx4kT69evHJ598wr59+1ixYkVLvy3hTDy8IPEqdVtOPHZNqe6K+3l/Fhar0nIvdOIXKM0FnzYQO6LlXscFmGt1wV0tXXDuQWsRP7RUHdsnLptTJEubNm0iODiYIUOG2B4bOnQowcHBbNy4scHnrlmzhvDwcBISEpgzZw7Z2TXrVu3YsYOqqiqSk5Ntj7Vv356ePXs2eNyKigoKCwvr3IQb0E48Kd+D0oIJgAsY1iWUYF9PzpVUsvVYC3bFHfhGve92DZg8W+51XMCWY3nklVTS1s+TYZ1D9Q5HtIaOgyAwSh3Td3St3tE4NadIlrKysggPD7/o8fDwcLKysup93pQpU/j0009ZtWoV//znP9m2bRvjx4+noqLCdlwvLy/atq07fTYiIqLB47744ou2sVPBwcFER0df5jsTTqXLePD0g4KTcGaX3tE4NE+T0bZW3NKW6oqzmGta+XpOb5nXcCE/Vs+Cm9wjUrrg3IXRWFOk9eB3+sbi5HT9j3nmmWcuGoB94W379u0Al6wyqyhKg9VnZ86cydSpU+nZsyfXXHMNP/30E4cOHeLHH39sMC57x507dy4FBQW2W0ZGRiPfsXBqXn4QX90KqbVoiHpN6VVdQmB/FtaW6Io7vl7tgvMNgU6jm//4LsRssfLzfumCc0vauKU0qRN3JTz0fPH777+fW265pcF9OnXqxN69ezl79uKVzHNycoiIaHwF2qioKGJjY0lPTwcgMjKSyspKzp8/X6d1KTs7m+HDh9d7HG9vb7y9vRv9usKF9LwRDn6rJkuTnpMCiA0Y0TWMQG8Psosq2HnyPAM7NfPsqwNfq/fdrwWTrqcyh7f1WB7nSipp4+fJsC7SBedWYoardeJKc9U6cV3G6R2RU9K1ZSksLIykpKQGbz4+PgwbNoyCggK2bq2ZhbRlyxYKCgoaTGoudO7cOTIyMoiKUq+sBgwYgKenJ8uXL7ftk5mZyf79+5t0XOFG4ieBVwAUZMCpbXpH49C8PUxMrO6K+2l//d3al8VSVVPzqod0wdlj64LrHomndMG5F5OHWlYDpEX8CjjFf023bt246qqrmDNnDps3b2bz5s3MmTOHadOmkZiYaNsvKSmJb75R/xiKi4t57LHH2LRpE8ePH2fNmjVcc801hIWFccMNNwAQHBzM7Nmz+cMf/sDKlSvZtWsXd9xxB7169WLixIm6vFfh4Dx9awpU7v9a31icQO1q3s3aFXdsLZSdB/92MgvODotVqZkF11u64NySdkGRshjMlfrG4qScIlkC+PTTT+nVqxfJyckkJyfTu3dvPv744zr7pKWlUVBQAIDJZGLfvn1cd911JCQk8Ktf/YqEhAQ2bdpEYGCg7TmvvfYa119/PTNmzGDEiBH4+fnx/fffYzKZWvX9CSeinXgOfKPW+RH1GpPQjgBvD84UlLPz5PnmO/D+6ivk7tdJF5wdW4/lkVusdsENly4499RpJAREqBcYR1frHY1TcpqzTEhICJ988kmD+yi1pnP7+vry88/2FxD08fHh9ddf5/XXX7/iGIWb6DIefIKhOAtOblJPROKSfDxNJHeP4Otdp1m850zzjFsyV0Kq1gV3w5Ufz8Ut0daC6x4hXXDuymhS/1e2vA37F0HCZL0jcjrynyNEU3l41VSLlq44u67p2x5QP7TNzbFW3NHVUF6gXinHDLvy47kwi1WxjReTWXBurueN6n3qj1BVpm8sTkiSJSEuh9YVd/A7td6PqNfIrmG09fMkt7iSjUfOXfkBtUGq3a9Xr5hFvbYcO0ducQXBvp6M6BqmdzhCTx0HqWvFVRbDIfu9LqIuSZaEuBxxY8AvtHo67jq9o3FoniajrVVj8Z4zV3awqnL1yhikC64Rvtul/ryv7iWz4NyewVBTvHX/In1jcULy3yPE5TB5qIOLQU48jXBd3w6AulZcedUVDIo/skpduiGwPUQPsb+/GyuvstgWMtZ+/sLNaV1x6cugXJbpagpJloS4XLbpuN/LdFw7Bsa2JSrYh6IKM2vSsu0/oT5aIcoe16tLOYh6rUnLoajcTFSwD4ObuyCocE6RvSA0HszlkLZE72icipxthLhcscMhIFIdbCzTcRtkNBq4po860Puyu+IqSyHtJ3VbuuDs+m73aQCu7dMeo1EqzQvUrrheN6nb0iLeJJIsCXG5jCa1hQPkxNMI11YnSytTsikqv4w1qlJ/VAentolVB6uKehWWV7EyVW3Bky44UYfWIn5kFZTm6RuLE5FkSYgrYZuOu0Rt+RD16tE+iM5h/lSYrSw/ePFaj3bt/Vy97z1T1uSzY+m+LCrNVuLDA+gWFWj/CcJ9tEtQu+OsZnU2r2gUSZaEuBIdB0GbGKgskjEAdhgMV9AVV5ytXgkD9Gl48W0B3+1Ru+Cu79cBgySW4kI9pSuuqSRZEuJKGAzQu/rDe8/n+sbiBK6tLlC5IT2XvJImDIrf9xUoVugwEEK7tFB0ruFsYbmtnpXW9SlEHdqYv+MboDBT31ichCRLQlwpraXjyEoouozuJTfSpV0APdoHYbYqtmU4GkXrgpNWJbu+33MGRVFnIEaH+OkdjnBEbWOrS28osO9LvaNxCpIsCXGlQrtAx8Fqy4eceOy6rrp1SZutZVd2KmTuAaNHzeBUUa/vdqtdnNrPWYhL6nOrer/7M6i1rqq4NEmWhGgOfaQrrrGu7dMBowG2HT/P8dwS+0/Yu1C97zoJ/ENbNjgndySnmH2nC/AwGpjaW5Il0YAeN4DJG3JSIHO33tE4PEmWhGgOPW4Akxec3QdZ+/SOxqFFBvswMr4dAIt2nmp4Z2ut1ro+M1s4Muf33S61tW50QjtC/L10jkY4NN820G2aur17ga6hOANJloRoDn4hkHCVui2tS3bdPKAjAIt2nMJqbaAL4MQvUJAB3sGQMKWVonNOiqLw3R7pghNN0Oc29X7fl7IKgR2SLAnRXLQxAPu+BItZ31gc3KTuEQT6eHCmoJxNR8/Vv6PWBdfjOvD0aZ3gnNTWY3mcOFeKv5eJSd0j9A5HOIMu49RVCMryIP1nvaNxaJIsCdFcuk4Ev1AoPgtH1+gdjUPz8TTZprV/taOerriqspqieb1lFpw9X2xXf47X9GmPn5eHztEIp2A01XRvS1dcgyRZEqK5eHjVFHvbIycee24eGA3AT/szKbzU8idpP0FFIQRHQ8ywVo7OuRSVV9lKMWg/VyEaReuKS/8ZSnL1jcWBSbIkRHPSZsWl/gDlhfrG4uD6dAyma3gA5VVWluy9RM0lbexX7xlglFNVQ37Ym0lZlYWu4QH0j2mjdzjCmYQnQft+6vInUvqkXnIGEqI5te8HYYlgLpd1l+wwGAzcVD3Q+6KuuILTcHi5uq1d+Yp6fbE9A4AZAzvK8iai6frert7v/kzfOByYJEtCNCeDoaZ1SU48dt3QT625tP3EeY7mFNd8Y9cnapHP2JEQ1lW/AJ1A+tkidp3Mx8No4IZ+HfUORzijnjeC0ROy9kLWfr2jcUiSLAnR3HrPBIMRTm6EnEN6R+PQIoJ8GJNwQc0lqwV2faxuD/iVTpE5D61VaXxSOO0CvXWORjglvxBI1EqfyHjLS5FkSYjmFtwB4ier2zvm6xqKM7hpgDog+eudp7FYFTiyWq2t5NMGul2rb3AOrtJs5eudaiHKGTKwW1wJrbt770KwXGLChZuTZEmIljDw1+r9ns+gqlzfWBzchG7hBPt6kllQzsYjubBzvvqNPrdKbSU7VqVmc66kknaB3oxNbKd3OMKZxU8C/3AoyYHUH/WOxuFIsiRES+g6UZ3yXnYeUhbrHY1D8/E02SpO/7hxj1oyAKQLrhG+rO6Cu7F/RzxMcjoXV8DkCf3vVLe3v69vLA5I/ruEaAlGU60Tz//0jcUJ3DIoBoCQ9K/UKcwdB0N4N52jcmxnC8tZnZYNwM0DZWC3aAYDfgUY4Ng6yE3XOxqHIsmSEC2l3x1gMKkDvbNT9Y7GoXVvH8SgmGBmGFepD0irkl2Ldp7CqsCgTm3p0i5A73CEK2gTAwnV4y3lIq8OSZaEaClB7WsW15WB3nY9HH+WTsazFOOHOek6vcNxaBarwudb1S44qdgtmtXA2er97k/VJYcEIMmSEC3LNtB7gZx47Bh6/gcAvjUPY8WREp2jcWxr0rI5mVdKsK8n1/Rur3c4wpV0nQDBMVCeDwe+1TsahyHJkhAtqcv4mhOPVPSuX8k5TGlqsrTAMp5PNp/QOSDHNn/jcQBmDorG18ukbzDCtRhNMPAudVsGettIsiRES5KB3o2zZwFYKqkM78VB4thwOJcjtSt6C5sjOcWsT8/FYIBZQ2P1Dke4on6z1Irep7ZB5l69o3EITpMsnT9/nlmzZhEcHExwcDCzZs0iPz+/wecYDIZL3v7xj3/Y9hk7duxF37/lllta+N0It6IN9M7YDNkpekfjeKwW2PoOAF5D7mZ8YjgAn24+qWdUDuvjTWqr24SkcKJD/HSORrikgHDodo26vf0DfWNxEE6TLN12223s3r2bpUuXsnTpUnbv3s2sWbMafE5mZmad2wcffIDBYODGG2+ss9+cOXPq7PfOO++05FsR7iYoChKnqNty4rlY6o+QfxJ8Q6D3TO4YpraWfLUjg7JKi87BOZbiCrNt0eFfDe+kbzDCtQ38jXq/70uoKNI3FgfgFMlSSkoKS5cu5b333mPYsGEMGzaMd999lx9++IG0tLR6nxcZGVnn9t133zFu3Dg6d+5cZz8/P786+wUHB7f0WxLuZvAc9X7Xp2qhSlFj81vq/cDfgKcvY+LbER3iS2G5me/3nNE3NgezaMcpiivMdGnnz8iuYXqHI1xZp5EQlgCVxeoSKG7OKZKlTZs2ERwczJAhQ2yPDR06lODgYDZu3NioY5w9e5Yff/yR2bNnX/S9Tz/9lLCwMHr06MFjjz1GUVHDWXRFRQWFhYV1bkI0KG4MRPSEqhIZu1TbmV1qHSqjBwy6GwCj0cAdQ9TWpY82H0dRFD0jdBhWq8KHm44DaquSwWDQNyDh2gyGmtalre+Bm/8fOkWylJWVRXh4+EWPh4eHk5WV1ahjfPjhhwQGBjJ9+vQ6j99+++0sWLCANWvW8OSTT7Jo0aKL9rnQiy++aBs7FRwcTHS01DkRdhgMMOz36vbW/4K5Ut94HIXWqtRjutpdWe3mgdF4eRjZf7qQ3Rn5+sTmYH45ksvRnBICvD2Y3l8qdotW0Pc28AqEnBQ4vELvaHSla7L0zDPP1DsIW7tt374d4JJXUYqiNPrq6oMPPuD222/Hx6fuwpxz5sxh4sSJ9OzZk1tuuYWvvvqKFStWsHPnznqPNXfuXAoKCmy3jIyMJrxr4bZ63gQBkVCUCQe+1jsa/RVmwv7qn8Ow39X5Voi/l61+0Lvrj7Z2ZA7pw+pyATcN6EiAt4e+wQj34BNcU03/l3/pG4vOdE2W7r//flJSUhq89ezZk8jISM6ePXvR83NycoiIiLD7OuvXryctLY27777b7r79+/fH09OT9PT618Xx9vYmKCiozk0Iuzy8YMg96vbGN9y+WZvt74O1CmKGQft+F337ntHq2MKf9mdxLNe9i1SePFfKylR1HbhZw6RcgGhFQ3+rdpMfXw+n629EcHW6JkthYWEkJSU1ePPx8WHYsGEUFBSwdetW23O3bNlCQUEBw4cPt/s677//PgMGDKBPnz529z1w4ABVVVVERUXZ3VeIJhvwa/D0g7P71MUq3VVVWc3MwKG/veQuiZGBjE8KR1Hgv+vcu3Vp/sbjKAqMig+TdeBE6wruCD2rZ5Bv/Le+sejIKcYsdevWjauuuoo5c+awefNmNm/ezJw5c5g2bRqJiYm2/ZKSkvjmm2/qPLewsJAvv/zykq1KR44c4bnnnmP79u0cP36cJUuWcPPNN9OvXz9GjBjR4u9LuCG/EOh7u7q96Q19Y9HT3i+g9Jy6cGfStHp3u29MF0BdNDa7qLy1onMoeSWVLNiq1pyaPTJO52iEWxr+oHp/8DvIO6ZvLDpxmo7vTz/9lAcffJDk5GQArr32Wt54o+6HTVpaGgUFBXUe+/zzz1EUhVtvvfWiY3p5ebFy5Ur+9a9/UVxcTHR0NFOnTuXpp5/GZGr+JQQsFgtVVVXNflxRPy8vL4xGB7smGPpb2PYepC+DnDRol2j/Oa5EUWoGdg++V61yXo9BndrSP6YNO0/m879fjvP4VUmtFKTj+GDDMcqqLPTqEMyYhHZ6hyPcUWRP6DIBjqyEzW/C1f+w/xwXY1BkXu4VKywsJDg4mIKCgkuOX1IUhaysLLsVx0XzMxqNxMXF4eXlpXcodX1+O6T+AP1/Bde6WdN2+gr49EbwCoBHD6qDSBuw7EAW93y8g0AfDzb+eTyBPp6tFKj+CsurGPH3VRSVm3n7jv5c1VOGBwidHF0DH12nDiN45IDaSu4C7H1+a5ymZcmZaYlSeHg4fn5+Uh+llVitVs6cOUNmZiYxMTGO9XMfdr+aLO35HMY/CQFu0mKgKLD2JXW7/6/sJkoAE7tF0KWdP0dySvhsy0nure6acwcfbzpBUbmZ+PAAkrtH6h2OcGdxYyCyN2TtVVvGx/xJ74halSRLLcxisdgSpdDQUL3DcTvt2rXjzJkzmM1mPD0dqEUiZih0GACnd8Dm/8DEZ/SOqHUcWQWntoKHD4x4qFFPMRoN3Du6C39atJf3NxzjrhGd8PZo/m5yR1NWaeGDDer4kN+N64LR6EDJvnA/BoP6P7toNmx5B4Y/AJ6+ekfVahxsMIfr0cYo+fnJgpd60LrfLBYHW2PMYIBRj6nbW96B4hx942kNigJr/q5uD5wNgfbLfmiu69eeiCBvsosq+HbX6RYK0LEs2HqScyWVRIf42mpOCaGr7tdDcAyU5sLuz/SOplVJstRKHKoLyI049M89cQq07w9VpfDLPL2jaXmX0aqk8fYw2WaCvbPuKFaraw+1rDBbbOUS7hvTBQ+TnKqFAzB51KxEsP5VqHKfGaryHygum8Fg4Ntvv9U7DOdlMMC4J9Ttbe+pFa1d1RW0KmluHRxDoI8HR3NKWOziC+x+vfM0WYXlRAR5c9MAWdpEOJABd0FQByg8VVMrzQ1IsiTqlZ2dzb333ktMTAze3t5ERkYyefJkNm3aBEBmZiZTpkzROUon13UCRA8BczlseFXvaFrOFbQqaQJ9PLm3uqr3P5enUWm2NmeEDsNssfL22iMAzBnV2S3GZwkn4ulTM7h7/T+houGF512FJEuiXjfeeCN79uzhww8/5NChQyxevJixY8eSl5cHQGRkJN7e3jpH6eRqty7tmA/5LrjOYDO0Kml+MzKOdoHeZOSV8dmWE80UoGP5dvcZTpwrJcTfi9uGxOgdjhAX63s7hHRRxy5pNdNcnCRL4pLy8/PZsGEDL730EuPGjSM2NpbBgwczd+5cpk6dCtTthjt+/DgGg4Gvv/6acePG4efnR58+fWytUJqNGzcyevRofH19iY6O5sEHH6SkxL3X/aLzGOg0CiyVsM4Fi701Q6uSxs/Lg4cmxAPw+qrDFFeYmyNCh1FWaeGVn9MAdW08Py+ZsCwckMkTxv2fur3xdSjN0zeeViDJUitTFIXSSrMut6bUHw0ICCAgIIBvv/2WioqKRj/viSee4LHHHmP37t0kJCRw6623YjarH2j79u1j8uTJTJ8+nb1797Jw4UI2bNjA/fff3+Sfo8sZ/xf1fvenrrWcQDO2KmlmDoomLsyfcyWVvLfetdaMe2/9UbIKy+nQxpe7hnfSOxwh6tdjOkT0gopC2PCa3tG0OKng3QwaqgBaXl7OsWPHiIuLw8fHh9JKM92f+lmXOA8+N7lJV6qLFi1izpw5lJWV0b9/f8aMGcMtt9xC7969AbVl6ZtvvuH666/n+PHjxMXF8d577zF79mz19Q4epEePHqSkpJCUlMSdd96Jr68v77zzju01NmzYwJgxYygpKcHHx6d53zAX//wd2sfT1eUE+twGN7hI0/bB7+CLO9VWpYf2NkuyBPDD3jPc/9ku/L1MrP3TOMICnL87OLuonLH/WENppYV/3dKX6/p20DskIRp26Gf4bIb6//3gbghyvgrzja3gLS1Lol433ngjZ86cYfHixUyePJk1a9bQv39/5s+fX+9ztEQKICpK/cfJzs4GYMeOHcyfP9/WahUQEMDkyZOxWq0cO+ZCrSmXSxu7tPdzOHtQ31iaQ2Up/Fz9nkY81GyJEsDVPaPo1SGYkkoLb6w63GzH1dNryw9RWmmhT3Qbru0jdZWEE4hPhuih6gSVdS/rHU2Lkg7xVubraeLgc5N1e+2m8vHxYdKkSUyaNImnnnqKu+++m6effpq77rrrkvvXrpKt1TiyWq22+3vvvZcHH3zwoufFxMhAVjoOgG7XQMr3sOQxuOtHdQC4s/rlX1CQAUEdYcTDzXpoo9HAn6ckcft7W/h0ywlmj4wjOsR5C7+mZRWxcJs6uP/Jqd0cuz6YEBqDASY8BfOvhp0fqcs4hbrmckTSstTKDAYDfl4eutya4wTcvXv3yx6Q3b9/fw4cOEDXrl0vujncQrd6mfwCePjCiV9g7xd6R3P5zp+oKbQ5+a/g1fyJzIiuYYyKD6PKovDPZWnNfvzW9MKSFKwKTOkZycBOrrFAqXATnUZA14lgNcPSP6vjFF2QJEviks6dO8f48eP55JNP2Lt3L8eOHePLL7/k5Zdf5rrrrrusYz7++ONs2rSJ3//+9+zevZv09HQWL17MAw880MzRO7E2MTDmj+r2sr9AWb6u4Vy2ZU+oTfOdRqlLJLSQx69KAtTp9puOnGux12lJ6w7lsPZQDp4mg+39COFUJr8IJi9IXwYHv9U7mhYhyZK4pICAAIYMGcJrr73G6NGj6dmzJ08++SRz5szhjTfeuKxj9u7dm7Vr15Kens6oUaPo168fTz75pG1sk6g27H4I7Qol2bD6Bb2jabojq9WuRIMJprzUol2JPTsEc3t1LaL/+2Yf5VUOtgagHWaLlReWpAAwa2gnOoX56xyREJehXQKMfFTd/ulxKC/QN54WILPhmkFTZsOJ1uW0P/8jq+Hj68FghHvWQlRvu09xCJYqeHsk5KTC4Hvh6pYf9FlYXsWkV9dytrCC343twp+cqHXmzTWHeXlpGkE+Hqz70zja+El3tHBSVeXw9gg4dxgG3Q1T/6l3RI0is+GEcGZdxkGPG0Cxwo9/AKuTLO2x9V01UfILhXFzW+Ulg3w8ef66noC6yO7BM4Wt8rpXKi2riHnL0wF4clp3SZSEc/P0gWnV9Za2vQ8Z2/SNp5lJsiSEo5r8AngFqNWvd3+qdzT2nTsCq/+mbk94CnzbttpLJ/eI5OpekVisCn/+ei9mi2Mnl1UWK3/4cjeVFisTksJlsVzhGuJGq3XiUOCHh9WWZhchyZIQjiqoPYz9s7q9/EkoOK1vPA2xVMGiu6GyGGKGQ79ZrR7CM9f2IMjHg72nCpi/8Xirv35TvLn6CPtPFxLs68mL03tJqQDhOpL/Cr4hcHY/bH5T72iajSRLQjiyIfdBVB8oOw+LZoPFQddCW/0CnNkJPsEw/b9gbHpNrysVHujDE1O7AfDKsjROnitt9Rga48CZAl5fpXa/PXddD8KDnGgsnRD2+IeqCRPA6hchN13feJqJJEtCODKTJ9z0P/AKhJObarq5HMmxdTVrQ13zb2gTrVsoMwZGM6xzKOVVVh77cg9VDtYdV2m28ocv9mC2KlzVI1IqdQvX1Pc2iBsD5jJYeAdUFOsd0RWTZEkIRxfaBa79t7q94VU4vELfeGorzYOv7wUUteutx/W6hmMwGHhxei8CvD3YejyPv/7gWMvG/HtlOqlZRYT4e/HXG3pK95twTQYDTH8XAiLVCR+LH3D6YpWSLAnhDHpOh4HqAsV8fQ8UntE3HlBPfosfgKIzal2oKS/pHREAncL8eW1mXwA+3HSCL6qXEdHbT/sy+c8adR27v17f0yUW/xWiXoERMONDMHrAga9hs3MvDi7JkhDOYvILENkLSs+pg6n1Hr+07T1I/QGMnnDj++DlOAUVJ3WP4JGJCQD85dv97Dx5Xtd4dp08z8MLd6MocOewWK7uJYVYhRuIGaqet0CdpHJio77xXAFJloRwFp4+cPOHajmBE7/Aymf0i+XAN/DTn9TtCU9B+776xVKPB8Z3Jbl7BJUWK/d9vIPswnJd4sjIK+XuD7dTYbYyPimcp6Z11yUOIXQx+B7oeZO6dtyXd0FRlt4RXRZJloRu5s+fT5s2bfQOw7nUHr+08XVY8/fWjyF9OSyaoxbMHHAXDHfMtf2MRgOvzuxLfHgA2UUV3PvJDirMrbscSkFpFXf9byvnSirpHhXE67f2w8Mkp13hRgwG9ZwV3h2Kz8LCWVBRpHdUTSb/taJBWVlZPPDAA3Tu3Pn/27v3uKjq/PHjr+E23EFBLuaooLKoaApIiZqWt7y1lvfw0mb9MlAXMUuzDFuBYjc0TfHyUPCyrrmZbmm6khomZCLeSPjiHbyjyUVFQeD8/ph1NlYZSYGDzvv5eMxD58znfM57zkNn3vM5n/P+oNVq0el0DBo0iB07djxy3yNGjODYsWM1EKWJ8RsCff5zV9wPMfBDHc4VOpOiv7ul4o4+jgFxtbr226Oy11qwbGwgjtYWHMwtIOzvB7hVWjcJU2lZBRPWpHPyyk08HK1Z8Von7LQWdXJsIeoVKzsYsQa0Tvoiu6sG68uhPEYkWRJVOnPmDAEBAezcuZPY2FgyMjLYtm0bzz//PGFhYY/cv42NDW5ubjUQqQkKnvjfWiY/RENy7a/BxvkDsHYElN0Gnxfh5SWq1FP6vZq72rEwxB+thRnfZ+UxZvnPFBbXbmXhW6XlTP7HQX469St2VuaseK0THk5ST0mYMJcWMHaTvrL/+f2QOAhuXFE7qmqTZElUKTQ0FI1Gw759+xg6dCg+Pj60bduWiIgI9u7dC0Bubi5//OMfsbe3x9HRkeHDh3P58mVDH4cPH+b555/HwcEBR0dHAgIC2L9/P3DvZbjIyEg6dOjA6tWrad68OU5OTowcOZLr1/87ZKsoCrGxsXh7e2NjY8PTTz/NV199VTcnpL4JngS9P9b/fVcU7P5r7R3rfDqsGQKl16F5NxiWqK8B9Zjo1qoRq8c/g4O1Bftz8hm+5CcuFdbOHKYLBbcYujiVbUcvYWmuYWGIP20aV71ApxAm4yl/eO07sHODyxmQ0K9+r0zwG5Is1TVFgdKb6jx+R52La9eusW3bNsLCwrCzu/cuJ2dnZxRFYfDgwVy7do3k5GSSkpI4efIkI0aMMLQLCQmhSZMmpKWlkZ6ezvTp07G0rPpL9uTJk2zatInNmzezefNmkpOT+eST/87L+eCDD0hISCA+Pp6jR48yZcoURo8eTXJycrXf2xOly5+hV6T+7zvn6BfdLb1Zc/1XVEDK57C8L9y6Bk8FwKh/gKVNzR2jjgR5NeSfEzrj5qAl+/J1hsSnciKvZovlpefk89IXKRy9UERDOyv+/saz9PiDjJ4KYeDeBl7fBo5N4NfjkPCifl3Jek4uoNe1O8UQrVLV3vcvVPv27hMnTqAoCr6+vlW2+f777zly5AinT59Gp9NXbV69ejVt27YlLS2NTp06kZuby7Rp0wz9tGrVyuhxKyoqSExMxMHBAYAxY8awY8cOoqKiuHnzJnFxcezcuZPOnTsD4O3tzZ49e1iyZAndu3ev1nt74nSdok+Ed8zW385/4nv440Jo3vXR+r1+GTa+Bad26Z+3fgleWgBah0ePWSW+Ho5seDuYcSv2cerqTYYtTuXDgW0Y3OEpzMwebe7VV+nneP/rDErLK/D1cGDZ2EB0DW1rKHIhniAuLfQJ06qX4NopWNxNvw7ms2/X2xHrx2ZkKSoqiuDgYGxtbat9B5WiKERGRtK4cWNsbGzo0aMHR48erdSmpKSESZMm4erqip2dHS+99BLnzp2rhXfweFH+MwplrMJwVlYWOp3OkCgBtGnTBmdnZ7KysgCIiIjgjTfeoFevXnzyySecPGn8F0Tz5s0NiRKAp6cneXl5AGRmZnL79m169+6Nvb294bFq1aoH9vvE6xYBo7/W/1rLPwOJA+C7dx9ulElR4Nh2iA/WJ0oWNjDocxi+CmycazryOqdraMs/J3TmaZ0z+cV3iFh/mMGLUkg7c+2h+juYm88bK9N455+HKS2voG9bdza8HSyJkhDGOOvgT9ugaWe4c1Nfh2lxN/1NJPXQYzOyVFpayrBhw+jcuTPLly+v1j6xsbHExcWRmJiIj48Pc+bMoXfv3mRnZxu+kMPDw/n2229Zt24dLi4uTJ06lYEDB5Keno65eS1MXrW01Y/wqMGy+h/erVq1QqPRkJWVxeDBg+/bRlGU+yZTv90eGRnJq6++ypYtW9i6dSsfffQR69at4+WXX75/iP9ziU6j0VBRoV/f6+6fW7Zs4amnnqrUTquVasi07AmhqbD9AziwCvYtgWPbwH+sflSokY/x/W9ehSPr4dDf9SuGA7j7wdAV0OgPtR9/HXKx1/Ll/3uWhJQzLNx1giPnChm2+CcGtPNkah8fvBvZG91fURR+OvUrC3edIOXEr4D+psBJz7ckvJfPI49SCWESHNz1c5gO/0OfLF3JgsT+8PQoCHoTPDvUm5tIHptkafbs2YB+UnB1KIrCvHnzmDlzJq+88goAK1euxN3dnbVr1/LWW29RWFjI8uXLWb16Nb169QJgzZo16HQ6vv/+e/r27Vvzb0SjqVeVjqvSsGFD+vbty8KFC5k8efI985YKCgpo06YNubm5nD171jC6lJmZSWFhIa1btza09fHxwcfHhylTpjBq1CgSEhKqTJaMadOmDVqtltzcXNO95PYg1k76S2VtBsM3k6EgB3b+Rf9w/QO0HqT/JVdRBuUlUFaivzR8PAmO/VtfEgDAXAudxkPPj/TFMJ9A1pbmvN2jBUMDmhCXdIwv03LZknGRLRkX8XSypmNTZzrqGtChqTNW5mbkXism91ox5/KL+eV8ERnnCwGwMNMwuONTvN2jBS0ekGQJIf6HmRl0DIE/9IMdH0N6oj55OvwP/edZ8276RXm9u4Orj2qlSh6bZOn3On36NJcuXaJPnz6GbVqtlu7du5Oamspbb71Feno6d+7cqdSmcePG+Pn5kZqaWmWyVFJSQklJieF5UVFR7b0RFS1atIjg4GCCgoL4+OOPad++PWVlZSQlJREfH09mZibt27cnJCSEefPmUVZWRmhoKN27dycwMJBbt24xbdo0hg4dipeXF+fOnSMtLY0hQ4Y8VDwODg688847TJkyhYqKCrp27UpRURGpqanY29szbty4Gj4Dj7GWPSFsL/yyAbI2w6kf4Go2/JhtfL/GHaFDCLQbqr/F1wQ0ctAS80o7xgU3I+a7/+PH41e4WHibixmX+C6j6mrDVhZmjAjU8VZ3b5o0kEtuQjwS24YwaJ5+Qe49cXB6N9wu1C+p9H+b9W16zNDPbVLBE5ssXbqk/5Bzd3evtN3d3Z2cnBxDGysrKxo0aHBPm7v7309MTIxhpOtJ5uXlxYEDB4iKimLq1KlcvHiRRo0aERAQQHx8PBqNhk2bNjFp0iSee+45zMzMePHFF1mwYAEA5ubm/Prrr4wdO5bLly/j6urKK6+88kjn7i9/+Qtubm7ExMRw6tQpnJ2d8ff35/3336+pt/3k0DroK2wHvKb/0Dm2HbK+0d95YmEFFtZgbgUWWv1ltqdf1d+pYqJ8PRxZ+XoQN0vKyDhfyMHcAg7m5nP4XAEATRvaomtoi66BLU0b2tLNxxU3hydz1E0I1TQJgJF/1699efGQ/ofe6WTI/RmadFItLI2i/I77yWtYZGTkA78409LSCAwMNDxPTEwkPDycgoICo/ulpqbSpUsXLly4gKfnfxetfPPNNzl79izbtm1j7dq1/OlPf6o0SgTQu3dvWrRoweLFi+/b9/1GlnQ6HYWFhTg6Vq6ncvv2bU6fPo2XlxfW1vLBWtfk/AshxBPgzi0ws6jxu+WKiopwcnK67/f3b6k6sjRx4kRGjhxptE3z5s0fqm8PDw9AP3r022QpLy/PMNrk4eFBaWkp+fn5lUaX8vLyCA4OrrJvrVYrE4qFEEKIuqJybTdVkyVXV1dcXV1rpW8vLy88PDxISkqiY8eOgP6OuuTkZD79VL+WVkBAAJaWliQlJTF8+HAALl68yC+//EJsbB0sHyGEEEKIeu+xmbOUm5vLtWvXyM3Npby8nEOHDgHQsmVL7O31d6D4+voSExPDyy+/jEajITw8nOjoaFq1akWrVq2Ijo7G1taWV199FQAnJyfGjx/P1KlTcXFxoWHDhrzzzju0a9fOcHecEEIIIUzbY5MszZo1i5UrVxqe3x0t2rVrFz169AAgOzubwsJCQ5t3332XW7duERoaSn5+Ps888wzbt2+vVPRw7ty5WFhYMHz4cG7dukXPnj1JTEysnRpLQgghhHjsqDrB+0lhbIKYTDBWl5x/IYQQVanuBO/HZrmTx53kpOqQ8y6EEOJRSbJUy+4u31FcXKxyJKaptLQUQC6rCiGEeGiPzZylx5W5uTnOzs6GxWBtbW2NLk4rak5FRQVXrlzB1tYWCwv5py6EEOLhyDdIHbhb8+luwiTqjpmZGU2bNpUEVQghxEOTZKkOaDQaPD09cXNz486dO2qHY1KsrKwwM5OrzUIIIR6eJEt1yNzcXObOCCGEEI8Z+ckthBBCCGGEJEtCCCGEEEZIsiSEEEIIYYTMWaoBdwsfFhUVqRyJEEIIIarr7vf2gwoYS7JUA65fvw6ATqdTORIhhBBC/F7Xr1/HycmpytdlbbgaUFFRwYULF3BwcKjRej5FRUXodDrOnj1rdM0aUTvk/KtLzr+65PyrS85/3VAUhevXr9O4cWOjZWZkZKkGmJmZ0aRJk1rr39HRUf6zqEjOv7rk/KtLzr+65PzXPmMjSnfJBG8hhBBCCCMkWRJCCCGEMEKSpXpMq9Xy0UcfodVq1Q7FJMn5V5ecf3XJ+VeXnP/6RSZ4CyGEEEIYISNLQgghhBBGSLIkhBBCCGGEJEtCCCGEEEZIsiSEEEIIYYQkS/XYokWL8PLywtramoCAAH788Ue1QzIJMTExdOrUCQcHB9zc3Bg8eDDZ2dlqh2WyYmJi0Gg0hIeHqx2KyTh//jyjR4/GxcUFW1tbOnToQHp6utphmYSysjI++OADvLy8sLGxwdvbm48//piKigq1QzNpkizVU19++SXh4eHMnDmTgwcP0q1bN/r160dubq7aoT3xkpOTCQsLY+/evSQlJVFWVkafPn24efOm2qGZnLS0NJYuXUr79u3VDsVk5Ofn06VLFywtLdm6dSuZmZl89tlnODs7qx2aSfj0009ZvHgxX3zxBVlZWcTGxvLXv/6VBQsWqB2aSZPSAfXUM888g7+/P/Hx8YZtrVu3ZvDgwcTExKgYmem5cuUKbm5uJCcn89xzz6kdjsm4ceMG/v7+LFq0iDlz5tChQwfmzZundlhPvOnTp5OSkiIj2SoZOHAg7u7uLF++3LBtyJAh2Nrasnr1ahUjM20yslQPlZaWkp6eTp8+fSpt79OnD6mpqSpFZboKCwsBaNiwocqRmJawsDAGDBhAr1691A7FpHzzzTcEBgYybNgw3Nzc6NixI8uWLVM7LJPRtWtXduzYwbFjxwA4fPgwe/bsoX///ipHZtpkId166OrVq5SXl+Pu7l5pu7u7O5cuXVIpKtOkKAoRERF07doVPz8/tcMxGevWrePAgQOkpaWpHYrJOXXqFPHx8URERPD++++zb98+Jk+ejFarZezYsWqH98R77733KCwsxNfXF3Nzc8rLy4mKimLUqFFqh2bSJFmqxzQaTaXniqLcs03UrokTJ3LkyBH27Nmjdigm4+zZs/z5z39m+/btWFtbqx2OyamoqCAwMJDo6GgAOnbsyNGjR4mPj5dkqQ58+eWXrFmzhrVr19K2bVsOHTpEeHg4jRs3Zty4cWqHZ7IkWaqHXF1dMTc3v2cUKS8v757RJlF7Jk2axDfffMPu3btp0qSJ2uGYjPT0dPLy8ggICDBsKy8vZ/fu3XzxxReUlJRgbm6uYoRPNk9PT9q0aVNpW+vWrdmwYYNKEZmWadOmMX36dEaOHAlAu3btyMnJISYmRpIlFcmcpXrIysqKgIAAkpKSKm1PSkoiODhYpahMh6IoTJw4ka+//pqdO3fi5eWldkgmpWfPnmRkZHDo0CHDIzAwkJCQEA4dOiSJUi3r0qXLPaUyjh07RrNmzVSKyLQUFxdjZlb5q9nc3FxKB6hMRpbqqYiICMaMGUNgYCCdO3dm6dKl5ObmMmHCBLVDe+KFhYWxdu1a/vWvf+Hg4GAY4XNycsLGxkbl6J58Dg4O98wPs7Ozw8XFReaN1YEpU6YQHBxMdHQ0w4cPZ9++fSxdupSlS5eqHZpJGDRoEFFRUTRt2pS2bdty8OBB4uLieP3119UOzaRJ6YB6bNGiRcTGxnLx4kX8/PyYO3eu3LpeB6qaF5aQkMBrr71Wt8EIAHr06CGlA+rQ5s2bmTFjBsePH8fLy4uIiAjefPNNtcMyCdevX+fDDz9k48aN5OXl0bhxY0aNGsWsWbOwsrJSOzyTJcmSEEIIIYQRMmdJCCGEEMIISZaEEEIIIYyQZEkIIYQQwghJloQQQgghjJBkSQghhBDCCEmWhBBCCCGMkGRJCCGEEMIISZaEECbthx9+QKPRUFBQ8FD7N2/e/HcVy0xMTMTZ2fmhjvVbGo2GTZs2PXI/QogHk2RJCGHSgoODuXjxIk5OTmqHIoSop2RtOCGESbOyssLDw0PtMIQQ9ZiMLAkh6oUrV67g4eFBdHS0YdvPP/+MlZUV27dvr3K/9957Dx8fH2xtbfH29ubDDz/kzp07ACiKQq9evXjxxRe5u7JTQUEBTZs2ZebMmcC9l+FycnIYNGgQDRo0wM7OjrZt2/Ldd99V+33ExcXRrl077Ozs0Ol0hIaGcuPGjXvabdq0CR8fH6ytrenduzdnz56t9Pq3335LQEAA1tbWeHt7M3v2bMrKyqodhxCi5kiyJISoFxo1asSKFSuIjIxk//793Lhxg9GjRxMaGkqfPn2q3M/BwYHExEQyMzP5/PPPWbZsGXPnzgX083pWrlzJvn37mD9/PgATJkzA3d2dyMjI+/YXFhZGSUkJu3fvJiMjg08//RR7e/tqvw8zMzPmz5/PL7/8wsqVK9m5cyfvvvtupTbFxcVERUWxcuVKUlJSKCoqYuTIkYbX//3vfzN69GgmT55MZmYmS5YsITExkaioqGrHIYSoQYoQQtQjoaGhio+PjxISEqL4+fkpt27d+l37x8bGKgEBAZW2rV+/XtFqtcqMGTMUW1tbJTs72/Darl27FEDJz89XFEVR2rVrp0RGRlb7eM2aNVPmzp1b5evr169XXFxcDM8TEhIUQNm7d69hW1ZWlgIoP//8s6IoitKtWzclOjq6Uj+rV69WPD09Dc8BZePGjdWOUwjx8GTOkhCiXvnb3/6Gn58f69evZ//+/VhbWxtt/9VXXzFv3jxOnDjBjRs3KCsrw9HRsVKbYcOGsXHjRmJiYoiPj8fHx6fK/iZPnszbb7/N9u3b6dWrF0OGDKF9+/bVjn/Xrl1ER0eTmZlJUVERZWVl3L59m5s3b2JnZweAhYUFgYGBhn18fX1xdnYmKyuLoKAg0tPTSUtLqzSSVF5ezu3btykuLsbW1rba8QghHp1chhNC1CunTp3iwoULVFRUkJOTY7Tt3r17GTlyJP369WPz5s0cPHiQmTNnUlpaWqldcXEx6enpmJubc/z4caN9vvHGG5w6dYoxY8aQkZFBYGAgCxYsqFbsOTk59O/fHz8/PzZs2EB6ejoLFy4EMMyjukuj0dyz/91tFRUVzJ49m0OHDhkeGRkZHD9+/IHJoxCi5snIkhCi3igtLSUkJIQRI0bg6+vL+PHjycjIwN3d/b7tU1JSaNasmWGyNnDfBGvq1KmYmZmxdetW+vfvz4ABA3jhhReqjEOn0zFhwgQmTJjAjBkzWLZsGZMmTXpg/Pv376esrIzPPvsMMzP9b9H169ff066srIz9+/cTFBQEQHZ2NgUFBfj6+gLg7+9PdnY2LVu2fOAxhRC1T5IlIUS9MXPmTAoLC5k/fz729vZs3bqV8ePHs3nz5vu2b9myJbm5uaxbt45OnTqxZcsWNm7cWKnNli1bWLFiBT/99BP+/v5Mnz6dcePGceTIERo0aHBPn+Hh4fTr1w8fHx/y8/PZuXMnrVu3rlb8LVq0oKysjAULFjBo0CBSUlJYvHjxPe0sLS2ZNGkS8+fPx9LSkokTJ/Lss88akqdZs2YxcOBAdDodw4YNw8zMjCNHjpCRkcGcOXOqFYsQogapPWlKCCEURT/R2sLCQvnxxx8N23JychQnJydl0aJFVe43bdo0xcXFRbG3t1dGjBihzJ07V3FyclIURVHy8vIUd3f3SpOl79y5owQFBSnDhw83HJffTPCeOHGi0qJFC0Wr1SqNGjVSxowZo1y9erXK4//vBO+4uDjF09NTsbGxUfr27ausWrWqUv8JCQmKk5OTsmHDBsXb21uxsrJSXnjhBeXMmTOV+t22bZsSHBys2NjYKI6OjkpQUJCydOlSw+vIBG8h6oxGUf5TfEQIIYQQQtxDJngLIYQQQhghyZIQQgghhBGSLAkhhBBCGCHJkhBCCCGEEZIsCSGEEEIYIcmSEEIIIYQRkiwJIYQQQhghyZIQQgghhBGSLAkhhBBCGCHJkhBCCCGEEZIsCSGEEEIYIcmSEEIIIYQR/x+bsd3U+AIlwQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y_sin = np.sin(x)\n", + "y_cos = np.cos(x)\n", + "\n", + "# Plot the points using matplotlib\n", + "plt.plot(x, y_sin)\n", + "plt.plot(x, y_cos)\n", + "plt.xlabel('x axis label')\n", + "plt.ylabel('y axis label')\n", + "plt.title('Sine and Cosine')\n", + "plt.legend(['Sine', 'Cosine'])" + ] + }, + { + "cell_type": "markdown", + "id": "8a1be4c8-3eb0-418a-811d-711a8c307e6d", + "metadata": { + "id": "R5IeAY03L9ja" + }, + "source": [ + "### `subplots` 绘图\n", + "\n", + "你可以使用subplot函数在同一个图中绘制一组不同的图形。下面是一个例子。" + ] + }, + { + "cell_type": "code", + "execution_count": 94, + "id": "608674d3-388e-42ec-aa89-cdb89f442b46", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 281 + }, + "id": "dM23yGH9L9ja", + "outputId": "14dfa5ea-f453-4da5-a2ee-fea0de8f72d9", + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGxCAYAAACqUFbqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB8SklEQVR4nO3dd1yV9f/G8ddhHfZBQJaCggNRVBS3aZqFuXJV2jBblpqZ2jBHWZaaZmbDkWVZZmU5K82ycuZGEAfiFmQIyN5wzv37A+P7M82RHO4z3s/H4/zh7X04F2SH69z3Z2gURVEQQgghhDATNmoHEEIIIYS4FVJehBBCCGFWpLwIIYQQwqxIeRFCCCGEWZHyIoQQQgizIuVFCCGEEGZFyosQQgghzIqUFyGEEEKYFSkvQgghhDArUl6EEDVu7969DBw4kKCgILRaLb6+vnTs2JEXX3yx6pxu3brRrVs39UIKIUyWRrYHEELUpA0bNnDffffRrVs3RowYgb+/P6mpqRw4cIDvvvuOCxcuAHDs2DEAmjZtqmZcIYQJkvIihKhRd955J8nJyRw/fhw7O7sr/s5gMGBjIxeEhRDXJ+8SQogadenSJby9va8qLsAVxeWft43OnTuHRqNh7ty5zJs3j+DgYFxdXenYsSN79uy56msdOHCA++67D09PTxwdHWnVqhXff/+9Ub4nIUTNkvIihKhRHTt2ZO/evYwdO5a9e/dSXl5+S89fsGABmzdvZv78+axYsYLCwkJ69+5Nbm5u1Tlbtmyhc+fO5OTksHjxYtavX09ERARDhgxh2bJl1fwdCSFqmtw2EkLUqEuXLjFgwAB27twJgL29PW3btqVfv36MGTMGV1dXgKqrLlu3bgUqr7wEBwfTvHlzYmJisLW1BWD//v20a9eOb7/9lqFDhwIQFhaGk5MT+/btu+IKT79+/YiOjubChQtye0oIMyb/9wohapSXlxc7duxg//79vPPOO/Tv358TJ04wadIkmjdvTmZm5nWf36dPn6riAtCiRQsAzp8/D8CpU6c4fvw4jzzyCAAVFRVVj969e5OamkpCQoKRvjshRE2Q8iKEUEWbNm2YOHEiP/zwAykpKYwfP55z584xZ86c6z7Py8vrij9rtVoAiouLAbh48SIAL730Evb29lc8Ro8eDXDDgiSEMG1Xj5gTQogaZm9vz7Rp03j//fc5cuTIbX0tb29vACZNmsSgQYOueU5oaOhtvYYQQl1SXoQQNSo1NRV/f/+rjsfHxwMQEBBwW18/NDSURo0acejQIWbOnHlbX0sIYZqkvAghalTPnj2pW7cu/fr1o0mTJhgMBmJjY3nvvfdwdXXlhRdeuO3X+OSTT+jVqxc9e/bk8ccfp06dOmRlZREfH8/Bgwf54YcfquE7EUKoRcqLEKJGTZ06lfXr1/P++++TmppKaWkp/v7+3H333UyaNImwsLDbfo3u3buzb98+ZsyYwbhx48jOzsbLy4umTZvy4IMPVsN3IYRQk0yVFkIIIYRZkdlGQgghhDArUl6EEEIIYVakvAghhBDCrEh5EUIIIYRZkfIihBBCCLMi5UUIIYQQZsXi1nkxGAykpKTg5uaGRqNRO44QQgghboKiKOTn5xMQEHDDXd8trrykpKQQGBiodgwhhBBC/AdJSUnUrVv3uudYXHlxc3MDKr95d3d3ldMIIYQQ4mbk5eURGBhY9Xv8eiyuvPx9q8jd3V3KixBCCGFmbmbIh1EH7G7fvp1+/foREBCARqNh3bp1N3zOtm3biIyMxNHRkZCQEBYvXmzMiEIIIYQwM0YtL4WFhbRs2ZKPP/74ps4/e/YsvXv3pkuXLsTExDB58mTGjh3L6tWrjRlTCCGEEGbEqLeNevXqRa9evW76/MWLFxMUFMT8+fMBCAsL48CBA8ydO5fBgwcbKaUQQgghzIlJjXnZvXs3UVFRVxzr2bMnS5cupby8HHt7+6ueU1paSmlpadWf8/LyjJ5T/I+iKKTklnA0OZdjqXlk5JdSWmGgpFxPaYWB0goDtZztqVvLibq1nKlby4nAWs7U83KWqexCCFXkFpdzKj2fkxcLOJleQHp+KQZFQVEUFAUMioLOyZ6Q2q6EeLsQUtuVel7O2NvK0mimwqTKS1paGr6+vlcc8/X1paKigszMTPz9/a96zqxZs3jzzTdrKqIAEi8V8VNcCrtOZ3I0JY+covJb/ho+blrubFybbqE+3NHQG53z1cVUCCGqQ0m5nh0nM/nlSCq7Tl0iLa/klr+GnY2GFnV13NPUj6hmvjSo7WqEpOJmmVR5gatHGSuKcs3jf5s0aRITJkyo+vPfU61E9UrPK+HnuFR+PJRCbFLOFX9nZ6OhoY8rzQJ01K3lhKO9LY72NmjtbLG31ZBVWMaF7GIuZBdxIbuY81lFpOeX8kP0BX6IvoCtjYbWQR481C6Ivi0CcLCTTzdCiNtTVmFg87GLbDySypbj6RSV6a/4e3+dIw19XGnk40adWk7YasDGRoMGQKPhUkEppzMKOZNRwNnMQorK9BxMzOFgYg6zNx0npLYLUU39GNI2kGBvF1W+R2tmUuXFz8+PtLS0K46lp6djZ2eHl5fXNZ+j1WrRarU1Ec8qnbiYzwe/n+SXI6kYKnskNhro1MCbe8P9aFnXg0a+rjja29701ywp13PgXDZbE9LZeiKDU+kF7D+Xzf5z2cz65TiPdajHw+2D8HKV/65CiFtTVFbBd/uS+HTHGVJz/3eFJUDnSM9wP6Ka+tGsjjvujjd/tVdRFC5kF7P1RAa/HU1jz5lLnMkoZPG203yy/TT3NvNj5J0NaBnoYYTvSFyLRvn70oaxX0ijYe3atQwYMOBfz5k4cSI//fQTx44dqzo2atQoYmNj2b179029Tl5eHjqdjtzcXFnn5TacuJjPB3+cZOPhVP7+F9IqyIP7WgbQp4U/Pm6O1fZaF7KLWBeTzFe7z5OeXzl+SWtnw6DWdRl/T6NqfS0hhGXKLS5n+e5zfP7XObIKy4DK29ODWtelV7gfLerqqm2cXV5JOdsSMlgbk8yfx9OrjncM8WJktwZ0beQtY/r+g1v5/W3U8lJQUMCpU6cAaNWqFfPmzaN79+54enoSFBTEpEmTSE5O5quvvgIqp0qHh4fz7LPPMmLECHbv3s3IkSP59ttvb3q2kZSX25OUVcTsTcfZ8P9KS69wP8b2aESYv3F/nmUVBjYeTmXpzrMcTs4FwFVrxws9GjG8U325nSSEuIrBoLBiXyJzNh0nv6QCgCBPZ0be2YBBrevc0lXh/yIhLZ8l28+wPjaZisuXp7s2rs1b/ZtRz0tuJ90KkykvW7dupXv37lcdHz58OMuWLePxxx/n3LlzbN26tervtm3bxvjx4zl69CgBAQFMnDiRkSNH3vRrSnn5bwwGheV7zjN70/Gqe8P3NvPjhbuNX1r+SVEU9p3NYubGeA5dqCwxDWq78MZ9zejSqHaNZhFCmK741Dwmrz1MTGIOAI18XHmue0P6tvDHroZnBqXkFLN051mW7z5Pmd6Ag50Nz3VryMhuIWjtjFugLIXJlBc1SHm5daczCpi4Ko4D57MBaFu/Fm/c14xmATpVcxkMCj9EJzFnUwKXLl8G7tPcnxkDw/FwdlA1mxBCPcVlej744ySf7ThDhUHBVWvHS1GNGdaxPrY26t6uOZtZyOvrj7DjZCYAId4uvDUgnM4NvVXNZQ6kvEh5uSl6g8In208z//eTlFUYcHGw5dVeTXikfT1sVH4D+P9yi8uZ//sJvtp9Hr1BIUDnyIcPtaJNfU+1owkhatiR5Fye++Yg5y8VAZVXiKfd1xR/nZPKyf5HURR+jkvlrZ+PVY3je7ZrCC/1DJW1Yq5DyouUlxvKKSpj7HexbD+RAcCdjWszY2A4dWs5q5zs3x2+kMvz3x7k3KUibG00jL+7EaO6NVT9k5YQomas3J/Ia+uPUlZhIEDnyPT+4dzd1PfGT1RJXkk5725KYPme8wBE1qvFxw+3MqmiZUqkvEh5ua5jKXk8+/UBkrKKcbS3YXr/cB6IrGsWo+MLSit4bd0R1sYkA5Wj++cPjcDXXWYkCWGpisv0vL7+CD9EXwDgriY+zHuwpdncPt50JJWXf4gjv7QCTxcH3h8SwZ2NZfzeP0l5kfLyr9bHJjNxdRwl5QYCPZ345NE2NA0wv5/T6ugLvLb+CEVlenzdtXz+eFvVx+gIIarfucxCRn4dzfG0fGw08GJUKKPubGBSt7ZvxvlLhYxecZCjKXloNPD8XY0Yf3cjs/jQWFOkvEh5uYrBoDBzYzyf7TwLVE7l+3BohNl8crmW0xkFjFwezcn0AlwcbFn4aKR8mhHCgsQkZvPksv1kF5Xj5eLARw+1opMZD3wtKdfz9oZjfL0nEYBBreswe3ALGQdzmZQXKS9XKKsw8OIPh/jpUAoAz3VvwIR7Qi1irEhucTkjl0ez+8wlbG00zBwYzpC2QWrHEkLcpi0J6Yz++iDF5Xpa1tXxybA2+Oks4/bw9/uTmLT2MHqDQtfGtVn0SGtctCa14L0qbuX3t9Q9C1dcpueZ5Qf46VAKdjYaPhgawcs9m1hEcQHQOdnz5ZPtGNSqDnqDwsTVh5n7awIW1smFsCqroy/w9JcHKC7X07Vxbb4Z0cFiigvAg20D+Wx4G5zsbdl+IoOhS/aQcXlWkrg5Ul4sWG5ROY8u3cvWhAwc7W34bHgb+kfUUTtWtXOws+G9B1sy9q6GAHy85RST1hzGYJACI4Q5URSFxdtO8+IPh9AbFAa1qsPS4W0s8qpE91AfvnumA14uDhxOzmXwol2cyyxUO5bZkPJiodLzShiyZDfR57Nxd7Tj66fa0y3UR+1YRqPRaJgQFcqcwS2w0cB3+5OkwAhhRhRFYc6vCbzzy3Ggcl2UuQ+0tOjxIC0DPVg9qhNBns4kZhXx4Ce7OSsF5qZY7r8KK5aRX8rQJXs4npZPbTctK5/taDULuj3YNpD3h0Rgo4GVB6TACGEu3v/9JIu2ngZgSu8wJvUOM7sZRf9FfW8XVo/qRBM/N9LzS3n40z0kXl6AT/w7KS8WJqeojGFL93Ims5A6Hk6sGtmxxvcmUlv/iDpSYIQwIwu2nOLDP04CMK1fU0Z0DVE5Uc2q7abl66fb08jHldTcEh76dA9JWVJgrkfKiwXJLyln+Of7qq64rHi6vdXuaioFRgjz8NmOM7z7awIAk3o14YnOwSonUoe3q5YVI9oTUtuF5JxiHvp0D8k5xWrHMllSXixEcZmep748wKELudRytmfF0+2p722dxeVv/ywwb204JrOQhDAhX+46x9sb4gGYcE9jnr2zgcqJ1OXj5si3IzpQ38uZC9nFPPzpHtJyS9SOZZKkvFiA0go9z34dzb6zWbhp7Vj+VHsa+7qpHcsk9I+ow3sPtgTgi7/OsWT7GZUTCSGgcjr0tB+PAjCme0PG9mikciLT4OvuyDcjOhDo6cT5S0UM/3wfeSXlascyOVJezJzBoPDi94fYfiIDJ3tblj3ZlvA6skz+/zewVV2m9A4DYNYvx1kbc0HlREJYt79OZTJxdRwAT98RzItRjVVOZFoCPJz45ukO+LhpSbiYz6ivoymrMKgdy6RIeTFz721O4Oe4VOxtNXz6WBsi61nHrKJbNaJrCE/dUXkv/eUf4thxMkPlREJYp4S0fEYuj6bCoNCvZQCTe4fJ/j7XEOjpzOePt8XFwZa/Tl3i1dVxctv7/5HyYsa+35/Egi2VUwvfGdSCOxqZ754fNWFK7zD6tQygwqAwcnk0R5Jz1Y4khFW5mFfCE1/sI7+0gnbBnsx9oIVVTIf+r8Lr6Fj4aCS2NhrWxCTz3m8n1I5kMqS8mKm/TmUyee1hAMb2aMTgyLoqJzJ9NjYa5j7Qgo4hXhSW6Xn8i/0yml+IGlJQWsETX+wnJbeEkNouLBkWidbOVu1YJu/OxrWZNbA5ULl6+Dd7E1VOZBqkvJihkxfzGfl15WXX/hEBjL9bBrrdLK2dLZ88FkkTPzcyC0p5dvkBisv0ascSwqJV6A08t+Igx1Lz8HZ14Msn2pn1jvY17cG2gVUDml9bf4TtJ+S2t5QXM5NZUMoTy/aTX1JB2/q1mD24hdwvvkXujvZ8NrwNni4OHEnO4xW5lyyEUb3zy3G2najcY23p8LYEejqrHcnsjL+7EYNb10VvUHj+2xjOX7LubQSkvJiR8sufXi5kF1Pfy5lPhrXB0V4uu/4XdWs5s+iR1tjZaPjpUAqLtp1WO5IQFml9bDKf7TwLwPsPRtAy0EPdQGZKo9Ewc1A4rYI8yC0u55mvoiksrVA7lmqkvJiRd345zt6zWbhq7fhseFs8XeSy6+1oH+LFG/c1A+DdXxP48/hFlRMJYVmOJOfyyqrKKdHPdW9Ar+b+Kicyb1o7WxY/Gknty1OoX/z+kNVeNZbyYibWxyaz9PKnl7kPtKShj6vKiSzDox3q8Uj7IBQFXvg2llPp+WpHEsIiZBWW8ezyaEorDHQLrc2Ee0LVjmQRfN0dWfxoJA62Nmw6msaCLafUjqQKKS9mID41r2pBp+e6N+DecD+VE1mWaf2a0S7Yk/zSCkZ8FU2+rGYpxG2p0BsY881BknOKqeflzAdDWmErU6KrTWS9WkzvX3nV+L3NJ/gj3vquGkt5MXG5ReWM/DqaknIDXRp5y6cXI3Cws2HRI60J0DlyNrOQV1cfttpLsUJUh9mbjrPr9CWcHWxZMqwNOmd7tSNZnKHtghjWoR6KAuO+i+VcpnUN4JXyYsIMBoVxK2M4f6mIurWc+HCofHoxFi9XLR9fHsC74XAqy/ecVzuSEGZp05E0Pt1ReYv7vQdaEuon+6wZy2t9m9K2fi3ySyt47puDlJRbz7IPUl5M2KJtp9mSkIHWzobFj0ZSSwboGlXroFpMurwH0ts/xxN3IUfdQEKYmaSsIl5ZdQiAZ7qGyABdI3Ows+HDh1rh6eLA0ZQ8Zm6MVztSjZHyYqKiz2cxb3PlUtDT+zeTzRZryJOd69OzmS9legOjVxwkt0jGvwhxM8oqDDz/bQx5JRW0CvLg5Z5yi7sm+OucmPdgSwC+2n2ejYdTVU5UM2qkvCxcuJDg4GAcHR2JjIxkx44d/3ru1q1b0Wg0Vz2OHz9eE1FNQk5RGWO/jUV/eQXdB9sEqh3Jamg0Gubc35JATycuZBfz0irrnYooxK1499fjxCbl4O5ox4dDW2FvK5+Na0q3UB9GdWsAwMRVcVaxgJ3R/3WtXLmScePGMWXKFGJiYujSpQu9evUiMfH6+zMkJCSQmppa9WjUyDqWwFcUhVdWxVWN0n97QLisoFvDdE72LHy4ciri5mMXq6aoCyGu7Y/4i1XjXN59oKWsoKuCF+9pTJt6/xv/Ulph2eNfjF5e5s2bx1NPPcXTTz9NWFgY8+fPJzAwkEWLFl33eT4+Pvj5+VU9bG2vvZJsaWkpeXl5VzzM2fI95/nt2EXsbTV8/FBr3BxllL4amtfV8Vq/pkDlzAnZgVqIa0vJKebFHyrHuTzeqT49m8lSDmqws7Xho4dbUcvZniPJeczcYNnjX4xaXsrKyoiOjiYqKuqK41FRUezateu6z23VqhX+/v706NGDLVu2/Ot5s2bNQqfTVT0CA833FsvRlFze/rnyH9ykXmE0ryvjXNT0aPsgejbzpVyvMPa7GNnAUYh/0BsUxn0XS05ROc3r6JjUu4nakaxa5fiXCAC+3H3eotd/MWp5yczMRK/X4+vre8VxX19f0tLSrvkcf39/lixZwurVq1mzZg2hoaH06NGD7du3X/P8SZMmkZubW/VISkqq9u+jJhSVVfD8NzGU6Q3cHebDE53rqx3J6mk0Gt4Z1AJfdy1nMgp5a8MxtSMJYVIWbzvNvnOVW5Z8/HArtHay15raujfx4ak7ggF4ZVUcGfmlKicyDruaeJF/jtlQFOVfx3GEhoYSGvq/UeodO3YkKSmJuXPn0rVr16vO12q1aLXa6g2sghkb4jmTWYifuyPv3t9SxrmYiFouDsx7MIJHPtvLN3sT6da4NlFyWVwI4i7k8P7lGZFv3NeMel4uKicSf3u5Zyh/ncrkeFo+r6w6xOePt7W43ylGvfLi7e2Nra3tVVdZ0tPTr7oacz0dOnTg5MmT1R3PZGw5ns6KvZUDmN97sKWs52JiOjf05pmuIQBMXB1Hel6JyomEUFdRWQXjvoulwqDQu7kfg1vXUTuS+H8c7W35YGgrHOxs2JKQwdd7rz9BxhwZtbw4ODgQGRnJ5s2brzi+efNmOnXqdNNfJyYmBn9/y1zs6FJBKS9f3nX1yc7BdG7orXIicS0vRjWmWYA72UXlvPjDIQwGmT4trNfMjZVXin3dtcwY0NziPtVbglA/N169t3IM0owNxziVXqByoupl9NlGEyZM4LPPPuPzzz8nPj6e8ePHk5iYyMiRI4HKMSuPPfZY1fnz589n3bp1nDx5kqNHjzJp0iRWr17NmDFjjB21ximKwqQ1h8ksKKWRjyuv3CuLOpkqrV3lJxlHext2nMzki13n1I4khCr+iL/I13sqP8nPfUCuFJuyxzvVp0sjb0rKDYxbGUNZhUHtSNXG6OVlyJAhzJ8/n+nTpxMREcH27dvZuHEj9erVAyA1NfWKNV/Kysp46aWXaNGiBV26dGHnzp1s2LCBQYMGGTtqjfsh+kLVtOj5QyNwtJfBbqasoY8rU/tUTp+es+m4xX2SEeJGMvJLeeXyleKn7gimS6PaKicS12Njo6ksmJenT/+9arsl0CgWtnxoXl4eOp2O3Nxc3N3d1Y7zrxIvFdHrg+0UlumZeG+TqtURhWlTFIXhX+xn+4kMWgZ6sHpkR+xkJVFhBRRF4Znl0Ww+dpFQXzfWj+ksH7jMxKYjaYz8OhobDfwwshOR9WqpHemabuX3t7zrqkBvUHjxh1gKy/S0q+9ZNRhUmD6NRsOcwS1wd7TjUFIOn2w/o3YkIWrEuthkNl++Uvz+ELlSbE7uDfdjUKs6GBR4+YdDFrFmlZQXFXzx11n2n8vGxcGW9x5sia2NDHYzJ346R97s3wyA+b+f4FiKea/qLMSNXMwrYdr6owC80KMRTQNM96q2uLZp/ZpVrlmVWci7vyaoHee2SXmpYWcyCqr+4UzuEyZ7gJipARF1iGpaufruhO9jLWognBD/n6IovLo6jrySClrU1THyTrnFbY50zva8M7gFAF/sOsveM5dUTnR7pLzUIL1B4eVVcZRWGLijoTcPtwtSO5L4jzQaDTMGNsfTxYHjafl8+IflrkMkrNsP0RfYkpCBg60N7z3QUsZ4mbHuoT4MaROIosDLq+IoLK1QO9J/Jv8Ka9AXf50l+nw2rlo73hksayOYu9puWt4eEA7Awq2niE3KUTeQENUsJaeYt36q3BZjQlRjGvm6qZxI3K6pfcOo4+FEYlYR7/xyXO04/5mUlxpy+v/dLprSJ4y6teR2kSXo3dyf/hEBGBR4ZdUhi9+GXlgPRVGYuDqO/NIKWgV5MKKLTCywBG6O9sy5v/L20fI95/nrVKbKif4bKS81QG9QePmHQ5RWGOjSyJuhbc1352txtTf6NcPb1YETFwtY8OcpteMIUS1W7k9ix8lMtHY2zH1AJhZYks4NvRnWoXKttYmrzfP2kZSXGvD5zrMcTMzBTWvH7MEt5HaRhanl4sD0/n/fPjrN0ZRclRMJcXvSckuYsSEeqNzkr0FtV5UTier2aq8m1PFw4kJ2sVnOPpLyYmTnMgt5b/P/bhcFeDipnEgYQ+/m/vQK96PCoPDKqjjK9TL7SJgnRVGYuu4w+aUVRAR68ETnYLUjCSNw0doxa1BzAL7cfY4D57JUTnRrpLwYkaIovLomjpJyA50bejFEbhdZtDf7N8PD2Z6jKXkskcXrhJn68VAKv8enY2+rYc79LeR2kQXr2rg2D0TWRVHgldVxlJSbz5g9KS9G9O2+JPacycLJ3pZZA+V2kaXzcXNkWr/KvY8++P0kJy/mq5xIiFtzqaCUNy/PLnr+rkY0ltlFFm9qn6bUdtNyJqOQD8xoyQcpL0aSmlvMrI2V94xf6hlKkJfMLrIGAyLqcFcTH8r0Bl5eFYfeYFFbhwkL9+ZPx8gqLKOJn5ssRmcldM72VUs+LNl+hiPJ5jFmT8qLESiKwtS1R6ruGT/eqb7akUQNqVy8Lhw3rR2xSTl8ueuc2pGEuCm/H7vIj4dSsNHAnPtb4GAnvx6sRc9mfvRp4V+1kKo5jNmTf51G8FNcKn8cl3vG1spf58SrvZsAMPe3BJKyilROJMT15ZWUM2XdYQBGdA2hRV0PdQOJGvfmfc2o5WxPfGoen2w7rXacG5LyUs2yCst448fKDczGdJd7xtbqobZBtKvvSVGZninrjqAocvtImK53fjnOxbxSgr1dGH93Y7XjCBV4u2qZ1q9yw9kP/zzFmYwClRNdn5SXavb2z5X3jEN93RjVTe4ZWysbGw2zBjfHwdaG7ScyWB+bonYkIa5p39ksvtmbCMCsQc1xtLdVOZFQS/+IALo2rk1ZhYFJaw5jMOExe1JeqtH2ExmsiUlGo4F3BjeXe8ZWrkFtV8b2aAjA9MulVghTUlqhZ9KaOACGtg2kQ4iXyomEmjQaDTMGhONkb8ves1l8fyBJ7Uj/Sn67VpOisoqqe8bDO9anVVAtlRMJU/BM1wY08XMjq7CMt38+pnYcIa6wYMtpTmcU4u2qZVKvMLXjCBMQ6OnMi1GVtw5nbownPa9E5UTXJuWlmnzw+0mSsooJ0DnyUs9QteMIE+FgZ8M7g1ug0cCamGS2nchQO5IQAJy4mM+irZV7cb15XzN0zvYqJxKm4onOwbSoqyOvpII3fjqqdpxrkvJSDY4k5/LZzrMAvDUgHFetncqJhCmJCPTgiU6VS6xPXnOYojLz2wRNWBaDQWHSmsOU6xXuDvOhd3M/tSMJE2Jro+GdQZUzZTceTmPzsYtqR7qKlJfbVKE38OqaysXI+rTwp0eYr9qRhAl6MaoxdTycSM4pZv7v5rOKpbBMK/aeJ/p8Ni4OtkzvHy6rf4urNA1w55muIQC8tu4I+SXlKie6kpSX27Rs1zmOJOfh7mhXtTS8EP/korWrWsVy6c6zZrOKpbA8abklzN5UuVnsK/c2kc1ixb96oUcj6ns5k5ZXwlwT23laysttSMoq4r3fTgAwuXcYPm6OKicSpqx7E5+qVSwnrz0sWwcIVUz78QgFpRW0CvLg0Q711I4jTJijvS0zBlbuPP3VnvPEJGarnOh/pLz8R4qi8Nr6IxSX62kX7Ck7RoubMq1fU9wc7Yi7kCtbB4ga9+vRNH49ehE7Gw2zBjWX1b/FDXVu6M2g1nVQFC6PkzKNrQOkvPxHP8elsjUhAwdbG2YNai73jMVN8XFzrJqSOve3BJJzilVOJKxFfkk509ZXzhwZ0TWEJn7uKicS5mJqn6bUcrbneFo+n+04q3YcQMrLf5JbVF61bfzo7g1oUNtV5UTCnAxtG0jb+rUoKtPzumwdIGrIe7+dIC2vhHpezrzQo5HacYQZ8XRxYGqfyjGdH/xxgvOXClVOJOXlP3ln03EyC0ppUNtFtgAQt8zm8iV7e1sNfxxP55cjaWpHEhYuNimHL3efA2DGANkCQNy6Qa3r0LmhFyXlBqaawIeuGikvCxcuJDg4GEdHRyIjI9mxY8d1z9+2bRuRkZE4OjoSEhLC4sWLayLmTdl/Lotv91XuAzJzYHO0dvImIG5dQx83RnWr3DrgjR+Pkmdi0xCF5SjXV+5ToygwsFUd7mjkrXYkYYY0Gg1vD6jc9mbHyUzV92szenlZuXIl48aNY8qUKcTExNClSxd69epFYmLiNc8/e/YsvXv3pkuXLsTExDB58mTGjh3L6tWrjR31hsoqDExeU7kFwJA2gbSXfUDEbRjdrQEh3i6k55cyZ9NxteMIC7V051niU/PwcLZnah/ZAkD8d8HeLlW3HN/6+RjZKu7XZvTyMm/ePJ566imefvppwsLCmD9/PoGBgSxatOia5y9evJigoCDmz59PWFgYTz/9NE8++SRz5841dtQb+mTbaU6mF+Dt6sCk3k3UjiPMnKO9LW8PrFz7ZcXeRKLPm840RGEZkrKKmP/7/5Zz8HLVqpxImLsRXUII9XWji8pX8IxaXsrKyoiOjiYqKuqK41FRUezateuaz9m9e/dV5/fs2ZMDBw5QXn71pfXS0lLy8vKueBjD2cxCPtpSuQ/Ia32b4uHsYJTXEdalUwNv7o+si6JUbh1gKtMQhflTFIUp645QUm6gQ4gnD0TWVTuSsAAOdjasHt2J+UNbUctFvd+DRi0vmZmZ6PV6fH2vXDLf19eXtLRrD1JMS0u75vkVFRVkZmZedf6sWbPQ6XRVj8BA46y34u3qwMPtgugWWpv7WgYY5TWEdZrSOwxPFwcSLuazZPsZteMIC/FTXCrbT1Qu5zBjoCznIKqPKezfVyMDdv/5P42iKNf9H+la51/rOMCkSZPIzc2teiQlJVVD4qu5Odrzxn3NWDq8rbwJiGpVy8WhaizCh3+cNIlpiMK85RaVM/3ybsDPdW8oyzkIi2PU8uLt7Y2tre1VV1nS09OvurryNz8/v2ueb2dnh5fX1QNktVot7u7uVzyMSVakFMYwsFXlNMTSCtOYhijM2zub4sksKKNBbRdGdgtRO44Q1c6o5cXBwYHIyEg2b958xfHNmzfTqVOnaz6nY8eOV53/22+/0aZNG+zt7Y2WVQg1/XMa4rrYZLUjCTO172wW3+6rvAI9a1ALWc5BWCSj3zaaMGECn332GZ9//jnx8fGMHz+exMRERo4cCVTe9nnssceqzh85ciTnz59nwoQJxMfH8/nnn7N06VJeeuklY0cVQlXB3i6Mvaty7Ze3fo5XdRqiME+lFXomr61czmFo20DaBXuqnEgI4zD6qJshQ4Zw6dIlpk+fTmpqKuHh4WzcuJF69Sp3M01NTb1izZfg4GA2btzI+PHjWbBgAQEBAXz44YcMHjzY2FGFUN0zXRvw46EUTlwsYMbGeOY+0FLtSMKMfLLtDKcuL+fwai9ZzkFYLo1iYTfX8/Ly0Ol05ObmGn38ixDGEH0+m/sX70JR4Jun29OpoayIKm7sdEYBvebvoExv4IOhEfSPqKN2JCFuya38/pa9jYQwMZH1avFo+8ork5PXHqakXK9yImHqDAaFSWsOU6Y3cGdjWc5BWD4pL0KYoJfvDcXXXcu5S0V8/OcpteMIE/dDdBL7zmbhZG/L2wPCZTkHYfGkvAhhgtwd7XnzvsqtAxZvO01CWr7KiYSpSs8vYcaGeAAm3NOYQE9nlRMJYXxSXoQwUfeG+3FPU18qDAqvronDYLCo4Wmimkz/6Rh5JRWE13Hnic711Y4jRI2Q8iKECZvevxkuDrbEJOawfM95teMIE/Pn8Yv8HJeKjQbeGdQCO1t5SxfWQf6lC2HC/HVOTLw85XXOpuOk5BSrnEiYisLSCqauPQLAU3cEE15Hp3IiIWqOlBchTNyj7evROsiDwjI9r8nWAeKyd39NICW3hLq1nBh/T2O14whRo6S8CGHibGw0zB7cAgdbG/44ns5PcalqRxIqiz6fzZe7zwEwY2BznB3U3+VXiJok5UUIM9DI143nulduHfDmj0dl6wArVlqh59XVcSgKDGpdhzsb11Y7khA1TsqLEGZiVLcGNPZ15VJhGW9fnhorrM/CLac5mV6Al4sDr/VpqnYcIVQh5UUIM+FgZ8OsQS3QaGD1wQtsP5GhdiRRwxLS8lm4tXLRwjf7N6OWi4PKiYRQh5QXIcxIZL1aDO9YH6jcOqCwtELdQKLG6A0KE1fHUa5XuDvMlz7N/dWOJIRqpLwIYWZe7hlKHQ8nLmQX8+6vCWrHETVk2a5zxCbl4Ka1ky0AhNWT8iKEmXHR2jFrUHMAvtx9jv3nslROJIwtKauIuZeL6qTeYfjpHFVOJIS6pLwIYYa6Nq7Ng23qoijwyqo4istk52lLZTAovLzqEMXletoHezK0baDakYRQnZQXIczUlD5N8XXXcjazkHmb5faRpVqx9zx7zlTuGD3n/hbY2MjtIiGkvAhhpnRO9lW3j5buPMvBxGyVE4nqlpRVxKxfjgMw8d5Q6nm5qJxICNMg5UUIM3ZXE18GtaqD4fLto5JyuX1kKQwGhVdWxVFUpqddsCePXZ5lJoSQ8iKE2Xu9X1O8XbWcSi/gwz9Oqh1HVJMV+xLZfeYSjvY2vCu3i4S4gpQXIcych7MDbw8IB2DxttPEyO0js5eUVcSsjZWrKE+8t4ncLhLiH6S8CGEB7g33o39EAAYFXvz+kMw+MmOGy4vRFZXpaVffs2pRQiHE/0h5EcJCTL8vHF93LWcyC5m96bjaccR/9OXuc+w6XXm7aLbcLhLimqS8CGEhdM72zLm/JVC5GutfpzJVTiRu1an0fN65PLtocu8wgr3ldpEQ1yLlRQgLcmfj2jzSPgiAl384RF5JucqJxM0q1xsYv/IQpRUGujTyZliHempHEsJkSXkRwsJM7h1GPS9nUnJLmP7TMbXjiJv00Z+nOJyci87Jnnfvbyl7FwlxHVJehLAwLlo73nugJRoNrIq+wG9H09SOJG4gJjGbBVtOAfD2gHDZu0iIG5DyIoQFalPfk2e7NgDg1TWHSc8rUTmR+DdFZRVM+P4QeoPCfS0D6NcyQO1IQpg8o5aX7Oxshg0bhk6nQ6fTMWzYMHJycq77nMcffxyNRnPFo0OHDsaMKYRFGn9PI5r6u5NVWMaLPxzCYFDUjiSuYebGeM5mFuLn7shb/cPVjiOEWTBqeXn44YeJjY1l06ZNbNq0idjYWIYNG3bD5917772kpqZWPTZu3GjMmEJYJK2dLR8+1ApHext2nMxk6c6zakcS//Dr0TS+3pMIwLsPtEDnbK9yIiHMg52xvnB8fDybNm1iz549tG/fHoBPP/2Ujh07kpCQQGho6L8+V6vV4ufnZ6xoQliNhj6uTOvXjElrDjPn1+N0bOBFeB2d2rEEkJJTzCur4gB4pmsIXRrVVjmREObDaFdedu/ejU6nqyouAB06dECn07Fr167rPnfr1q34+PjQuHFjRowYQXp6+r+eW1paSl5e3hUPIcT/DG0byL3N/CjXK4z9Noaisgq1I1k9vUFh3Hex5BaX07Kujpei/v3DnBDiakYrL2lpafj4+Fx13MfHh7S0f5/90KtXL1asWMGff/7Je++9x/79+7nrrrsoLS295vmzZs2qGlOj0+kIDAystu9BCEug0Wh4Z3Bz/HWOnMks5M0fZfq02j768yT7zmXhqrXjw4da4WAncyeEuBW3/H/MG2+8cdWA2n8+Dhw4AHDNdQoURbnu+gVDhgyhT58+hIeH069fP3755RdOnDjBhg0brnn+pEmTyM3NrXokJSXd6rckhMXzcHbg/SERaDSw8kASP8elqB3Jau09c6lq9++3B4TLpotC/Ae3POZlzJgxDB069Lrn1K9fn7i4OC5evHjV32VkZODr63vTr+fv70+9evU4efLkNf9eq9Wi1Wpv+usJYa06hHjxXLeGfLzlFK+uPkxTf3dCaruqHcuqZBeWMW5lLAYFBreuy4BWddSOJIRZuuXy4u3tjbe39w3P69ixI7m5uezbt4927doBsHfvXnJzc+nUqdNNv96lS5dISkrC39//VqMKIf5h3N2N2H8ui71nsxi94iBrR3fGycFW7VhWwWBQeOmHQ6TmlhDs7cL0/s3UjiSE2TLajdawsDDuvfdeRowYwZ49e9izZw8jRoygb9++V8w0atKkCWvXrgWgoKCAl156id27d3Pu3Dm2bt1Kv3798Pb2ZuDAgcaKKoTVsLO14aOHW1HbTcvxtHymrjuCosj6LzVhwZZT/HE8HQc7Gz56qBUuWqNN9hTC4hl1lNiKFSto3rw5UVFRREVF0aJFC5YvX37FOQkJCeTm5gJga2vL4cOH6d+/P40bN2b48OE0btyY3bt34+bmZsyoQlgNHzdHPnqoFTYaWH3wAt8fkHFixrb9RAbzfj8BVI5zkenqQtwejWJhH7vy8vLQ6XTk5ubi7u6udhwhTNbCraeYsykBBzsb1o7uRLMA+YVqDBeyi+j70U5yisp5qF0gswa1UDuSECbpVn5/y/w8IazUyK4N6NHEh7IKA6O+PkhucbnakSxOSbme0SsOklNUTou6Oqb1k3EuQlQHKS9CWCkbGw3vPdiSurWcSMwq4vlvY6jQG9SOZVHe/OkocRdy8XC2Z+EjrXG0l8HRQlQHKS9CWDEPZwc+GRaJk70t209kMHPjcbUjWYxv9iby7b4kNBr4cGgr6tZyVjuSEBZDyosQVq5ZgI55D7YE4PO/zvLdvkSVE5m/nSczeW39EQBeigqla2PZt0iI6iTlRQhBr+b+TLinMQCvrT/C3jOXVE5kvk6lFzBqRTR6g8LAVnUY3a2B2pGEsDhSXoQQADx/V0P6tvCnXK8wasVBkrKK1I5kdrIKy3jqy/3kl1TQpl4t3hnc/LrboQgh/hspL0IIoHIvsnfvb0nzOjqyCst4+ssD5JXIDKSbVVqhZ+TyaM5fKiLQ04lPhkWitZMBukIYg5QXIUQVJwdbljwWiY+bloSL+Yz48gAl5Xq1Y5k8RVGYvOYI+85l4aa14/PhbfFylT3XhDAWKS9CiCv465z44om2uGnt2Hs2ixe+i0FvsKi1LKvdnF8TWH3wArY2GhY80ppGvrIiuBDGJOVFCHGVZgE6ljzWBgdbG349epGp6w7LHkj/YvG20yzaehqAGQPCZWaREDVAyosQ4po6NvDig6ERaDTw7b4k5m0+oXYkk/PN3kTe+aVybZzJvZswtF2QyomEsA5SXoQQ/6pXc3/eHhAOwEd/nmLZX2dVTmQ6fjyUwpR1hwF4rnsDnukqU6KFqClSXoQQ1/VI+3qMv7tyDZg3fjrGV7vPqRvIBGw5ns6ElbEoCgzrUI+XokLVjiSEVZHyIoS4obE9GvJM1xAAXl9/lM92nFE5kXr+iL/Is19HU2FQ6B8RwJv3NZO1XISoYVJehBA3pNFomNSrSdVqsW9viGfxttMqp6p5Px1K4dnl0ZRVGOjZzJe5D7TExkaKixA1TcqLEOKmaDQaXu4ZytgejQB455fjfPznSZVT1Zzv9iUy9rsYKi4v+7/g4dbY28pbqBBqkP/zhBA3TaPRMOGexrx4eR+kub+dYM6m4xgsfB2YpTvP8uqawygKPNI+iPceaImdFBchVCP/9wkhbtnzPRox8d4mACzceprnv4uxyJV4DQaFeb8l8NbPxwB4pmsIbw8Il1tFQqjMTu0AQgjzNKpbA7xcHZiy9jAb4lK5kF3Mp49F4uPmqHa0alFYWsGL3x9i09E0ACbc05jn72oog3OFMAFy5UUI8Z892CaQ5U+1x8PZnkNJOQxcsIvjaXlqx7ptSVlFDF60i01H03CwtWHO/S0Y26ORFBchTISUFyHEbekQ4sXa0Z0J9nYhOaeY+xft5pfDqWrH+s92ncrkvo93cjwtn9puWr59pgMPtglUO5YQ4v+R8iKEuG3B3i6sHd2J9sGeFJRWMGrFQV5ZdYjC0gq1o920Cr2BhVtPMezzfWQXldOyro6fxtxBZL1aakcTQvyDlBchRLXwcHZg+VPtGd2tARoNfH/gAn0+3EFsUo7a0W7oVHoB9y/ezZxNCegvT4Ve+WxH/HSWMX5HCEujUSxsq9i8vDx0Oh25ubm4u7urHUcIq7TnzCUmrIwlJbcEWxsNL/RoxMg7G+BgZ1qfl/QGhc93nuXd3xIoqzDg5mjHtH7NGNy6joxvEaKG3crvbykvQgijyC0qZ8q6w/wcVzn+Jdjbhcm9w7g7zMckisGR5Fze+PEoB85nA9C1cW1mD26Ov85J5WRCWCcpL1JehDAJiqKwLjaZGRviySwoA6BTAy+m9AmjWYBOlUynMwqY99sJNlweVOziYMvUvk0Z2jbQJEqVENZKyouUFyFMSn5JOYu2nuaznWcpqzCg0cDAVnV4snMw4XVqpsSk5BTzwe8nWXXwAnqDgkYD/VsG8FLPUOrWcq6RDEKIf2cy5WXGjBls2LCB2NhYHBwcyMnJueFzFEXhzTffZMmSJWRnZ9O+fXsWLFhAs2bNbuo1pbwIYbqSsoqY82sCPx1KqToWEejBsA716NPCH0d722p9vbIKA38ev8iq6AtsSchAf3kbg7vDfHipZyhN/OQ9QghTYTLlZdq0aXh4eHDhwgWWLl16U+Vl9uzZzJgxg2XLltG4cWPefvtttm/fTkJCAm5ubjd8vpQXIUxfbFIOn+88yy9HUinXV74F1XK2p3dzfzo39KZDiBeeLg7/6WsXl+mJu5DDL0fSWB+bTHZRedXfdQjx5OWeoUTW86yW70MIUX1Mprz8bdmyZYwbN+6G5UVRFAICAhg3bhwTJ04EoLS0FF9fX2bPns2zzz57w9eS8iKE+cjIL+X7A0l8szeR5JziK/6uiZ8bHRt40dDHFR83R3zctPi6O1LLxZ6SMgN5JeXklZSTX1JBcnYxsUk5xCRlE5+aX3WFBcDHTcug1nW5P7IODX1u/AFICKGOW/n9bVJ7G509e5a0tDSioqKqjmm1Wu6880527dp1zfJSWlpKaWlp1Z/z8sx/aXIhrEVtNy3PdW/IyDsbsP1kBtsSMth9+hIJF/M5nlb5+C983LR0CPFiUOs63NHQW3aAFsLCmFR5SUur3ADN19f3iuO+vr6cP3/+ms+ZNWsWb775ptGzCSGMx9ZGQ/dQH7qH+gCQWVDKnjOX2H82i+ScEtLzS0jPKyWjoLTqqoqjvQ1ujva4Odrh7aKlRV0drYJq0SrIA3+do8wcEsKC3XJ5eeONN25YFvbv30+bNm3+c6h/vukoivKvb0STJk1iwoQJVX/Oy8sjMFD2IRHCnHm7aunbIoC+LQKuOK43KBSUVODkYGtyC94JIWrOLZeXMWPGMHTo0OueU79+/f8Uxs/PD6i8AuPv7191PD09/aqrMX/TarVotdr/9HpCCPNia6NB52yvdgwhhMpuubx4e3vj7e1tjCwEBwfj5+fH5s2badWqFQBlZWVs27aN2bNnG+U1hRBCCGFejHrdNTExkdjYWBITE9Hr9cTGxhIbG0tBQUHVOU2aNGHt2rVA5e2icePGMXPmTNauXcuRI0d4/PHHcXZ25uGHHzZmVCGEEEKYCaMO2H399df58ssvq/7899WULVu20K1bNwASEhLIzc2tOueVV16huLiY0aNHVy1S99tvv93UGi9CCCGEsHyyPYAQQgghVGe267xUh7+7mKz3IoQQQpiPv39v38w1FYsrL/n5lYtayXRpIYQQwvzk5+ej011/w1aLu21kMBhISUnBzc2t2hep+nsNmaSkJLklpQL5+atLfv7qkp+/uuTnb3yKopCfn09AQAA2NtefT2RxV15sbGyoW7euUV/D3d1d/vGqSH7+6pKfv7rk568u+fkb142uuPxNlqgUQgghhFmR8iKEEEIIsyLl5RZotVqmTZsm2xGoRH7+6pKfv7rk568u+fmbFosbsCuEEEIIyyZXXoQQQghhVqS8CCGEEMKsSHkRQgghhFmR8iKEEEIIsyLl5SYtXLiQ4OBgHB0diYyMZMeOHWpHshqzZs2ibdu2uLm54ePjw4ABA0hISFA7llWaNWsWGo2GcePGqR3FaiQnJ/Poo4/i5eWFs7MzERERREdHqx3LKlRUVDB16lSCg4NxcnIiJCSE6dOnYzAY1I5m9aS83ISVK1cybtw4pkyZQkxMDF26dKFXr14kJiaqHc0qbNu2jeeee449e/awefNmKioqiIqKorCwUO1oVmX//v0sWbKEFi1aqB3FamRnZ9O5c2fs7e355ZdfOHbsGO+99x4eHh5qR7MKs2fPZvHixXz88cfEx8czZ84c3n33XT766CO1o1k9mSp9E9q3b0/r1q1ZtGhR1bGwsDAGDBjArFmzVExmnTIyMvDx8WHbtm107dpV7ThWoaCggNatW7Nw4ULefvttIiIimD9/vtqxLN6rr77KX3/9JVd6VdK3b198fX1ZunRp1bHBgwfj7OzM8uXLVUwm5MrLDZSVlREdHU1UVNQVx6Oioti1a5dKqaxbbm4uAJ6enionsR7PPfccffr04e67777huXFxcTzxxBNVt1ldXV1p3bo1c+bMISsrq9qz1a9fn8cff7zav64p+PHHH2nTpg0PPPAAPj4+tGrVik8//VTtWFbjjjvu4I8//uDEiRMAHDp0iJ07d9K7d2+VkwmL25ixumVmZqLX6/H19b3iuK+vL2lpaSqlsl6KojBhwgTuuOMOwsPD1Y5jFb777jsOHjzI/v37b3jup59+yujRowkNDeXll1+madOmlJeXc+DAARYvXszu3btZu3ZtteZbu3atxW6Ud+bMGRYtWsSECROYPHky+/btY+zYsWi1Wh577DG141m8iRMnkpubS5MmTbC1tUWv1zNjxgweeughtaNZPSkvN0mj0VzxZ0VRrjomjG/MmDHExcWxc+dOtaNYhaSkJF544QV+++03HB0dr3vu7t27GTVqFPfccw/r1q27Yhn1e+65hxdffJFNmzZVe8ZWrVpV+9c0FQaDgTZt2jBz5kyg8ns9evQoixYtkvJSA1auXMnXX3/NN998Q7NmzYiNjWXcuHEEBAQwfPhwteNZNbltdAPe3t7Y2tpedZUlPT39qqsxwrief/55fvzxR7Zs2ULdunXVjmMVoqOjSU9PJzIyEjs7O+zs7Ni2bRsffvghdnZ26PX6qnNnzpyJRqNhyZIl19z/xcHBgfvuuw+o/KU8Z84cmjRpglarxcfHh8cee4wLFy5c8ZyYmBj69u2Lj48PWq2WgIAA+vTpc8V5/7xttHXrVjQaDd9++y1TpkwhICAAd3d37r777mvOUvv999/p0aMH7u7uODs707lzZ/7444/b/dFVC39/f5o2bXrFsbCwMJksUENefvllXn31VYYOHUrz5s0ZNmwY48ePl7GOJkDKyw04ODgQGRnJ5s2brzi+efNmOnXqpFIq66IoCmPGjGHNmjX8+eefBAcHqx3JavTo0YPDhw8TGxtb9WjTpg2PPPIIsbGx2NraAqDX6/nzzz+JjIwkMDDwhl931KhRTJw4kXvuuYcff/yRt956i02bNtGpUycyMzMBKCws5J577uHixYssWLCAzZs3M3/+fIKCgsjPz7/ha0yePJnz58/z2WefsWTJEk6ePEm/fv2uKFxff/01UVFRuLu78+WXX/L999/j6elJz549TaLAdO7c+arCdeLECerVq6dSIutSVFSEjc2VvyZtbW1lqrQpUMQNfffdd4q9vb2ydOlS5dixY8q4ceMUFxcX5dy5c2pHswqjRo1SdDqdsnXrViU1NbXqUVRUpHY0q3TnnXcqL7zwwhXH0tLSFEAZOnToDZ8fHx+vAMro0aOvOL53714FUCZPnqwoiqIcOHBAAZR169Zd9+vVq1dPGT58eNWft2zZogBK7969rzjv+++/VwBl9+7diqIoSmFhoeLp6an069fvivP0er3SsmVLpV27djf8Xoxt3759ip2dnTJjxgzl5MmTyooVKxRnZ2fl66+/VjuaVRg+fLhSp04d5eeff1bOnj2rrFmzRvH29lZeeeUVtaNZPSkvN2nBggVKvXr1FAcHB6V169bKtm3b1I5kNYBrPr744gu1o1ml2y0vCxcuVABl3759V/1dWFiY0r59e0VRFCUnJ0epVauWEhoaqixatEg5evToNb/ev5WXxYsXX3He8ePHFUD57rvvFEVRlM2bNyuAsmrVKqW8vPyKx8SJExWNRqMUFBTc8Psxtp9++kkJDw9XtFqt0qRJE2XJkiVqR7IaeXl5ygsvvKAEBQUpjo6OSkhIiDJlyhSltLRU7WhWTwbs3qTRo0czevRotWNYJUWWIjIpW7duveqYt7c3zs7OnD179obPv3TpElA5nuOfAgICOH/+PAA6nY5t27YxY8YMJk+eTHZ2Nv7+/owYMYKpU6dib29/3dfx8vK64s9/j8MpLi4G4OLFiwDcf//9//o1srKycHFxueH3ZEx9+/alb9++qmawVm5ubsyfP1/WNDJBUl6EELfN1taWHj168Msvv3DhwoXrDqj+u1SkpqZedV5KSgre3t5Vf27evDnfffcdiqIQFxfHsmXLmD59Ok5OTrz66qu3lfnv1/noo4/o0KHDNc+RQflCmCYZsCuEqBaTJk1CURRGjBhBWVnZVX9fXl7OTz/9xF133QVUDpb9//bv3098fDw9evS46rkajYaWLVvy/vvv4+HhwcGDB287b+fOnfHw8ODYsWO0adPmmg8HB4fbfh0hRPWTKy9CiGrRsWNHFi1axOjRo4mMjGTUqFE0a9aM8vJyYmJiWLJkCeHh4axdu5ZnnnmGjz76CBsbG3r16sW5c+d47bXXCAwMZPz48QD8/PPPLFy4kAEDBhASEoKiKKxZs4acnBzuueee287r6urKRx99xPDhw8nKyuL+++/Hx8eHjIwMDh06REZGxhVbggghTIeUFyFEtRkxYgTt2rXj/fffZ/bs2aSlpWFvb0/jxo15+OGHGTNmDACLFi2iQYMGLF26lAULFqDT6bj33nuZNWtW1W2lRo0a4eHhwZw5c0hJScHBwYHQ0FCWLVtWbQuEPfroowQFBTFnzhyeffZZ8vPz8fHxISIiwmK3HBDCEsjGjEIIIYQwKzLmRQghhBBmRcqLEEIIIcyKlBchhBBCmBUpL0IIIYQwK1JehBBCCGFWLG6qtMFgICUlBTc3NzQajdpxhBBCCHETFEUhPz+fgICAq3bz/ieLKy8pKSkEBgaqHUMIIYQQ/0FSUtJ1txgBCywvbm5uQOU37+7urnIaIYQQQtyMvLw8AgMDq36PX4/FlZe/bxW5u7tLeRFCCCHMzM0M+ZABu0IIIYQwK0YtL9u3b6dfv34EBASg0WhYt27dDZ+zbds2IiMjcXR0JCQkhMWLFxszohBCCCHMjFHLS2FhIS1btuTjjz++qfPPnj1L79696dKlCzExMUyePJmxY8eyevVqY8YUQgghhBkx6piXXr160atXr5s+f/HixQQFBTF//nwAwsLCOHDgAHPnzmXw4MFGSnnzTl7MR+dsTy1nB+xt5Y6bEMJ0KYpCaYWBgtIKFAW8XBywsZHlI4RlMKkBu7t37yYqKuqKYz179mTp0qWUl5djb29/1XNKS0spLS2t+nNeXp5RsukNCj3nb8dweQ9uN0c7vFwcqOXiQMParrQN9qRdfU/qeTnL+jJCiBqTW1ROXHIOh5JyiE3K5cTFfPJKyikoqaDi7zcswM5Gg4+bFl+dI37ujjSo7Uqnhl60DqqFo72tit+BELfOpMpLWloavr6+Vxzz9fWloqKCzMxM/P39r3rOrFmzePPNN42eraC0Ag9nB7KLylAUyC+pIL+kgnOXiohJzOGH6AsA1HbT0q6+J72a+xHV1A8HO7lCI4SoXucyC1kTk8yGuBROZxTe8HyNBioMCim5JaTkllQd/3jLKRztbWhb35PODb3p2cyPYG8XY0YXolqYVHmBq6dIKYpyzeN/mzRpEhMmTKj689/zxKubzsmeg6/dg96gkFtcTlZhGdlFZWTklxJ3IZf957KIu5BDRn4pGw6nsuFwKp4uDgxuXYchbYNo6ONa7ZmEENYjt6icnw+nsOZgMtHns6/4u3pezrSs60HLQA+aBbjj5eKAq6MdLlo7XBzsUBSFzIIyUnOLuZhXQmpuCXEXctl5KpOM/FJ2nMxkx8lM3vnlOF0aefNYx/rc1cQHW7nNJEyUSZUXPz8/0tLSrjiWnp6OnZ0dXl5e13yOVqtFq9XWRDwAbG00eLo44OniUHWsd/PKK0Il5XoOJeWw/WQGq6IvcDGvlE93nOXTHWdpV9+TsT0acUcj7xrLKoQwfzlFZSzedoYvd52juFwPgI0GujSqzaDWdejaqDa1/t/70bVp8NM54qdzvOKooiicSi9g56lMtiRksONkRlWRqePhxKMd6vFw+yB0TlffshdCTRrl70sbxn4hjYa1a9cyYMCAfz1n4sSJ/PTTTxw7dqzq2KhRo4iNjWX37t039Tp5eXnodDpyc3NVXaSuQm9ga0IG3+1P5M/j6VVjZbo08mZSrzCaBsgCekKIf1dQWsHnO8/y6fYz5JdWABDq68b9kXXpHxGAj7vjDb7CrUvKKuLrPedZeSCJnKJyADxdHBh/T2MeahuInUxUEEZ0K7+/jVpeCgoKOHXqFACtWrVi3rx5dO/eHU9PT4KCgpg0aRLJycl89dVXQOVU6fDwcJ599llGjBjB7t27GTlyJN9+++1NzzYylfLy/6XllrBk+xmW7zlHuV5Bo4GBrerwYlQodTyc1I4nhDAheoPC8t3n+PDPU2QVlgHQxM+Nl6JC6RHmUyMTAkrK9fx4KIVPtp2uGlPT2NeV1/o2pUuj2kZ/fWGdTKa8bN26le7du191fPjw4SxbtozHH3+cc+fOsXXr1qq/27ZtG+PHj+fo0aMEBAQwceJERo4cedOvaYrl5W+Jl4p497cEfjqUAoDWzoZXezVheMf6MoVRCMGZjAJeXhVXNaYl2NuF8fc0pm9zf1XeI8r1Br7Zm8j7v5+ouhLTo4kP0weEywcvUe1MpryowZTLy98OJeUwY2M8+85mAdC5oRdzH2iJv07eDISwRnqDwhd/neXdXxMorTDgqrVjYq8mJnOrJqeojA/+OMny3eepMCi4O9oxc1Bz+rYIUDuasCBSXky8vEDlQLmv95xnxsZ4SsoNuDva8daAcO5rGSDrxAhhRc5lFvLSD4c4cPlqS5dG3rwzuIVJXtk4lV7Aiz8c4lBSDgAPRNbljfua4aI1qbkfwkxJeTGD8vK30xkFTPj+f28G/VoGMHtwc5wd5M1ACEu35Xg6Y7+NIb+0AhcHW6b0acpD7QJN+gNMud7AB7+fZMHWUygK1Pdy5oOhrWgZ6KF2NGHmpLyYUXmByplJC7ac5sM/T6I3KDQLcOfTx9oQYIKfvIQQt09RFBZuPc3c3xJQFGhTrxbzh0ZQt5az2tFu2p4zlxi/MpbU3BLsbDTMHNicB9tW/xpbwnpIeTGz8vK3A+eyeHZ5NJcKy/B21fLpY5G0CqqldiwhRDUqKqvg5VVxbIhLBeCR9kFM69fMLFfjzi0qZ+LqODYdrVyfa3S3BrwUFSoTEMR/ciu/v83v/xYL1qa+J+ue60wTPzcyC0oZsmQP62KS1Y4lhKgmF7KLGLxoNxviUrG31TBjYDgzBjY3y+ICoHO2Z+EjrXn+roYALNx6mue/jaHk8mJ6QhiLef4fY8ECPZ1ZNaoTd4f5UlZhYNzKWOb9loCFXSATwuqcSi/ggcW7iU/Nw9vVgW9GdOCR9vXUjnXbbGw0vBgVytwHWmJvq2HD4VSGLtlDRn7pjZ8sxH8k5cUEuWrtWDIsklHdGgDw4Z+neOvneCkwQpipI8m5DPlkN6m5JTT0ceXHMXfQtr6n2rGq1f2RdVn+VHt0TvbEJuUwaNFfJGUVqR1LWCgpLybKxkbDxHub8Fb/ZgB8/tdZpq47gsEgBUYIc3LgXBYPfbqHS4VlNK+j4/tnO1rsYPwOIV6sHd2Jel7OJGUVM3TJHhIvSYER1U/Ki4kb1rE+cwa3QKOBFXsTeWV1HHopMEKYhR0nMxi2dB/5JRW0rV+LFSPaX7GpqyUKqe3K9892JMTbheScYoYs2c25zEK1YwkLI+XFDDzYNpD5QyKwtdGwKvoCL3wXQ7neoHYsIcR1bDmezlPLDlBcrufOxrX56sn2uDtax+7Mvu6OfPdMBxr6uJKaW8KQJbs5k1GgdixhQaS8mIn+EXVY8HAr7G01/ByXyriVsXIFRggTtffMJUZ+HU2Z3kCvcD8+fawNTg62aseqUT7ujnw7ogONfV25mFc5e/JUer7asYSFkPJiRu4N9+eTYZGVI/rjUnl9/REZxCuEiYm7kMNTXx6gtMJAjyY+fPhQK7OdCn27artp+XZEB5r4uZGRX8rDn+6VQbyiWljn/1Fm7K4mvrw/JKJqDMz7m0+oHUkIcdnJi/kM/3wfBaUVdAjxZMEjrbE3gY0V1eTlWllgQn3dSM8vZfjn+8gqLFM7ljBz1v1/lZnq2yKAt/qHA5XTqD/feVblREKIpKwiHl26l+yiclrW1fHZ8LY42lvXraJ/U8vFgS+fbEcdDyfOZBbyxLL9FJZWqB1LmDEpL2bq0Q71eCmqMQDTfz7G2pgLKicSwnpl5JfyyGd7uZhXSmNfV5Y90Q5X2Wn5Cn46R758sh0ezvYcSsph1IqDMvFA/GdSXszYc90b8mTnYABe+iGOrQnpKicSwvqUlOt5ZvkBErOKCPR0YvlT7all4dOh/6uGPq588XhbnOxt2X4ig1dWxcnaVeI/kfJixjQaDVP7hDGwVR30BoXnv4nh5EUZzS9ETTEYFF764RAxiTm4O9qx7Il2+Lo7qh3LpLUKqsXCR1tja6NhbUwy7/6WoHYkYYakvJg5GxsNswe3oF2wJ/mlFTz15QEZDCdEDZn/+wl+jkvFzkbD4mGRNKjtqnYks9A91IfZg1sAsGjrabntLW6ZlBcL4GBnw+JHIwn0dCIxq6hyfYkKuZcshDGtjbnAh3+eAmDmwOZ0auCtciLzcn9kXUZf3r9t4urDxCRmq5xImBMpLxbC08WBz4e3xU1rx76zWUxdd1jWgBHCSPafy2LiqsMAPHtnCA+2DVQ5kXl6KSqUu8N8Kasw8OzyaNJyS9SOJMyElBcL0sjXjY8eboWNBr4/cIGlMoVaiGqXklPMs8srV8+9t5kfE3s2UTuS2bKx0TB/aETVGjDPLD9ASble7VjCDEh5sTDdQn14rW9TAGZsjGf7iQyVEwlhOUor9IxacZCswjKa+rszb0hLbGw0ascya65aOz4b3oZazvbEXcjl5VVxctVY3JCUFwv0eKf6DG0biKLAC9/FkJxTrHYkISzCmz8d41BSDjonez4ZFomzg6zlUh0CPZ1Z9GgkdjYafjqUwpLtZ9SOJEyclBcLpNFoeOO+ZjSvoyO7qJzRKw5SWiGXYoW4Hd8fSOKbvYloNDB/aASBns5qR7IoHUK8mHZfMwDm/JrAvrNZKicSpkzKi4VytLdl4SOt0TlVrmY5Y0O82pGEMFtHknOZuu4IAON6NKZ7qI/KiSzTo+2DGBARULlu1bcHySwoVTuSMFFSXixYoKcz84dEAPDV7vOsi0lWN5AQZiinqKxq+YG7mvjw/F0N1Y5ksTQaDTMGNqehjysX80p54bsY9LICr7gGKS8WrnsTH8ZefrOdtOYwCWmyAq8QN8tgUBi/MpYL2cUEeTrz/oMRMkDXyFy0dix+tDXODrb8deoSH/x+Qu1IwgTVSHlZuHAhwcHBODo6EhkZyY4dO/713K1bt6LRaK56HD9+vCaiWqQX7m5Ml0beFJfrGfV1tOzmKsRN+vyvs2xJyMDBzoZFj7ZG52yvdiSr0NDHjVmDmgPw0ZZTbJNZk+IfjF5eVq5cybhx45gyZQoxMTF06dKFXr16kZiYeN3nJSQkkJqaWvVo1KiRsaNaLFsbDfOHROCvc+RMZiFv/HhU7UhCmLzDF3KZvanyQ9NrfZvSLECnciLr0j+iDo92CEJRYNx3MaTmyqxJ8T9GLy/z5s3jqaee4umnnyYsLIz58+cTGBjIokWLrvs8Hx8f/Pz8qh62trbGjmrRvFy1vD8kAo0Gfoi+wI+HUtSOJITJKiit4PlvD1KuV+jZzJdH2wepHckqvda3adWsyQkrD8n4F1HFqOWlrKyM6OhooqKirjgeFRXFrl27rvvcVq1a4e/vT48ePdiyZcu/nldaWkpeXt4VD3FtHUK8eL575fiXKWsOk5RVpHIiIUzTa+uOcO5SEXU8nJgzuCUajYxzUYPWzpYPH2qFs4Mtu89c4pPtp9WOJEyEUctLZmYmer0eX1/fK477+vqSlpZ2zef4+/uzZMkSVq9ezZo1awgNDaVHjx5s3779mufPmjULnU5X9QgMlD1Grmdsj0ZE1qtFfmkFL3wXQ4VeNnAU4v9bHX2BtTHJ2Npo+GBohIxzUVmwtwtvXF7/Zd5vJziUlKNuIGESamTA7j8/tSiK8q+fZEJDQxkxYgStW7emY8eOLFy4kD59+jB37txrnj9p0iRyc3OrHklJSdWe35LY2dowf0gEbo52HEzM4YM/TqodSQiTcSajgNfWV67nMv7uRrSp76lyIgHwQGRd+rTwp8Kg8MJ3MRTIpAOrZ9Ty4u3tja2t7VVXWdLT06+6GnM9HTp04OTJa/+S1Wq1uLu7X/EQ1xfo6Vw1kv/jLafYffqSyomEUF+53sC4lbEUlenpGOLFqG6ynoup0Gg0zBzQnDoeTpy7VCSTDoRxy4uDgwORkZFs3rz5iuObN2+mU6dON/11YmJi8Pf3r+54Vq1viwCGtKnc/2jC97HkFperHUkIVX305yniLuSic7Ln/SER2Mp6LiZF51z538VGA6tk0oHVM/ptowkTJvDZZ5/x+eefEx8fz/jx40lMTGTkyJFA5W2fxx57rOr8+fPns27dOk6ePMnRo0eZNGkSq1evZsyYMcaOanWm3deU+l7OpOaW8KZ8khFWLCYxmwVbTgHw9oBw/HSOKicS19Iu2JMxf086WHuYFNl01moZvbwMGTKE+fPnM336dCIiIti+fTsbN26kXr16AKSmpl6x5ktZWRkvvfQSLVq0oEuXLuzcuZMNGzYwaNAgY0e1Os4Odsy7/ElmTUwyvxxOVTuSEDWuqKyCCd9XTsPtHxFAv5YBakcS1zG2RyMiAj3IL6nglVVxGGT6tFXSKIpiUf/l8/Ly0Ol05ObmyviXmzT31wQ+3nKKWs72/Dq+Kz5u8qlTWI+p6w7z9Z5E/Nwd+XVcV5ldZAZOZxTQ58MdlJQbmN6/GY91rK92JFENbuX3t+xtJBjboxFN/d3JLipn0urDWFifFeJfbUlI5+s9lVd+5z7QUoqLmWhQ25WJ9zYBYObGeM5mFqqcSNQ0KS8CBzsb3h8SgYOtDX8cT2flfpluLixfdmEZr6yKA+CJzvW5o5G3yonErRjesT4dQ7woKTfw4vexsvqulZHyIgAI9XPj5Z6hALz18zESL8nqu8KyTfvxKBn5pTT0+d+neGE+bGw0vPtAC1y1lWtWyeq71kXKi6jy5B3BtAv2pLBMz0urDslAOGGxNh1J48dDKdjaaHjvgZY42sveaeaobi1nXu/XFID3N5/geJpsD2MtpLyIKn+/kTs72LLvbBZf7z2vdiQhql12YRlT11Wuovts1xBaBnqoG0jclgci63J3mA/leoUJKw9RLlueWAUpL+IKgZ7OVZfQ3/nluGzeKCzOGz8dJbOglEY+rrxwdyO144jbpNFomDWoBbWc7TmWmsfirXL7yBpIeRFXGdahHu2CPSkq0zNxdZzMPhIW49ejaayPTcFGA+8+0BKtndwusgS13bRVmzd++OdJEtLyVU4kjE3Ki7iKjY2GOYNb4Ghvw67Tl/hmX+KNnySEicsuLGPK2su3i+5sQITcLrIo97UMqLp99MqqQ1TI7SOLJuVFXFN9bxde7nl5HYUN8VzIlttHwry9+f9vF/WQ20WWRqPRMGNgc9wc7Th0IZfPdp5VO5IwIikv4l893qk+kfVqUVimZ9IaWbxOmK/fj11k3f+7XSSziyyTr7sjr/WtnH00b/MJTmcUqJxIGIuUF/GvbG00zLm/BVo7G3aczOT7A7J4nTA/eSXlVbOLRnQJkdtFFu6ByLp0bVybsgoDr6yKk8XrLJSUF3FdDWq78mJUYwBmbIgnPa9E5URC3Jp3fjlOWl4J9b2cGX9PY7XjCCOrnH3UHFetHdHns/ly1zm1IwkjkPIibujJzsE0r6Mjr6SCaT8eVTuOEDdtz5lLfLO3csD5rEEt5HaRlajj4cSk3pVj9t79NUGWfLBAUl7EDdnZ2jB7cAvsbDT8ciSNTUfS1I4kxA2VlOt5dXXl3kUPtQuiYwMvlROJmvRQ2yDaBXtSXK5nyrojMmbPwkh5ETelaYA7z94ZAsDr64+QW1yuciIhrm/+7yc5d6kIX3dt1adwYT1sbCpvHznY2bD9RAbrYpPVjiSqkZQXcdOev6sRId4upOeXMmtjvNpxhPhXR5Jz+XTHGQBmDGiOu6O9yomEGhrU/t+0+Ok/HeNSQanKiUR1kfIibpqjvS2zBjUH4Lv9Sew6nalyIiGuVq7/3yyTvi38ubupr9qRhIqe6RpCEz83sovKmf7zMbXjiGoi5UXckvYhXjzSPgiASWsOU1ymVzmREFdauvMsx1Lz8HC2r1oyXlgv+8tj9mw0sD42hS3H09WOJKqBlBdxy17t1QR/nSPnLxXx4Z8n1Y4jRJXzlwqZ//sJAKb2aYq3q1blRMIUtAz04MnOwQBMWXuYgtIKlROJ2yXlRdwyN0d7pvcPB2DJ9jPEp+apnEgIUBSFKWuPUFJuoFMDLwa3rqN2JGFCJkQ1pm4tJ1JyS5j7a4LaccRtkvIi/pN7mvrSK9wPvUHh1TWHZRVLobq1McnsPJWJ1s6GmQObo9Fo1I4kTIizgx0zB1aO2fty9zkOJeWoG0jcFikv4j97475muGntOJSUw/Ld59SOI6xYVmEZb10ejPnC3Y2o7+2iciJhiro2rs2AiAAUBV5dc5hy2XnabEl5Ef+Zr7sjE3v9bxXLlJxilRMJa/X2z8fILiqniZ8bI7qEqB1HmLCpfZvi4WxPfGoen8vO02ZLyou4LQ+3C6raefr19bKKpah5O05msCYmGY0GZg1qjr2tvK2Jf+ftqmVK7zAA3v/9BImXZOsAcyT/l4vb8vcqlva2Gn6PT5etA0SNKi7TM2Vt5Y7RwzvWp1VQLZUTCXNwf2RdOoZ4UVJuYMq6w/KhywxJeRG3rbGvG6PubADAtB+PklciWweImvHhnydJzCrCX+fISz1D1Y4jzIRGo2Hm5a0DdpzMZH1sitqRxC2S8iKqxejuDau2Dpiz6bjacYQVOJ6Wx6fbK7cAePO+Zrhq7VROJMxJsLdL1dYBb/18jOzCMpUTiVtRI+Vl4cKFBAcH4+joSGRkJDt27Lju+du2bSMyMhJHR0dCQkJYvHhxTcQUt8HR3pa3B1au/bJibyLR57NVTiQsmcGgMGnNYSoMCj2b+RLVzE/tSMIMjegSQqivG5cKy5gp+7WZFaOXl5UrVzJu3DimTJlCTEwMXbp0oVevXiQmJl7z/LNnz9K7d2+6dOlCTEwMkydPZuzYsaxevdrYUcVt6tTAm/sj66IoMFmmIQojWrH3PDGJObhq7XjzvnC14wgz5WBnw8xBlf9+foi+wO7Tl1ROJG6W0cvLvHnzeOqpp3j66acJCwtj/vz5BAYGsmjRomuev3jxYoKCgpg/fz5hYWE8/fTTPPnkk8ydO9fYUUU1mNI7DE8XBxIu5rPk8iV9IarTxbwS5myqXCH15Z6h+OkcVU4kzFlkPc+q/dqmrDtMaYXs12YOjFpeysrKiI6OJioq6orjUVFR7Nq165rP2b1791Xn9+zZkwMHDlBefvVA0NLSUvLy8q54CPXUcnFgap/KaYgf/nGS85cKVU4kLM0bPx4lv7SCiEAPHu1QT+04wgK8cm8TartpOZNRyMItp9WOI26CUctLZmYmer0eX98rt6T39fUlLe3aU2rT0tKueX5FRQWZmZlXnT9r1ix0Ol3VIzAwsPq+AfGfDGxVh84NvSitMDB1naz9IqrP78cu8suRNGwvT9G3tZEtAMTt0znZM61fUwAWbT3NqfQClROJG6mRAbv/3GNEUZTr7jtyrfOvdRxg0qRJ5ObmVj2SkpKqIbG4HRqNhhkDmqOVaYiiGhWWVvD6+so1XZ6+I5gwf3eVEwlL0qe5P91Da1OmNzB57WEMsl+bSTNqefH29sbW1vaqqyzp6elXXV35m5+f3zXPt7Ozw8vL66rztVot7u7uVzyE+up7uzBWpiGKavTebydIyS0h0NOJF+5upHYcYWE0Gg3T+4fjZG/LvrNZ/BAtH4RNmVHLi4ODA5GRkWzevPmK45s3b6ZTp07XfE7Hjh2vOv+3336jTZs22NvbGy2rqH4juoTQ2NdVpiGK23b4Qi7LdlXuQ/P2gOY4O8iaLqL6BXo6M/6eymI8c+NxMgtKVU4k/o3RbxtNmDCBzz77jM8//5z4+HjGjx9PYmIiI0eOBCpv+zz22GNV548cOZLz588zYcIE4uPj+fzzz1m6dCkvvfSSsaOKauZgZ8OsQZVb0P8QfYFdp68esyTEjVToDby6Jg6DAve1DODOxrXVjiQs2JOdg2nq705ucXnVTuXC9Bi9vAwZMoT58+czffp0IiIi2L59Oxs3bqRevcpZAqmpqVes+RIcHMzGjRvZunUrERERvPXWW3z44YcMHjzY2FGFEVwxDXHtEUrKZRqiuDXLdp3jaEoe7o52vNa3qdpxhIWzs6380GWjgfWxKWxNSFc7krgGjWJhU0Hy8vLQ6XTk5ubK+BcTkVtczj3ztpGeX8rYuxoyIUr2oBE3JymriKj3t1NcruedQc0Z2i5I7UjCSrz501G++OscdWs58dv4rnKrsgbcyu9v2dtIGJ3OyZ437msGwKJtpzl5MV/lRMIcKIrC6+uPUFyup12wJw+2kWUQRM15MSqUAJ0jF7KL+eD3k2rHEf8g5UXUiF7hfvRo4kO5vnJPGpmGKG5kw+FUtiRk4GBrw8yBzbGRNV1EDXLV2jG9f+XWAZ/tPMvRlFyVE4n/T8qLqBEajYbpA8JxdrDlwPlsvt1/7b2thADILSrnjR8rB0uO6taAhj6uKicS1ujupr70bu6H/vJGoHr50GUypLyIGlPHw4kXL493eWfjcS7mlaicSJiqmRvjySwopaGPK6O7N1A7jrBib/RrhpujHXEXcvly1zm144jLpLyIGvV4p/q0rKsjv7SCaeuPqh1HmKDdpy+x8kDlAmGzBjVHa2erciJhzXzcHXm1VxMA5v6WQHJOscqJBEh5ETWsck+aFtjZaNh0NI1NR669x5WwTiXleiavPQzAI+2DaFvfU+VEQsBDbYNoW78WRWV6pq49LPu1mQApL6LGNQ1w55muIQC8vv4IeSVX7xYurNNHf57kbGYhvu5aJl7+tCuE2mwuf+hysLVhS0IGPx6S/drUJuVFqGJsj0YEe7uQnl/K7F+Oqx1HmID41Dw+2XYGgDfvC8fdUbYDEaajoY8rz9/VEIA3fzpGluzXpiopL0IVjva2zBxYuXXAir2J7DubpXIioaa/Z3NUGBR6NvPl3nA/tSMJcZVn72xAqK8bWYVlvL1Btg5Qk5QXoZqODbwY2rZy4bFJa+Jk6wArtmzXOWKTcnD7f2trCGFqHOxseGdwczQaWHMwme0nMtSOZLWkvAhVTeoVRm03LaczCvn4z1NqxxEqSLxUxNxfEwCY1DsMX3dHlRMJ8e9aBdXi8U71AZi89jBFZRXqBrJSUl6EqnTO9kz/f1sHyCqW1kVRFF5dE0dxuZ6OIV481E62ABCm76WoUOp4OHEhu5i5v55QO45VkvIiVNeruT+9witXsXxlVRzleoPakUQNWbk/iV2nL+Fo//fleNkCQJg+F60dMwZW3t78YtdZos9nq5zI+kh5ESbhzf7N8HC252hKHku2n1E7jqgBabklzNgQD1R+kq3n5aJyIiFuXrdQHwa3rouiwCurDsmYvRom5UWYBB83R17v2xSAD34/yal02XnakimKwtR1h8kvraBloAdPdA5WO5IQt+y1vv8bs/fBH7LzdE2S8iJMxsBWdegWWpsyvYFXVsXJJmgW7Ke4VH6PT8feVsO797fAVnaMFmbIw9mBtwdU3j5asv0McRdy1A1kRaS8CJOh0WiYObA5rlo7DibmsEw2QbNImQWlvPFj5b5WY7o3orGvm8qJhPjvejbzo1/LgKoxe2UVMmavJkh5ESYlwMOJSb0rl4V/99fjnMssVDmRqE6KovDauiNkFZbRxM+NUd1kx2hh/t7o1xRPFweOp+WzcKss+VATpLwIk/NQ2yA6hnhRUm7g5VWH5PaRBfkpLpVfjqRhZ6PhvQdb4mAnb0HC/Hm5annz8pIPH/95imMpeSonsnzyziFMjo2Nhjn3t8DFwZb957L54q+zakcS1SA9v4TX1x8BYMxdDWkWoFM5kRDVp28Lf6Ka+lJhUHjxh0Ny+8jIpLwIkxTo6cyUPpWzj+b8msCp9AKVE4nboSgKU9YeIaeonKb+7jzXvaHakYSoVhqNhhkDm1PL2Z741Dw+lNlHRiXlRZish9oF0rVxbcoqDLz4wyEqZPE6s7UuNpnNxy5ib1t5u8jeVt56hOWp7aZlxuUNZxduPUVMoixeZyzyDiJMlkajYfbg5rg52nEoKYdPZPE6s3Qxr4Rp6ytnF73QoxFh/u4qJxLCeHo39+e+lgEYFHjxB1m8zlikvAiT5q9zYlq/yoFw838/wfE0GQhnThRF4dXVceSVVNC8jo6Rd8rsImH5pvdvho+bljMZhczZlKB2HIsk5UWYvMGt63B3mC/leoUJK2UgnDn5em8iWxIycLCz4b0HW2Int4uEFfBwdmD24BYAfP7XWXafvqRyIssj7yTC5Gk0GmYOCqeWsz3HUvN4b7N8kjEHpzMKmLHhGACv3ttEFqMTVqV7Ex+Gtq3cJf3lVYfILylXOZFlkfIizIKPmyOzBlV+klmy/Yx8kjFx5XoD476LpaTcwB0NvXm8U321IwlR46b2bUrdWk5cyC5m2uVVpUX1kPIizMa94X4MbRuIosCE72PJLZJPMqbqg99Pcjg5F52TPXMfaImN7F0krJCr1o73h0Rgo4E1B5P58VCK2pEshlHLS3Z2NsOGDUOn06HT6Rg2bBg5OTnXfc7jjz+ORqO54tGhQwdjxhRm5LW+Tanv5UxqbgmT1x1GUWT1XVNz4FxW1RLpswY1x0/nqHIiIdTTtr4nYy6vazRl7WEuZBepnMgyGLW8PPzww8TGxrJp0yY2bdpEbGwsw4YNu+Hz7r33XlJTU6seGzduNGZMYUZctHbMH9oKWxsNG+JSWXMwWe1I4v/JLyln/PexGBQY3LouvZv7qx1JCNWN7dGIVkEe5JdUMH5lrKxZVQ2MVl7i4+PZtGkTn332GR07dqRjx458+umn/PzzzyQkXH/ApVarxc/Pr+rh6en5r+eWlpaSl5d3xUNYtohAD8bf3QiAaT8eJSlLPsmYAkVReH39UZKyiqlby4k37muqdiQhTIKdrQ0fDGmFq9aO/eeyWbj1tNqRzJ7Rysvu3bvR6XS0b9++6liHDh3Q6XTs2rXrus/dunUrPj4+NG7cmBEjRpCenv6v586aNavqtpROpyMwMLDavgdhukZ1a0jb+rUoKK1g7HcxlMsnGdX9EH2BtTHJ2NpomD8kAjdHe7UjCWEygrycmd6/cs2qD/44SfR5WX33dhitvKSlpeHj43PVcR8fH9LS0v71eb169WLFihX8+eefvPfee+zfv5+77rqL0tLSa54/adIkcnNzqx5JSUnV9j0I02Vro2HegxG4OdoRk5jDu7/K9Gk1nbiYX7Xp4oR7GtOm/r9fLRXCWg1sVYf7WgagNyiMWxlDbrFMOvivbrm8vPHGG1cNqP3n48CBA0Dl+hz/pCjKNY//bciQIfTp04fw8HD69evHL7/8wokTJ9iwYcM1z9dqtbi7u1/xENYh0NOZd+9vCVROn/792EWVE1mn4jI9z604SEm5gS6NvBklq+gKcU0ajYa3B4ZTt5YTSVnFvPzDIZl08B/dcnkZM2YM8fHx132Eh4fj5+fHxYtX/zLJyMjA19f3pl/P39+fevXqcfKk7NAprnZvuB9PdK4PVO4jkpxTrG4gKzTtxyOcTC+gtpu2clqoTIsW4l+5O9qz8JHWONja8NuxiyzdeVbtSGbJ7laf4O3tjbe39w3P69ixI7m5uezbt4927doBsHfvXnJzc+nUqdNNv96lS5dISkrC319mLYhrm9QrjIPnszl0IZcx3xzk+2c7yq7FNWRtzAW+P3ABjQY+GBqBt6tW7UhCmLwWdT2Y2jeM19cf5Z1fjtMqqBaR9WqpHcusGO0dPiwsjHvvvZcRI0awZ88e9uzZw4gRI+jbty+hoaFV5zVp0oS1a9cCUFBQwEsvvcTu3bs5d+4cW7dupV+/fnh7ezNw4EBjRRVmzsHOho8fbo375fEvczYdVzuSVTiVXsCUtZXjXMbe1YhODW78oUYIUWlYh3r0beFPhUFhzDcHySosUzuSWTHqx9MVK1bQvHlzoqKiiIqKokWLFixfvvyKcxISEsjNzQXA1taWw4cP079/fxo3bszw4cNp3Lgxu3fvxs1N9kUR/y7Q05l3H6gc//LpjrP8dvTfB4WL25dfUs6zyw9QVKanQ4gnY3s0UjuSEGZFo9HwzuAWhHi7kJpbwviVsRgMMv7lZmkUCxstlJeXh06nIzc3VwbvWqHpPx3j87/O4qq1Y91znWno46p2JItjMCiM/Dqa345dxM/dkZ+ev4PabnK7SIj/Ij41jwEL/qK0wsCL9zTmeSv+IHArv79lYICwKJN6N6FdsCcFpRU889UBmYpoBAu2nOK3YxdxsLVh0aOtpbgIcRvC/N15a0A4APN+PyGzJm+SlBdhUextbVj4SGsCdI6cySxk3Hcx6OVSbLXZcjydeb+fAOCtAc1oFSSDDIW4XQ+2CeTRDkEoCoxbGcvJi/lqRzJ5Ul6ExfF21bLksTY42tuwJSGD936TBeyqw7nMQsZ+F4OiwCPtgxjSNkjtSEJYjGn9mtEhpPKq8dNfHSCnSAbwXo+UF2GRwuvomD24BQALt57m5zjZiv525JeU88zyA+SXVBBZrxbT+jVTO5IQFqXyqnEkdWs5cf5SEc99c1A2cLwOKS/CYvWPqMMzXUMAePmHOI4k56qcyDyV6w2MXnGQExcL8HHTsuiR1jjYyVuHENXN08WBTx9rg7ODLX+dusTbG+LVjmSy5B1IWLSJ9zahSyNvisv1PLFsPxeyZQfqW6EoClPXHmHHyUyc7G35bHgbfNwd1Y4lhMUK83dn3oMRACzbdY5v9iaqG8hESXkRFs3WRsOCR1rTxM+NjPxSHv9iv9xLvgULt55m5YEkbDTw0UOtaFHXQ+1IQli8e8P9mHBPYwCmrjvMZpmBdBUpL8LiuTva88UTbfHXOXIqvYBnvoqmpFyvdiyTtz42uWq37jfua8bdTW9+TzIhxO15/q6GPNimLgYFxnxzkOjzWWpHMilSXoRV8Nc5seyJdrg52rHvXBYvfn9IVrO8jj1nLvHyD3EAjOgSzGMd66sbSAgro9FomDmwOXc18aG0wsBTXx7gVLpMof6blBdhNUL93PhkWCT2tho2HE5lxsZ42Y7+Go4k5zLiqwOU6Q30CvdjUq8wtSMJYZXsbG34+OFWRAR6kFNUzvDP95OWW6J2LJMg5UVYlU4NvJl7eQ+kpTvP8sEfJ1VOZFqOp+Xx6NK95JdU0LZ+Ld4fEoGNjUbtWEJYLWcHOz5/vC0h3i4k5xTz+Bf7yC2SlcOlvAir0z+iDq/1bQrA/N9P8pEUGKByl+hHP9tLTlE5LQM9+Pzxtjja26odSwir5+niwJdPtqO2m5bjafk8unSv1RcYKS/CKj11RzCTejUB4L3NJ1i49ZTKidR1/lIhj3y2h8yCMpr6u/PVE+1wc7RXO5YQ4rJAT2eWP9UOTxcHDifn8sjSPVY9c1LKi7Baz97ZgJd7hgIwZ1MCS7afVjmROpJzinn4071czCulkY8ry59qh85ZiosQpqaJnzvfjuiAl4sDR5LzeOSzvVZbYKS8CKv2XPeGVespzNx4nM92nFE5Uc06lV7Ag4t3k5xTTLC3CytGtMfLVXaJFsJUhfq58c3lAnM0JY+HP91LdqH1FRgpL8Lqje3RiLE9GgHw9oZ43vnluFVMo45NyuGBxbtIzikmxNuFFU+3x8dNVs8VwtSF+rnx7TMd8HZ14FhqHg9/tpf0fOuahSTlRQhg/N2NeCmq8grM4m2nmfB9LGUVlrsp2rYTGTy0ZA/ZReW0rKvjh5EdCfBwUjuWEOImNfZ149sRHfB21RKfmsfABbs4cdF61oGR8iIElQtCjbmrEe/e3wI7Gw3rYlN4Ytk+8kosb0T/uphknlq2n+JyPV0aeVdegpZbRUKYnUa+bqwe1ZHgy9OoBy/axV+nMtWOVSOkvAjx/zzQJpClj7et2tX1wcW7LWZRKINB4aM/TjJuZSwVBoX7WgawdHhbXLR2akcTQvxH9bxcWDOqE23r1yK/pILhn+/jhwNJascyOikvQvzDnY1r8/2zHfF2rVxToe9HO9ll5p9mcovKGfHVAd7bfAKAJzrXZ/6QCBzs5C1ACHNXy8WB5U+1576WAVQYFF5eFcecTcep0FvurW+NYmHro+fl5aHT6cjNzcXd3V3tOMKMJWUV8fSXB0i4mI9GAy/0aMTzdzXC1sxWnD2SnMuoFdEkZRXjYGfD2/3DebBtoNqxhBDVzGBQmLf5BB9vqVy3ql19T+YPjTCb8Wy38vtbyosQ11FcpueNH4+y8vJl2DsaevP+kAhqu5n+GBFFUfjhwAWmrj9CWYWBQE8nFj0SSXgdndrRhBBGtD42mSlrj1BQWoHOyZ7Zg1twb7if2rFuSMqLlBdRzVZHX2DquiMUl+up7aZl9uDm3NXEV+1Y/yolp5jX1x/h9/h0AHo08WHegxGy+JwQVuL8pULGfhvDoQu5ADzaIYipfZqa9JYfUl6kvAgjOJWez+gVBzlxsQCAe5r6Mq1fU+rWclY52f/oDQpf7T7H3F8TKCzTY2+rYdzdjRl1ZwPZYFEIK1NWYeC9zQl8sq1y8c16Xs5M7dOUu8N80GhM7/1AyouUF2EkxWV65v9+gqU7z1JhUHC0t+H5uxoxokuI6oNfj6bkMnnN4apPWm3q1WLWoOY08nVTNZcQQl3bT2Tw8qpDXMwrBaBLI2+m9WtKQx/Tem+Q8iLlRRjZiYv5TF13hH1nswAIqe3C83c1pG+LAOxta7bEHE3JZcGWU/xyJA1FATetHa/2bsJDbYPkaosQAoCC0goWbDnF0h1nKdMbsLPRMLxTfUZ3a2Ay6zxJeZHyImqAoiisi01mxobjZBZUfqIJ0Dny5B3BDG0XhKuR10+JPp/Ngi2n+PN4etWxvi38ea1vU3zdZZl/IcTVzmUW8vaGeH6PvwiAg50N/VsG8Hjn+jQLUHcwv8mUlxkzZrBhwwZiY2NxcHAgJyfnhs9RFIU333yTJUuWkJ2dTfv27VmwYAHNmjW7qdeU8iJqWl5JOct3n+eLv85VlRg3RzuGtAmkV3M/IgJrVdv06pScYjYeTmXD4VRiEnMAsNFA3xYBPNe9IaF+pnUZWAhhmrafyGDubwnEXb7NDNAu2JPhHevTLbS2KotXmkx5mTZtGh4eHly4cIGlS5feVHmZPXs2M2bMYNmyZTRu3Ji3336b7du3k5CQgJvbjd+YpbwItZSU61kXk8ySHWc4k1FYddzTxYG7mvhwd5gvHUI88XB2uOmvWVhawcn0AvadvcTGw2nEJuVU/Z2djYZBreswqltDgr1dqvNbEUJYAUVROJiYw7Jd5/jlcCoVlzektbfV0KaeJ10b16ZrY2+a+rvXyABfkykvf1u2bBnjxo27YXlRFIWAgADGjRvHxIkTASgtLcXX15fZs2fz7LPP3vC1pLwItRkMCn8eT+fHQylsSUgnv6Tiir/3cLanvpcLwd4u1PdywdHeBr2ioNcr6BWF4nI9p9MLSbiYR1JW8RXP1WigbX1Peof70au5v9weEkJUi7TcEpbvOcf62BQuZF/5vuOmtaNOLSfqeDhRp5YTAR6Vj17hftU6xs9sy8uZM2do0KABBw8epFWrVlXH+/fvj4eHB19++eVVzyktLaW0tLTqz3l5eQQGBkp5ESahXG9g/7ksfj+Wzp/HL3LuUtEtf43ablrC/N25J8yHns388JHCIoQwEkVROJtZyI6TmWw/kcHuM5coKtNfdZ6tjYYTb/eq1hXHb6W8mNSObGlpaQD4+l65+Jevry/nz5+/5nNmzZrFm2++afRsQvwX9rY2dGrgTacG3rzerylFZRWcv1TE2cxCzmYWknipiHKDAVuNBlubyoe9rQ3B3i409nWjsa+rycwEEEJYPo1GQ0htV0JquzK8U33KKgycv1RIck4xKTklJOcUkZJTQlmFQdWtUm65vLzxxhs3LAv79++nTZs2/znUP++tKYryr/fbJk2axIQJE6r+/PeVFyFMkbODHWH+7oT5y1VBIYTpc7CzoZGvm8mtF3XL5WXMmDEMHTr0uufUr1//P4Xx86vceyEtLQ1/f/+q4+np6VddjfmbVqtFq5VPpkIIIYS1uOXy4u3tjbe3tzGyEBwcjJ+fH5s3b64a81JWVsa2bduYPXu2UV5TCCGEEObFqEuBJiYmEhsbS2JiInq9ntjYWGJjYykoKKg6p0mTJqxduxaovF00btw4Zs6cydq1azly5AiPP/44zs7OPPzww8aMKoQQQggzYdQBu6+//voVM4T+vpqyZcsWunXrBkBCQgK5uf9bJOeVV16huLiY0aNHVy1S99tvv93UGi9CCCGEsHwWtz1Abm4uHh4eJCUlyVRpIYQQwkz8PeEmJycHne76WxWY1FTp6pCfnw8gM46EEEIIM5Sfn3/D8mJxV14MBgMpKSm4ublV+3LGf7dCuaqjDvn5q0t+/uqSn7+65OdvfIqikJ+fT0BAADY21x+Sa3FXXmxsbKhbt65RX8Pd3V3+8apIfv7qkp+/uuTnry75+RvXja64/M2os42EEEIIIaqblBchhBBCmBUpL7dAq9Uybdo0WdFXJfLzV5f8/NUlP391yc/ftFjcgF0hhBBCWDa58iKEEEIIsyLlRQghhBBmRcqLEEIIIcyKlBchhBBCmBUpL0IIIYQwK1JebtLChQsJDg7G0dGRyMhIduzYoXYkqzFr1izatm2Lm5sbPj4+DBgwgISEBLVjWaVZs2ah0WgYN26c2lGsRnJyMo8++iheXl44OzsTERFBdHS02rGsQkVFBVOnTiU4OBgnJydCQkKYPn06BoNB7WhWT8rLTVi5ciXjxo1jypQpxMTE0KVLF3r16kViYqLa0azCtm3beO6559izZw+bN2+moqKCqKgoCgsL1Y5mVfbv38+SJUto0aKF2lGsRnZ2Np07d8be3p5ffvmFY8eO8d577+Hh4aF2NKswe/ZsFi9ezMcff0x8fDxz5szh3Xff5aOPPlI7mtWTdV5uQvv27WndujWLFi2qOhYWFsaAAQOYNWuWismsU0ZGBj4+Pmzbto2uXbuqHccqFBQU0Lp1axYuXMjbb79NREQE8+fPVzuWxXv11Vf566+/5EqvSvr27Yuvry9Lly6tOjZ48GCcnZ1Zvny5ismEXHm5gbKyMqKjo4mKirrieFRUFLt27VIplXXLzc0FwNPTU+Uk1uO5556jT58+3H333WpHsSo//vgjbdq04YEHHsDHx4dWrVrx6aefqh3Latxxxx388ccfnDhxAoBDhw6xc+dOevfurXIyYXG7Sle3zMxM9Ho9vr6+Vxz39fUlLS1NpVTWS1EUJkyYwB133EF4eLjacazCd999x8GDB9m/f7/aUazOmTNnWLRoERMmTGDy5Mns27ePsWPHotVqeeyxx9SOZ/EmTpxIbm4uTZo0wdbWFr1ez4wZM3jooYfUjmb1pLzcJI1Gc8WfFUW56pgwvjFjxhAXF8fOnTvVjmIVkpKSeOGFF/jtt99wdHRUO47VMRgMtGnThpkzZwLQqlUrjh49yqJFi6S81ICVK1fy9ddf880339CsWTNiY2MZN24cAQEBDB8+XO14Vk3Kyw14e3tja2t71VWW9PT0q67GCON6/vnn+fHHH9m+fTt169ZVO45ViI6OJj09ncjIyKpjer2e7du38/HHH1NaWoqtra2KCS2bv78/TZs2veJYWFgYq1evVimRdXn55Zd59dVXGTp0KADNmzfn/PnzzJo1S8qLymTMyw04ODgQGRnJ5s2brzi+efNmOnXqpFIq66IoCmPGjGHNmjX8+eefBAcHqx3JavTo0YPDhw8TGxtb9WjTpg2PPPIIsbGxUlyMrHPnzlctC3DixAnq1aunUiLrUlRUhI3Nlb8mbW1tZaq0CZArLzdhwoQJDBs2jDZt2tCxY0eWLFlCYmIiI0eOVDuaVXjuuef45ptvWL9+PW5ublVXwXQ6HU5OTiqns2xubm5XjS1ycXHBy8tLxhzVgPHjx9OpUydmzpzJgw8+yL59+1iyZAlLlixRO5pV6NevHzNmzCAoKIhmzZoRExPDvHnzePLJJ9WOJhRxUxYsWKDUq1dPcXBwUFq3bq1s27ZN7UhWA7jm44svvlA7mlW68847lRdeeEHtGFbjp59+UsLDwxWtVqs0adJEWbJkidqRrEZeXp7ywgsvKEFBQYqjo6MSEhKiTJkyRSktLVU7mtWTdV6EEEIIYVZkzIsQQgghzIqUFyGEEEKYFSkvQgghhDArUl6EEEIIYVakvAghhBDCrEh5EUIIIYRZkfIihBBCCLMi5UUIIYQQZkXKixBCCCHMipQXIYQQQpgVKS9CCCGEMCv/B8+TuSQZSCxxAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Compute the x and y coordinates for points on sine and cosine curves\n", + "x = np.arange(0, 3 * np.pi, 0.1)\n", + "y_sin = np.sin(x)\n", + "y_cos = np.cos(x)\n", + "\n", + "# Set up a subplot grid that has height 2 and width 1,\n", + "# and set the first such subplot as active.\n", + "plt.subplot(2, 1, 1)\n", + "\n", + "# Make the first plot\n", + "plt.plot(x, y_sin)\n", + "plt.title('Sine')\n", + "\n", + "# Set the second subplot as active, and make the second plot.\n", + "plt.subplot(2, 1, 2)\n", + "plt.plot(x, y_cos)\n", + "plt.title('Cosine')\n", + "\n", + "# Show the figure.\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "34673720-8427-4f00-baf7-46036206c1cc", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## PyTorch 基础操作" + ] + }, + { + "cell_type": "markdown", + "id": "21b0466c-fb43-4762-9166-2dd12c1034f2", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "首先,我们导入`torch`。请注意,虽然它被称为PyTorch,但是代码中使用`torch`而不是`pytorch`" + ] + }, + { + "cell_type": "code", + "execution_count": 95, + "id": "7072100b-254f-41d6-b9e8-d27a37a25ce1", + "metadata": { + "origin_pos": 5, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch" + ] + }, + { + "cell_type": "markdown", + "id": "ecd36dfc-bc63-4547-97a5-38711e5b02d3", + "metadata": { + "tags": [] + }, + "source": [ + "### 基础数据操作" + ] + }, + { + "cell_type": "markdown", + "id": "8e3d4848-c8d4-4355-afbc-b5c2d998fc6b", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "张量表示由一个数值组成的数组,这个数组可能有多个维度" + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "id": "00494188-41fc-43a5-9d11-3bd6484a0c2c", + "metadata": { + "origin_pos": 12, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])" + ] + }, + "execution_count": 96, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = torch.arange(12)\n", + "x" + ] + }, + { + "cell_type": "markdown", + "id": "d9bdb205-33a2-4768-a536-a36fc9756b7d", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "可以通过张量的`shape`属性来访问张量(沿每个轴的长度)的*形状*\n", + "和张量中元素的总数" + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "id": "257a7b06-2f7c-404f-b83e-5380b61bc16d", + "metadata": { + "origin_pos": 15, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([12])" + ] + }, + "execution_count": 97, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "id": "8c9f723b-e97c-4bc3-a306-065c328bbd82", + "metadata": { + "origin_pos": 18, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "12" + ] + }, + "execution_count": 98, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x.numel()" + ] + }, + { + "cell_type": "markdown", + "id": "6c16fb49-9a2b-4078-9a14-465b3f0b4775", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "要想改变一个张量的形状而不改变元素数量和元素值,可以调用`reshape`函数" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "id": "705f5253-c52b-4f7d-97b9-f144f8401919", + "metadata": { + "origin_pos": 21, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 0, 1, 2, 3],\n", + " [ 4, 5, 6, 7],\n", + " [ 8, 9, 10, 11]])" + ] + }, + "execution_count": 99, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = x.reshape(3, 4)\n", + "X" + ] + }, + { + "cell_type": "markdown", + "id": "1d098270-4487-41a4-b7e3-b49e0034fd09", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "使用全0、全1、其他常量,或者从特定分布中随机采样的数字" + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "id": "2c571f46-7e48-481b-8a5e-4e2dd2f2aa96", + "metadata": { + "origin_pos": 25, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[0., 0., 0., 0.],\n", + " [0., 0., 0., 0.],\n", + " [0., 0., 0., 0.]],\n", + "\n", + " [[0., 0., 0., 0.],\n", + " [0., 0., 0., 0.],\n", + " [0., 0., 0., 0.]]])" + ] + }, + "execution_count": 100, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.zeros((2, 3, 4))" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "id": "9950f805-d1e4-4948-b541-3c7de8b5a8e0", + "metadata": { + "origin_pos": 29, + "slideshow": { + "slide_type": "-" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[1., 1., 1., 1.],\n", + " [1., 1., 1., 1.],\n", + " [1., 1., 1., 1.]],\n", + "\n", + " [[1., 1., 1., 1.],\n", + " [1., 1., 1., 1.],\n", + " [1., 1., 1., 1.]]])" + ] + }, + "execution_count": 101, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.ones((2, 3, 4))" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "id": "e7262b6f-2be5-47e4-b115-66c9579fdd69", + "metadata": { + "origin_pos": 33, + "slideshow": { + "slide_type": "-" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[-1.6135, 0.3384, -0.1105, -0.2204],\n", + " [ 1.8458, 1.0330, 1.3026, -1.9075],\n", + " [-0.9892, -1.4300, 0.3078, -0.5878]])" + ] + }, + "execution_count": 102, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.randn(3, 4)" + ] + }, + { + "cell_type": "markdown", + "id": "de202a80-5b43-45c5-bd4f-13ff3f3dc89a", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "通过提供包含数值的Python列表(或嵌套列表),来为所需张量中的每个元素赋予确定值" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "id": "e0ea9924-9165-4c67-9b76-a2bd10ea5127", + "metadata": { + "origin_pos": 37, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[2, 1, 4, 3],\n", + " [1, 2, 3, 4],\n", + " [4, 3, 2, 1]])" + ] + }, + "execution_count": 103, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.tensor([[2, 1, 4, 3], [1, 2, 3, 4], [4, 3, 2, 1]])" + ] + }, + { + "cell_type": "markdown", + "id": "770bc9b0-79a3-479b-b862-304376c754ae", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "常见的标准算术运算符(`+`、`-`、`*`、`/`和`**`)都可以被升级为按元素运算" + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "id": "f77581a2-5e47-4f8a-bf07-5371ae54a7b5", + "metadata": { + "origin_pos": 41, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([ 3., 4., 6., 10.]),\n", + " tensor([-1., 0., 2., 6.]),\n", + " tensor([ 2., 4., 8., 16.]),\n", + " tensor([0.5000, 1.0000, 2.0000, 4.0000]),\n", + " tensor([ 1., 4., 16., 64.]))" + ] + }, + "execution_count": 104, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = torch.tensor([1.0, 2, 4, 8])\n", + "y = torch.tensor([2, 2, 2, 2])\n", + "x + y, x - y, x * y, x / y, x ** y" + ] + }, + { + "cell_type": "markdown", + "id": "c40e3586-87a1-4608-a50b-7a4cb3874e57", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "“按元素”方式可以应用更多的计算" + ] + }, + { + "cell_type": "code", + "execution_count": 105, + "id": "5b38f2a0-73ce-4318-9306-d359ce3ffa7f", + "metadata": { + "origin_pos": 45, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([2.7183e+00, 7.3891e+00, 5.4598e+01, 2.9810e+03])" + ] + }, + "execution_count": 105, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.exp(x)" + ] + }, + { + "cell_type": "markdown", + "id": "e08e33e3-008b-439d-9faf-fc86475eb1c4", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "我们也可以把多个张量*连结*(concatenate)在一起" + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "id": "e3cd445d-03a1-4143-84af-dcbe10af8612", + "metadata": { + "origin_pos": 49, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([[ 0., 1., 2., 3.],\n", + " [ 4., 5., 6., 7.],\n", + " [ 8., 9., 10., 11.],\n", + " [ 2., 1., 4., 3.],\n", + " [ 1., 2., 3., 4.],\n", + " [ 4., 3., 2., 1.]]),\n", + " tensor([[ 0., 1., 2., 3., 2., 1., 4., 3.],\n", + " [ 4., 5., 6., 7., 1., 2., 3., 4.],\n", + " [ 8., 9., 10., 11., 4., 3., 2., 1.]]))" + ] + }, + "execution_count": 106, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.arange(12, dtype=torch.float32).reshape((3,4))\n", + "Y = torch.tensor(\n", + " [[2.0, 1, 4, 3], [1, 2, 3, 4], [4, 3, 2, 1]])\n", + "torch.cat((X, Y), dim=0), torch.cat((X, Y), dim=1)" + ] + }, + { + "cell_type": "markdown", + "id": "10a03ef1-696f-4e85-ac98-beda0866d211", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "通过*逻辑运算符*构建二元张量" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "id": "3dea4fba-f0ec-4e46-8a42-903e82814e03", + "metadata": { + "origin_pos": 52, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[False, True, False, True],\n", + " [False, False, False, False],\n", + " [False, False, False, False]])" + ] + }, + "execution_count": 107, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X == Y" + ] + }, + { + "cell_type": "markdown", + "id": "a1d0b36e-7dc6-4c37-a2c7-910e54e19b88", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "对张量中的所有元素进行求和,会产生一个单元素张量" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "id": "c46d5a6f-101c-4645-9040-4a69796eefb7", + "metadata": { + "origin_pos": 54, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(66.)" + ] + }, + "execution_count": 108, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X.sum()" + ] + }, + { + "cell_type": "markdown", + "id": "553ec6c4-7890-4f41-834f-b096d0d54d53", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "即使形状不同,我们仍然可以通过调用\n", + "*广播机制*(broadcasting mechanism)来执行按元素操作" + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "id": "e153c79d-3787-4a77-9ffe-281f12f8d9df", + "metadata": { + "origin_pos": 58, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([[0],\n", + " [1],\n", + " [2]]),\n", + " tensor([[0, 1]]))" + ] + }, + "execution_count": 109, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = torch.arange(3).reshape((3, 1))\n", + "b = torch.arange(2).reshape((1, 2))\n", + "a, b" + ] + }, + { + "cell_type": "code", + "execution_count": 110, + "id": "83ab5c84-7920-427d-b175-77d8fec956f1", + "metadata": { + "origin_pos": 61, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0, 1],\n", + " [1, 2],\n", + " [2, 3]])" + ] + }, + "execution_count": 110, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a + b" + ] + }, + { + "cell_type": "markdown", + "id": "1d9b5a62-173f-4c23-b8df-181968dd84ab", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "可以用`[-1]`选择最后一个元素,可以用`[1:3]`选择第二个和第三个元素" + ] + }, + { + "cell_type": "code", + "execution_count": 111, + "id": "d64d3449-137f-41d9-9d1b-5d08f917c876", + "metadata": { + "origin_pos": 63, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([ 8., 9., 10., 11.]),\n", + " tensor([[ 4., 5., 6., 7.],\n", + " [ 8., 9., 10., 11.]]))" + ] + }, + "execution_count": 111, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X[-1], X[1:3]" + ] + }, + { + "cell_type": "markdown", + "id": "2105d701-9c03-4110-94d6-deb5f9462af4", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "除读取外,我们还可以通过指定索引来将元素写入矩阵" + ] + }, + { + "cell_type": "code", + "execution_count": 112, + "id": "685988cd-f1aa-4a45-beec-55aa248e36d6", + "metadata": { + "origin_pos": 66, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 0., 1., 2., 3.],\n", + " [ 4., 5., 9., 7.],\n", + " [ 8., 9., 10., 11.]])" + ] + }, + "execution_count": 112, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X[1, 2] = 9\n", + "X" + ] + }, + { + "cell_type": "markdown", + "id": "a49aaed7-4dd8-4431-8fe3-7caaddf0cc58", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "为多个元素赋值相同的值,我们只需要索引所有元素,然后为它们赋值" + ] + }, + { + "cell_type": "code", + "execution_count": 113, + "id": "cd8257b2-02c8-4883-bc5c-c5266bb90be1", + "metadata": { + "origin_pos": 69, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[12., 12., 12., 12.],\n", + " [12., 12., 12., 12.],\n", + " [ 8., 9., 10., 11.]])" + ] + }, + "execution_count": 113, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X[0:2, :] = 12\n", + "X" + ] + }, + { + "cell_type": "markdown", + "id": "933dd1bd-77cd-4461-a0e7-6adfc64bcc77", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "运行一些操作可能会导致为新结果分配内存" + ] + }, + { + "cell_type": "code", + "execution_count": 114, + "id": "b96e0f82-ff89-4032-b1b0-bf2da61f9445", + "metadata": { + "origin_pos": 72, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 114, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "before = id(Y)\n", + "Y = Y + X\n", + "id(Y) == before" + ] + }, + { + "cell_type": "markdown", + "id": "cdb325dc-833f-4999-a1ca-6b01b62d0d33", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "执行原地操作" + ] + }, + { + "cell_type": "code", + "execution_count": 115, + "id": "f2cf3cbf-6a17-4c16-82dc-d1bbb1f18c10", + "metadata": { + "origin_pos": 77, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "id(Z): 140424225140144\n", + "id(Z): 140424225140144\n" + ] + } + ], + "source": [ + "Z = torch.zeros_like(Y)\n", + "print('id(Z):', id(Z))\n", + "Z[:] = X + Y\n", + "print('id(Z):', id(Z))" + ] + }, + { + "cell_type": "markdown", + "id": "e1879c6c-338c-4207-a9eb-86a5dc7d6c7e", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "如果在后续计算中没有重复使用`X`,\n", + "我们也可以使用`X[:] = X + Y`或`X += Y`来减少操作的内存开销" + ] + }, + { + "cell_type": "code", + "execution_count": 116, + "id": "d9898688-95b8-4397-96cf-f6f2c59ff828", + "metadata": { + "origin_pos": 81, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 116, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "before = id(X)\n", + "X += Y\n", + "id(X) == before" + ] + }, + { + "cell_type": "markdown", + "id": "0d0d06a1-04ea-4979-952e-fb5b23a0db34", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "转换为NumPy张量(`ndarray`)" + ] + }, + { + "cell_type": "code", + "execution_count": 117, + "id": "66091d08-8118-49ff-9950-e38322c494e0", + "metadata": { + "origin_pos": 87, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(numpy.ndarray, torch.Tensor)" + ] + }, + "execution_count": 117, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A = X.numpy()\n", + "B = torch.tensor(A)\n", + "type(A), type(B)" + ] + }, + { + "cell_type": "markdown", + "id": "0d7b1448-f79b-4b14-a5c5-1996fdcb6a0d", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "将大小为1的张量转换为Python标量" + ] + }, + { + "cell_type": "code", + "execution_count": 118, + "id": "21495ba4-a629-4a4c-ab74-a96e7185e649", + "metadata": { + "origin_pos": 91, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([3.5000]), 3.5, 3.5, 3)" + ] + }, + "execution_count": 118, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = torch.tensor([3.5])\n", + "a, a.item(), float(a), int(a)" + ] + }, + { + "cell_type": "markdown", + "id": "8e5412a5-ea69-4f6c-b2b2-ddc920f54259", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 数据预处理\n", + "\n", + "首选创建一个人工数据集,并存储在CSV(逗号分隔值)文件" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "e4b52867-b4f3-46db-92a6-64b92efc2074", + "metadata": { + "origin_pos": 1, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.makedirs(os.path.join('..', 'data'), exist_ok=True)\n", + "data_file = os.path.join('..', 'data', 'house_tiny.csv')\n", + "with open(data_file, 'w') as f:\n", + " f.write('NumRooms,Alley,Price\\n')\n", + " f.write('NA,Pave,127500\\n')\n", + " f.write('2,NA,106000\\n')\n", + " f.write('4,NA,178100\\n')\n", + " f.write('NA,NA,140000\\n')" + ] + }, + { + "cell_type": "markdown", + "id": "0f5a8d30-86f9-490f-b907-ac86447750d8", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "从创建的CSV文件中加载原始数据集" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "7aecf9e4-b604-415f-a5bb-83414e1ccc6a", + "metadata": { + "origin_pos": 3, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " NumRooms Alley Price\n", + "0 NaN Pave 127500\n", + "1 2.0 NaN 106000\n", + "2 4.0 NaN 178100\n", + "3 NaN NaN 140000\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "data = pd.read_csv(data_file)\n", + "print(data)" + ] + }, + { + "cell_type": "markdown", + "id": "b79dde6b-c078-4ec1-a9c3-5f1cedd07025", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "为了处理缺失的数据,典型的方法包括*插值法*和*删除法*,\n", + "这里,我们将考虑插值法" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "317fa349-cad0-4cc9-9315-85d2c8a22fa0", + "metadata": { + "origin_pos": 5, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " NumRooms Alley\n", + "0 3.0 Pave\n", + "1 2.0 NaN\n", + "2 4.0 NaN\n", + "3 3.0 NaN\n" + ] + } + ], + "source": [ + "inputs, outputs = data.iloc[:, 0:2], data.iloc[:, 2]\n", + "# 用同一列的均值替换“NaN”项\n", + "inputs['NumRooms'] = inputs['NumRooms'].fillna(inputs['NumRooms'].mean())\n", + "print(inputs)" + ] + }, + { + "cell_type": "markdown", + "id": "43d3dae7-20bc-4955-a323-ec40e087b927", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "对于`inputs`中的类别值或离散值,我们将“NaN”视为一个类别" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f23dd19-079e-4514-9edc-e7167813fd78", + "metadata": { + "origin_pos": 7, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " NumRooms Alley_Pave Alley_nan\n", + "0 3.0 1 0\n", + "1 2.0 0 1\n", + "2 4.0 0 1\n", + "3 3.0 0 1\n" + ] + } + ], + "source": [ + "# get_dummies 是 pandas 实现one hot encode的方式。\n", + "inputs = pd.get_dummies(inputs, dummy_na=True).astype('float32')\n", + "print(inputs)" + ] + }, + { + "cell_type": "markdown", + "id": "f5a23184-6c3d-492b-94ce-4872167d8d6e", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "现在`inputs`和`outputs`中的所有条目都是数值类型,它们可以转换为张量格式" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "f5bda8b1-62c2-4518-b7ba-575b668af195", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([[3., 1., 0.],\n", + " [2., 0., 1.],\n", + " [4., 0., 1.],\n", + " [3., 0., 1.]], dtype=torch.float64),\n", + " tensor([127500, 106000, 178100, 140000]))" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "\n", + "X, y = torch.tensor(inputs.values), torch.tensor(outputs.values)\n", + "X, y" + ] + }, + { + "cell_type": "markdown", + "id": "31feece7-041b-41b9-9cf7-f36a8bd5b614", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 查阅文档" + ] + }, + { + "cell_type": "markdown", + "id": "7ce98e3a-903e-4215-ab19-3376d4f79ecb", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "查询随机数生成模块中的所有属性" + ] + }, + { + "cell_type": "code", + "execution_count": 124, + "id": "0645a1ad-92fc-4ed0-a97d-3d44b9dabc5e", + "metadata": { + "attributes": { + "classes": [], + "id": "", + "n": "1" + }, + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['AbsTransform', 'AffineTransform', 'Bernoulli', 'Beta', 'Binomial', 'CatTransform', 'Categorical', 'Cauchy', 'Chi2', 'ComposeTransform', 'ContinuousBernoulli', 'CorrCholeskyTransform', 'CumulativeDistributionTransform', 'Dirichlet', 'Distribution', 'ExpTransform', 'Exponential', 'ExponentialFamily', 'FisherSnedecor', 'Gamma', 'Geometric', 'Gumbel', 'HalfCauchy', 'HalfNormal', 'Independent', 'IndependentTransform', 'Kumaraswamy', 'LKJCholesky', 'Laplace', 'LogNormal', 'LogisticNormal', 'LowRankMultivariateNormal', 'LowerCholeskyTransform', 'MixtureSameFamily', 'Multinomial', 'MultivariateNormal', 'NegativeBinomial', 'Normal', 'OneHotCategorical', 'OneHotCategoricalStraightThrough', 'Pareto', 'Poisson', 'PowerTransform', 'RelaxedBernoulli', 'RelaxedOneHotCategorical', 'ReshapeTransform', 'SigmoidTransform', 'SoftmaxTransform', 'SoftplusTransform', 'StackTransform', 'StickBreakingTransform', 'StudentT', 'TanhTransform', 'Transform', 'TransformedDistribution', 'Uniform', 'VonMises', 'Weibull', 'Wishart', '__all__', '__builtins__', '__cached__', '__doc__', '__file__', '__loader__', '__name__', '__package__', '__path__', '__spec__', 'bernoulli', 'beta', 'biject_to', 'binomial', 'categorical', 'cauchy', 'chi2', 'constraint_registry', 'constraints', 'continuous_bernoulli', 'dirichlet', 'distribution', 'exp_family', 'exponential', 'fishersnedecor', 'gamma', 'geometric', 'gumbel', 'half_cauchy', 'half_normal', 'identity_transform', 'independent', 'kl', 'kl_divergence', 'kumaraswamy', 'laplace', 'lkj_cholesky', 'log_normal', 'logistic_normal', 'lowrank_multivariate_normal', 'mixture_same_family', 'multinomial', 'multivariate_normal', 'negative_binomial', 'normal', 'one_hot_categorical', 'pareto', 'poisson', 'register_kl', 'relaxed_bernoulli', 'relaxed_categorical', 'studentT', 'transform_to', 'transformed_distribution', 'transforms', 'uniform', 'utils', 'von_mises', 'weibull', 'wishart']\n" + ] + } + ], + "source": [ + "import torch\n", + "\n", + "print(dir(torch.distributions))" + ] + }, + { + "cell_type": "markdown", + "id": "02436ac3-7d34-4530-9870-4f8fc6ce9b57", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "查看张量`ones`函数的用法" + ] + }, + { + "cell_type": "code", + "execution_count": 125, + "id": "15555ccb-2632-4944-9640-cb699be1985a", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on built-in function ones in module torch:\n", + "\n", + "ones(...)\n", + " ones(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor\n", + " \n", + " Returns a tensor filled with the scalar value `1`, with the shape defined\n", + " by the variable argument :attr:`size`.\n", + " \n", + " Args:\n", + " size (int...): a sequence of integers defining the shape of the output tensor.\n", + " Can be a variable number of arguments or a collection like a list or tuple.\n", + " \n", + " Keyword arguments:\n", + " out (Tensor, optional): the output tensor.\n", + " dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor.\n", + " Default: if ``None``, uses a global default (see :func:`torch.set_default_tensor_type`).\n", + " layout (:class:`torch.layout`, optional): the desired layout of returned Tensor.\n", + " Default: ``torch.strided``.\n", + " device (:class:`torch.device`, optional): the desired device of returned tensor.\n", + " Default: if ``None``, uses the current device for the default tensor type\n", + " (see :func:`torch.set_default_tensor_type`). :attr:`device` will be the CPU\n", + " for CPU tensor types and the current CUDA device for CUDA tensor types.\n", + " requires_grad (bool, optional): If autograd should record operations on the\n", + " returned tensor. Default: ``False``.\n", + " \n", + " Example::\n", + " \n", + " >>> torch.ones(2, 3)\n", + " tensor([[ 1., 1., 1.],\n", + " [ 1., 1., 1.]])\n", + " \n", + " >>> torch.ones(5)\n", + " tensor([ 1., 1., 1., 1., 1.])\n", + "\n" + ] + } + ], + "source": [ + "help(torch.ones)" + ] + }, + { + "cell_type": "markdown", + "id": "2bd40831-b4ee-49c1-8184-52254f7e8f46", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "运行一个快速测试" + ] + }, + { + "cell_type": "code", + "execution_count": 126, + "id": "6a89c5b8-4e64-41b8-b01d-6051364ccc6f", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([1., 1., 1., 1.])" + ] + }, + "execution_count": 126, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.ones(4)" + ] + }, + { + "cell_type": "markdown", + "id": "dc1d218c-74fb-43f5-8881-7722ca7e8966", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 线性代数\n", + "\n", + "标量由只有一个元素的张量表示" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "cd2c4055-b850-49c0-a143-f62bd7e3bcd9", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor(5.), tensor(6.), tensor(1.5000), tensor(9.))" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "\n", + "x = torch.tensor(3.0)\n", + "y = torch.tensor(2.0)\n", + "\n", + "x + y, x * y, x / y, x**y" + ] + }, + { + "cell_type": "markdown", + "id": "b73e9dd4-ec21-4a0e-8af0-b7852469586f", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "你可以将向量视为标量值组成的列表" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "169ea89c-53cf-4fa6-b236-cda1804d625f", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0, 1, 2, 3])" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = torch.arange(4)\n", + "x" + ] + }, + { + "cell_type": "markdown", + "id": "3c18f409-b2b5-4738-b4b5-711aa3b6e818", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "通过张量的索引来访问任一元素" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5f10955c-3598-40a0-9072-0bd1d87a9cf6", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(3)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x[3]" + ] + }, + { + "cell_type": "markdown", + "id": "2c7e94dd-4b55-43bd-a171-1a51e78ed383", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "访问张量的长度" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "98ec7ee3-391a-473b-832a-1abf3a22fd24", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "4" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(x)" + ] + }, + { + "cell_type": "markdown", + "id": "a8b9cf99-a1ca-4594-ab14-96cafa4913b3", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "只有一个轴的张量,形状只有一个元素" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d5b671d0-a3c8-4712-a998-3d8f8a824dfb", + "metadata": { + "origin_pos": 18, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([4])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x.shape" + ] + }, + { + "cell_type": "markdown", + "id": "5cec9658-5806-4b33-83ef-e1d65cb65698", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "通过指定两个分量$m$和$n$来创建一个形状为$m \\times n$的矩阵" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "eab50756-2d26-4ae3-9af8-cb4a9ba510cc", + "metadata": { + "origin_pos": 22, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 0, 1, 2, 3],\n", + " [ 4, 5, 6, 7],\n", + " [ 8, 9, 10, 11],\n", + " [12, 13, 14, 15],\n", + " [16, 17, 18, 19]])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A = torch.arange(20).reshape(5, 4)\n", + "A" + ] + }, + { + "cell_type": "markdown", + "id": "96ad2a88-6be4-4813-8784-c46394e573ea", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "矩阵的转置" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "a89cb776-5542-486a-85d4-5d8bb023f583", + "metadata": { + "origin_pos": 26, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 0, 4, 8, 12, 16],\n", + " [ 1, 5, 9, 13, 17],\n", + " [ 2, 6, 10, 14, 18],\n", + " [ 3, 7, 11, 15, 19]])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A.T" + ] + }, + { + "cell_type": "markdown", + "id": "89520f25-eddb-4c50-8698-a9fb2f8a9ae0", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "*对称矩阵*(symmetric matrix)$\\mathbf{A}$等于其转置:$\\mathbf{A} = \\mathbf{A}^\\top$" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a259368d-ef93-40e3-98a9-cea2ca08170c", + "metadata": { + "origin_pos": 30, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[1, 2, 3],\n", + " [2, 0, 4],\n", + " [3, 4, 5]])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "B = torch.tensor([[1, 2, 3], [2, 0, 4], [3, 4, 5]])\n", + "B" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "99671df2-5907-40ac-a2dc-ddbcae7fb15b", + "metadata": { + "origin_pos": 34, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[True, True, True],\n", + " [True, True, True],\n", + " [True, True, True]])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "B == B.T" + ] + }, + { + "cell_type": "markdown", + "id": "77eaa5c0-d699-4c71-96b6-341ff9a2f786", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "就像向量是标量的推广,矩阵是向量的推广一样,我们可以构建具有更多轴的数据结构" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "553c27c6-7879-474d-b629-579cbff9a51d", + "metadata": { + "origin_pos": 38, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[ 0, 1, 2, 3],\n", + " [ 4, 5, 6, 7],\n", + " [ 8, 9, 10, 11]],\n", + "\n", + " [[12, 13, 14, 15],\n", + " [16, 17, 18, 19],\n", + " [20, 21, 22, 23]]])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.arange(24).reshape(2, 3, 4)\n", + "X" + ] + }, + { + "cell_type": "markdown", + "id": "ed485a8f-84fd-4b04-b919-d337d4f0e7b3", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "给定具有相同形状的任意两个张量,任何按元素二元运算的结果都将是相同形状的张量" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "ab355da7-80f1-4873-9861-69a06fec9ff4", + "metadata": { + "origin_pos": 42, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([[ 0., 1., 2., 3.],\n", + " [ 4., 5., 6., 7.],\n", + " [ 8., 9., 10., 11.],\n", + " [12., 13., 14., 15.],\n", + " [16., 17., 18., 19.]]),\n", + " tensor([[ 0., 2., 4., 6.],\n", + " [ 8., 10., 12., 14.],\n", + " [16., 18., 20., 22.],\n", + " [24., 26., 28., 30.],\n", + " [32., 34., 36., 38.]]))" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A = torch.arange(20, dtype=torch.float32).reshape(5, 4)\n", + "B = A.clone()\n", + "A, A + B" + ] + }, + { + "cell_type": "markdown", + "id": "2bafc57a-4995-4499-bcee-b26fb036081a", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "两个矩阵的按元素乘法称为*Hadamard积*(Hadamard product)(数学符号$\\odot$)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "81a6e1d8-fa9f-4652-8adc-5a2a0ff139ea", + "metadata": { + "origin_pos": 46, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 0., 1., 4., 9.],\n", + " [ 16., 25., 36., 49.],\n", + " [ 64., 81., 100., 121.],\n", + " [144., 169., 196., 225.],\n", + " [256., 289., 324., 361.]])" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A * B" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "dfb1beb4-70c2-414f-b3e7-9b174610711d", + "metadata": { + "origin_pos": 50, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([[[ 2, 3, 4, 5],\n", + " [ 6, 7, 8, 9],\n", + " [10, 11, 12, 13]],\n", + " \n", + " [[14, 15, 16, 17],\n", + " [18, 19, 20, 21],\n", + " [22, 23, 24, 25]]]),\n", + " torch.Size([2, 3, 4]))" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = 2\n", + "X = torch.arange(24).reshape(2, 3, 4)\n", + "a + X, (a * X).shape" + ] + }, + { + "cell_type": "markdown", + "id": "c64a6a97-a816-4c59-b822-0335764d4542", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "计算其元素的和" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "f94739c3-2d30-4c9b-9187-3bfa32e0cf59", + "metadata": { + "origin_pos": 54, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([0., 1., 2., 3.]), tensor(6.))" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = torch.arange(4, dtype=torch.float32)\n", + "x, x.sum()" + ] + }, + { + "cell_type": "markdown", + "id": "6c275f66-ded8-400e-8584-664ee1d383de", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "表示任意形状张量的元素和" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "c1251c8a-e9de-49b5-aad3-fa3d249272b3", + "metadata": { + "origin_pos": 58, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(torch.Size([5, 4]), tensor(190.))" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A.shape, A.sum()" + ] + }, + { + "cell_type": "markdown", + "id": "440e1d56-71c3-4e40-a1e9-e75156ccc879", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "指定张量沿哪一个轴来通过求和降低维度" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "7d853e52-6c4b-4859-b795-04a7ec943bd6", + "metadata": { + "origin_pos": 62, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([40., 45., 50., 55.]), torch.Size([4]))" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A_sum_axis0 = A.sum(axis=0)\n", + "A_sum_axis0, A_sum_axis0.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "b25421d8-1f82-4599-829e-5395d8a3874e", + "metadata": { + "origin_pos": 66, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([ 6., 22., 38., 54., 70.]), torch.Size([5]))" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A_sum_axis1 = A.sum(axis=1)\n", + "A_sum_axis1, A_sum_axis1.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "d165e4b0-2e2d-4c8e-9023-20cf64b6cd16", + "metadata": { + "origin_pos": 70, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(190.)" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A.sum(axis=[0, 1])" + ] + }, + { + "cell_type": "markdown", + "id": "c9e64a9b-0e01-4c69-9861-70ec4dffe99e", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "一个与求和相关的量是*平均值*(mean或average)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "8a336685-119e-42c5-8cc2-672635751655", + "metadata": { + "origin_pos": 74, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor(9.5000), tensor(9.5000))" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A.mean(), A.sum() / A.numel()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "393ba1e4-585a-481a-8cd1-875e2f810ef6", + "metadata": { + "origin_pos": 78, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([ 8., 9., 10., 11.]), tensor([ 8., 9., 10., 11.]))" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A.mean(axis=0), A.sum(axis=0) / A.shape[0]" + ] + }, + { + "cell_type": "markdown", + "id": "970d1362-391d-4415-bfc3-d0a77c19e44d", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "计算总和或均值时保持轴数不变" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "dde63412-60a3-4836-ae9d-7704f12da74d", + "metadata": { + "origin_pos": 82, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 6.],\n", + " [22.],\n", + " [38.],\n", + " [54.],\n", + " [70.]])" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum_A = A.sum(axis=1, keepdims=True)\n", + "sum_A" + ] + }, + { + "cell_type": "markdown", + "id": "fb401f55-958f-442e-8a61-c5ecd9beadba", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "通过广播将`A`除以`sum_A`" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "fac69c9b-be07-4e92-9c5c-7b9625f12bc1", + "metadata": { + "origin_pos": 86, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0000, 0.1667, 0.3333, 0.5000],\n", + " [0.1818, 0.2273, 0.2727, 0.3182],\n", + " [0.2105, 0.2368, 0.2632, 0.2895],\n", + " [0.2222, 0.2407, 0.2593, 0.2778],\n", + " [0.2286, 0.2429, 0.2571, 0.2714]])" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A / sum_A" + ] + }, + { + "cell_type": "markdown", + "id": "767cda68-07c3-4ba9-82c5-7f62b1cf136c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "某个轴计算`A`元素的累积总和" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "81e39538-8f7f-4298-b7e7-64497f3a6bac", + "metadata": { + "origin_pos": 90, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 0., 1., 2., 3.],\n", + " [ 4., 6., 8., 10.],\n", + " [12., 15., 18., 21.],\n", + " [24., 28., 32., 36.],\n", + " [40., 45., 50., 55.]])" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A.cumsum(axis=0)" + ] + }, + { + "cell_type": "markdown", + "id": "e5b9654a-7e34-410c-86aa-96b1c5614bb8", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "点积是相同位置的按元素乘积的和" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "4619a1d3-de75-4752-8812-32e6fac520cf", + "metadata": { + "origin_pos": 94, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([0., 1., 2., 3.]), tensor([1., 1., 1., 1.]), tensor(6.))" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "y = torch.ones(4, dtype = torch.float32)\n", + "x, y, torch.dot(x, y)" + ] + }, + { + "cell_type": "markdown", + "id": "c4fb029c-aa94-429d-a589-43092d7feb85", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "我们可以通过执行按元素乘法,然后进行求和来表示两个向量的点积" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "c077fa90-787e-481f-b1ca-24996d86ab2a", + "metadata": { + "origin_pos": 98, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(6.)" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.sum(x * y)" + ] + }, + { + "cell_type": "markdown", + "id": "d311339c-f0bb-47b7-b42b-6d73fceb085c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "矩阵向量积$\\mathbf{A}\\mathbf{x}$是一个长度为$m$的列向量,\n", + "其第$i$个元素是点积$\\mathbf{a}^\\top_i \\mathbf{x}$" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "d8d3d075-5b31-4d6c-8b9f-e543742b44ab", + "metadata": { + "origin_pos": 105, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(torch.Size([5, 4]), torch.Size([4]), tensor([ 14., 38., 62., 86., 110.]))" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "A.shape, x.shape, torch.mv(A, x)" + ] + }, + { + "cell_type": "markdown", + "id": "207c6561-32ac-4af4-bb9e-b46dc9a64f42", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "我们可以将矩阵-矩阵乘法$\\mathbf{AB}$看作是简单地执行$m$次矩阵-向量积,并将结果拼接在一起,形成一个$n \\times m$矩阵" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "284ba1a4-57aa-48d8-aa85-2500c08c25ac", + "metadata": { + "origin_pos": 109, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 6., 6., 6.],\n", + " [22., 22., 22.],\n", + " [38., 38., 38.],\n", + " [54., 54., 54.],\n", + " [70., 70., 70.]])" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "B = torch.ones(4, 3)\n", + "torch.mm(A, B)" + ] + }, + { + "cell_type": "markdown", + "id": "3d9c2b99-2e75-48a6-b466-e9ba55feac2e", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "$L_2$*范数*是向量元素平方和的平方根:\n", + "$$\\|\\mathbf{x}\\|_2 = \\sqrt{\\sum_{i=1}^n x_i^2}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "e0f0052d-5bad-4846-9d8c-34f7a242cc11", + "metadata": { + "origin_pos": 113, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(5.)" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "u = torch.tensor([3.0, -4.0])\n", + "torch.norm(u)" + ] + }, + { + "cell_type": "markdown", + "id": "498deac2-e4e8-4713-8a87-bb31f93b59e4", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "$L_1$范数,它表示为向量元素的绝对值之和:\n", + "$$\\|\\mathbf{x}\\|_1 = \\sum_{i=1}^n \\left|x_i \\right|$$" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "fe719ad6-ad4e-4485-9d92-af4781facd3c", + "metadata": { + "origin_pos": 117, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(7.)" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.abs(u).sum()" + ] + }, + { + "cell_type": "markdown", + "id": "8634d5e6-8428-4abd-b989-73f9d6108dc7", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "矩阵\n", + "的*Frobenius范数*(Frobenius norm)是矩阵元素平方和的平方根:\n", + "$$\\|\\mathbf{X}\\|_F = \\sqrt{\\sum_{i=1}^m \\sum_{j=1}^n x_{ij}^2}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "ccca6045-62c6-42ec-b760-4413065bf1be", + "metadata": { + "origin_pos": 121, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(6.)" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.norm(torch.ones((4, 9)))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/02_decision_tree/02_decision_tree.ipynb b/02_decision_tree/02_decision_tree.ipynb new file mode 100644 index 0000000..d739405 --- /dev/null +++ b/02_decision_tree/02_decision_tree.ipynb @@ -0,0 +1,687 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "73b96d3d-406a-405d-850b-38dd7772624e", + "metadata": {}, + "source": [ + "# 决策树\n", + "\n", + "## 目录\n", + "\n", + "- 决策树学习器" + ] + }, + { + "cell_type": "markdown", + "id": "72a368ee-02cc-4d92-8d0f-d4323d31d12c", + "metadata": { + "tags": [] + }, + "source": [ + "## 决策树学习器\n", + "\n", + "### 概述\n", + "\n", + "#### 决策树\n", + "\n", + "决策树是一个流程图,它使用决策树及其可能的后果进行分类。在树的每个非叶子节点上,输入的一个属性被测试,根据这个测试结果,选择通往子节点的相应分支。在叶子节点上,根据这个叶子节点的类别标签,对输入进行分类。从根到叶的路径代表分类规则,根据这些规则给叶节点分配类标签。\n", + "\n", + "![decision tree](decisiontree_fruit.jpg)\n", + "\n", + "#### 决策树学习\n", + "\n", + "决策树学习是指从有类标记的训练数据中构建决策树。数据预计是一个元组,其中元组的每个记录都是用于分类的属性。决策树是自上而下构建的,通过在每一步选择一个变量来最好地分割项目集。有不同的指标来衡量 \"最佳分割\"。这些指标通常衡量子集内目标变量的同质性。\n", + "\n", + "#### 信息增益\n", + "\n", + "信息增益是基于信息理论中的熵的概念。熵的定义为:\n", + "\n", + "$$H(p) = -\\sum{p_i \\log_2{p_i}}$$\n", + "\n", + "信息增益是指父代的熵和子代的熵的加权和之间的差异。用于分割的特征是提供最大信息增益的特征。\n", + "\n", + "#### 伪代码" + ] + }, + { + "cell_type": "markdown", + "id": "1744317d-b45d-440c-b3ed-77ab64b6e10e", + "metadata": {}, + "source": [ + "__function__ DECISION-TREE-LEARNING(_examples_, _attributes_, _parent\\_examples_) __returns__ a tree \n", + " __if__ _examples_ 是空集 __then return__ PLURALITY\\-VALUE(_parent\\_examples_) \n", + " __else if__ _examples_ 分类结果都相同 __then return__ 分类结果 \n", + " __else if__ _attributes_ 是空集 __then return__ PLURALITY\\-VALUE(_examples_) \n", + " __else__ \n", + "   _A_ ← argmax_a_ ∈ _attributes_ IMPORTANCE(_a_, _examples_) \n", + "   _tree_ ← 以特征 _A_ 为根检测节点的决策树 \n", + "   __for each__ 特征 _A_ 的取值 _vk_ __do__ \n", + "     _exs_ ← \\{ _e_ : _e_ ∈ _examples_ __and__ _e_._A_ = _vk_ \\} \n", + "     _subtree_ ← DECISION-TREE-LEARNING(_exs_, _attributes_ − _A_, _examples_) \n", + "     将标签为 \\(_A_ = _vk_\\) 的子树 _subtree_ 添加为 _tree_ 的分支 \n", + "   __return__ _tree_ " + ] + }, + { + "cell_type": "markdown", + "id": "f8565216-f7c5-4b7f-a791-ebf538a6aaab", + "metadata": {}, + "source": [ + "### 实现\n", + "\n", + "由我们的学习算法构建的树的节点,根据它们是内部节点还是叶节点,分别使用`DecisionFork`或`DecisionLeaf`来存储。" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "13283545-a69a-4d96-ab5d-157eaa395f3a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.insert(1, '../')\n", + "from utils.utils import *\n", + "\n", + "from utils.dataset4learners import *\n", + "from DecisionTreeLearner_3 import *" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c3e70aae-3ef5-4e41-b39d-bc0f0b57270a", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "

\n", + "\n", + "
class DecisionFork:\n",
+       "    """\n",
+       "    A fork of a decision tree holds an attribute to test, and a dict\n",
+       "    of branches, one for each of the attribute's values.\n",
+       "    """\n",
+       "\n",
+       "    def __init__(self, attr, attr_name=None, default_child=None, branches=None):\n",
+       "        """Initialize by saying what attribute this node tests."""\n",
+       "        self.attr = attr\n",
+       "        self.attr_name = attr_name or attr\n",
+       "        self.default_child = default_child\n",
+       "        self.branches = branches or {}\n",
+       "\n",
+       "    def __call__(self, example):\n",
+       "        """Given an example, classify it using the attribute and the branches."""\n",
+       "        attr_val = example[self.attr]\n",
+       "        if attr_val in self.branches:\n",
+       "            return self.branches[attr_val](example)\n",
+       "        else:\n",
+       "            # return default class when attribute is unknown\n",
+       "            return self.default_child(example)\n",
+       "\n",
+       "    def add(self, val, subtree):\n",
+       "        """Add a branch. If self.attr = val, go to the given subtree."""\n",
+       "        self.branches[val] = subtree\n",
+       "\n",
+       "    def display(self, indent=0):\n",
+       "        name = self.attr_name\n",
+       "        print('Test', name)\n",
+       "        for (val, subtree) in self.branches.items():\n",
+       "            print(' ' * 4 * indent, name, '=', val, '==>', end=' ')\n",
+       "            subtree.display(indent + 1)\n",
+       "\n",
+       "    def __repr__(self):\n",
+       "        return 'DecisionFork({0!r}, {1!r}, {2!r})'.format(self.attr, self.attr_name, self.branches)\n",
+       "
\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "psource(DecisionFork)" + ] + }, + { + "cell_type": "markdown", + "id": "0ef40288-b753-4155-a813-25153163044f", + "metadata": {}, + "source": [ + "`DecisionFork`持有属性,在该节点进行测试,以及一个分支的决定。分支存储了子节点,每个属性的值都有一个。以输入元组为参数,以函数形式调用这个类的对象,根据属性测试的结果返回分类路径中的下一个节点。" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "79120735-ed3a-4b36-9fb2-5b590de076d3", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "

\n", + "\n", + "
class DecisionLeaf:\n",
+       "    """A leaf of a decision tree holds just a result."""\n",
+       "\n",
+       "    def __init__(self, result):\n",
+       "        self.result = result\n",
+       "\n",
+       "    def __call__(self, example):\n",
+       "        return self.result\n",
+       "\n",
+       "    def display(self):\n",
+       "        print('RESULT =', self.result)\n",
+       "\n",
+       "    def __repr__(self):\n",
+       "        return repr(self.result)\n",
+       "
\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "psource(DecisionLeaf)" + ] + }, + { + "cell_type": "markdown", + "id": "5b2935a2-15ff-4fdc-b2b3-930b1539e278", + "metadata": {}, + "source": [ + "叶子节点在`result`中存储类别标签。所有输入图元的分类路径都在`DecisionLeaf`上结束,其`result` 属性决定了它们的类别。" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "f272550b-1eac-42fd-b937-29627380b403", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "

\n", + "\n", + "
class DecisionTreeLearner:\n",
+       "    """DecisionTreeLearner: based on information gain"""\n",
+       "\n",
+       "    def __init__(self, dataset):\n",
+       "        self.dataset = dataset\n",
+       "        self.tree = self.decision_tree_learning(dataset.examples, dataset.inputs)\n",
+       "\n",
+       "    def decision_tree_learning(self, examples, attrs, parent_examples=()):\n",
+       "        raise NotImplementedError\n",
+       "\n",
+       "    def plurality_value(self, examples):\n",
+       "        """\n",
+       "        Return the most popular target value for this set of examples.\n",
+       "        (If target is binary, this is the majority; otherwise plurality).\n",
+       "        """\n",
+       "        popular = argmax_random_tie(self.dataset.values[self.dataset.target],\n",
+       "                                    key=lambda v: self.count(self.dataset.target, v, examples))\n",
+       "        return DecisionLeaf(popular)\n",
+       "\n",
+       "    def count(self, attr, val, examples):\n",
+       "        """Count the number of examples that have example[attr] = val."""\n",
+       "        return sum(e[attr] == val for e in examples)\n",
+       "\n",
+       "    def all_same_class(self, examples):\n",
+       "        """Are all these examples in the same target class?"""\n",
+       "        class0 = examples[0][self.dataset.target]\n",
+       "        return all(e[self.dataset.target] == class0 for e in examples)\n",
+       "\n",
+       "    def choose_attribute(self, attrs, examples):\n",
+       "        """Choose the attribute with the highest information gain."""\n",
+       "        return argmax_random_tie(attrs, key=lambda a: self.information_gain(a, examples))\n",
+       "\n",
+       "    def information_gain(self, attr, examples):\n",
+       "        """Return the expected reduction in entropy from splitting by attr."""\n",
+       "        raise NotImplementedError\n",
+       "\n",
+       "    def split_by(self, attr, examples):\n",
+       "        """Return a list of (val, examples) pairs for each val of attr."""\n",
+       "        return [(v, [e for e in examples if e[attr] == v]) for v in self.dataset.values[attr]]\n",
+       "\n",
+       "    def predict(self, x):\n",
+       "        return self.tree(x)\n",
+       "\n",
+       "    def __call__(self, x):\n",
+       "        return self.predict(x)\n",
+       "
\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "psource(DecisionTreeLearner)" + ] + }, + { + "cell_type": "markdown", + "id": "d22fbe3c-ed37-4c10-83da-170805f822a0", + "metadata": {}, + "source": [ + "上面的实现使用信息增益作为衡量标准来选择测试哪一个属性进行拆分。该函数以递归的方式自上而下地构建树。根据输入,它做出四个选择中的一个。\n", + "\n", + "1. 如果当前步骤的输入没有训练数据,我们将返回在父步骤(上一级递归)中收到的输入数据的类别模式。\n", + "2. 如果训练数据中的所有值都属于同一类别,它将返回一个`DecisionLeaf`,其类别标签是所有数据所属的类别。\n", + "3. 如果数据没有可以测试的属性,我们就返回训练数据中具有最高复数值的类。\n", + "4. 我们选择熵值最高的属性,并返回一个基于此属性的`DecisionFork`。每个分支递归地调用`decision_tree_learning`来构建子树。\n" + ] + }, + { + "cell_type": "markdown", + "id": "b454d72b-421e-48e8-ab0b-aa5fd81e22fb", + "metadata": {}, + "source": [ + "### 实现要点\n", + "\n", + "```py\n", + "def information_content(values):\n", + " \"\"\"Number of bits to represent the probability distribution in values.\"\"\"\n", + " probabilities = values 的归一化数值\n", + " return probabilities 代入信息熵公式\n", + "\n", + "def information_gain(self, attr, examples):\n", + " \"\"\"Return the expected reduction in entropy from splitting by attr.\"\"\"\n", + "\n", + " def I(examples):\n", + " return information_content([self.count(self.dataset.target, v, examples)\n", + " for v in self.dataset.values[self.dataset.target]])\n", + "\n", + " n = 样本数\n", + " remainder = 剩余信息熵值\n", + " return 信息增益\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "f3227e10-e821-411e-b5b3-8442040b7e47", + "metadata": {}, + "source": [ + "### 例子\n", + "\n", + "现在我们将使用决策树学习器对一个有数值的样本进行分类:5.1, 3.0, 1.1, 0.1." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d0e89769-62d8-4057-b2cc-bc165dedefac", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "setosa\n" + ] + } + ], + "source": [ + "iris = DataSet(name=\"iris\")\n", + "DTL = DecisionTreeLearner(iris)\n", + "print(DTL([5.1, 3.0, 1.1, 0.1]))\n", + "#print(DTL.predict([5.1, 3.0, 1.1, 0.1]))" + ] + }, + { + "cell_type": "markdown", + "id": "c923f2fe-c16c-462c-8d5d-6702094955f4", + "metadata": {}, + "source": [ + "正如预期的那样,决策树学习器将样本归类为 \"setosa\"。" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ffc16d0d-f0c6-4f42-87b3-4f7004ce2c19", + "metadata": {}, + "outputs": [], + "source": [ + "assert DTL.predict([5, 3, 1, 0.1]) == 'setosa'\n", + "assert DTL.predict([6, 5, 3, 1.5]) == 'versicolor'\n", + "assert DTL.predict([7.5, 4, 6, 2]) == 'virginica'" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/02_decision_tree/DecisionTreeLearner_1.py b/02_decision_tree/DecisionTreeLearner_1.py new file mode 100644 index 0000000..32da42d --- /dev/null +++ b/02_decision_tree/DecisionTreeLearner_1.py @@ -0,0 +1,140 @@ +import sys +sys.path.insert(1, '../') +from utils.utils import * + + +class DecisionFork: + """ + A fork of a decision tree holds an attribute to test, and a dict + of branches, one for each of the attribute's values. + """ + + def __init__(self, attr, attr_name=None, default_child=None, branches=None): + """Initialize by saying what attribute this node tests.""" + self.attr = attr + self.attr_name = attr_name or attr + self.default_child = default_child + self.branches = branches or {} + + def __call__(self, example): + """Given an example, classify it using the attribute and the branches.""" + attr_val = example[self.attr] + if attr_val in self.branches: + return self.branches[attr_val](example) + else: + # return default class when attribute is unknown + return self.default_child(example) + + def add(self, val, subtree): + """Add a branch. If self.attr = val, go to the given subtree.""" + self.branches[val] = subtree + + def display(self, indent=0): + name = self.attr_name + print('Test', name) + for (val, subtree) in self.branches.items(): + print(' ' * 4 * indent, name, '=', val, '==>', end=' ') + subtree.display(indent + 1) + + def __repr__(self): + return 'DecisionFork({0!r}, {1!r}, {2!r})'.format(self.attr, self.attr_name, self.branches) + + +class DecisionLeaf: + """A leaf of a decision tree holds just a result.""" + + def __init__(self, result): + self.result = result + + def __call__(self, example): + return self.result + + def display(self): + print('RESULT =', self.result) + + def __repr__(self): + return repr(self.result) + + +class DecisionTreeLearner: + """DecisionTreeLearner: based on information gain""" + + def __init__(self, dataset): + self.dataset = dataset + self.tree = self.decision_tree_learning(dataset.examples, dataset.inputs) + + def decision_tree_learning(self, examples, attrs, parent_examples=()): + if len(examples) == 0: + return self.plurality_value(parent_examples) + if self.all_same_class(examples): + return DecisionLeaf(examples[0][self.dataset.target]) + if len(attrs) == 0: + return self.plurality_value(examples) + A = self.choose_attribute(attrs, examples) + tree = DecisionFork(A, self.dataset.attr_names[A], self.plurality_value(examples)) + for (v_k, exs) in self.split_by(A, examples): + subtree = self.decision_tree_learning(exs, remove_all(A, attrs), examples) + tree.add(v_k, subtree) + return tree + + def plurality_value(self, examples): + """ + Return the most popular target value for this set of examples. + (If target is binary, this is the majority; otherwise plurality). + """ + popular = argmax_random_tie(self.dataset.values[self.dataset.target], + key=lambda v: self.count(self.dataset.target, v, examples)) + return DecisionLeaf(popular) + + def count(self, attr, val, examples): + """Count the number of examples that have example[attr] = val.""" + return sum(e[attr] == val for e in examples) + + def all_same_class(self, examples): + """Are all these examples in the same target class?""" + class0 = examples[0][self.dataset.target] + return all(e[self.dataset.target] == class0 for e in examples) + + def choose_attribute(self, attrs, examples): + """Choose the attribute with the highest information gain.""" + return argmax_random_tie(attrs, key=lambda a: self.information_gain(a, examples)) + + def information_gain(self, attr, examples): + """Return the expected reduction in entropy from splitting by attr.""" + + def I(examples): + return information_content([self.count(self.dataset.target, v, examples) + for v in self.dataset.values[self.dataset.target]]) + + n = len(examples) + remainder = sum((len(examples_i) / n) * I(examples_i) + for (v, examples_i) in self.split_by(attr, examples)) + return I(examples) - remainder + + def split_by(self, attr, examples): + """Return a list of (val, examples) pairs for each val of attr.""" + return [(v, [e for e in examples if e[attr] == v]) for v in self.dataset.values[attr]] + + def predict(self, x): + return self.tree(x) + + def __call__(self, x): + return self.predict(x) + + +def information_content(values): + """Number of bits to represent the probability distribution in values.""" + raise NotImplementedError + + +if __name__ == "__main__": + from utils.dataset4learners import * + + iris = DataSet(name="iris") + DTL = DecisionTreeLearner(iris) + print(f'DTL.predict([5, 3, 1, 0.1]): {DTL.predict([5, 3, 1, 0.1])}') + assert DTL.predict([5, 3, 1, 0.1]) == 'setosa' + print(f'DTL.predict([6, 5, 3, 1.5]): {DTL.predict([6, 5, 3, 1.5])}') + assert DTL.predict([6, 5, 3, 1.5]) == 'versicolor' + print(f'DTL.predict([7.5, 4, 6, 2]): {DTL.predict([7.5, 4, 6, 2])}') + assert DTL.predict([7.5, 4, 6, 2]) == 'virginica' diff --git a/02_decision_tree/DecisionTreeLearner_2.py b/02_decision_tree/DecisionTreeLearner_2.py new file mode 100644 index 0000000..49d4448 --- /dev/null +++ b/02_decision_tree/DecisionTreeLearner_2.py @@ -0,0 +1,132 @@ +import sys +sys.path.insert(1, '../') +from utils.utils import * + + +class DecisionFork: + """ + A fork of a decision tree holds an attribute to test, and a dict + of branches, one for each of the attribute's values. + """ + + def __init__(self, attr, attr_name=None, default_child=None, branches=None): + """Initialize by saying what attribute this node tests.""" + self.attr = attr + self.attr_name = attr_name or attr + self.default_child = default_child + self.branches = branches or {} + + def __call__(self, example): + """Given an example, classify it using the attribute and the branches.""" + attr_val = example[self.attr] + if attr_val in self.branches: + return self.branches[attr_val](example) + else: + # return default class when attribute is unknown + return self.default_child(example) + + def add(self, val, subtree): + """Add a branch. If self.attr = val, go to the given subtree.""" + self.branches[val] = subtree + + def display(self, indent=0): + name = self.attr_name + print('Test', name) + for (val, subtree) in self.branches.items(): + print(' ' * 4 * indent, name, '=', val, '==>', end=' ') + subtree.display(indent + 1) + + def __repr__(self): + return 'DecisionFork({0!r}, {1!r}, {2!r})'.format(self.attr, self.attr_name, self.branches) + + +class DecisionLeaf: + """A leaf of a decision tree holds just a result.""" + + def __init__(self, result): + self.result = result + + def __call__(self, example): + return self.result + + def display(self): + print('RESULT =', self.result) + + def __repr__(self): + return repr(self.result) + + +class DecisionTreeLearner: + """DecisionTreeLearner: based on information gain""" + + def __init__(self, dataset): + self.dataset = dataset + self.tree = self.decision_tree_learning(dataset.examples, dataset.inputs) + + def decision_tree_learning(self, examples, attrs, parent_examples=()): + if len(examples) == 0: + return self.plurality_value(parent_examples) + if self.all_same_class(examples): + return DecisionLeaf(examples[0][self.dataset.target]) + if len(attrs) == 0: + return self.plurality_value(examples) + A = self.choose_attribute(attrs, examples) + tree = DecisionFork(A, self.dataset.attr_names[A], self.plurality_value(examples)) + for (v_k, exs) in self.split_by(A, examples): + subtree = self.decision_tree_learning(exs, remove_all(A, attrs), examples) + tree.add(v_k, subtree) + return tree + + def plurality_value(self, examples): + """ + Return the most popular target value for this set of examples. + (If target is binary, this is the majority; otherwise plurality). + """ + popular = argmax_random_tie(self.dataset.values[self.dataset.target], + key=lambda v: self.count(self.dataset.target, v, examples)) + return DecisionLeaf(popular) + + def count(self, attr, val, examples): + """Count the number of examples that have example[attr] = val.""" + return sum(e[attr] == val for e in examples) + + def all_same_class(self, examples): + """Are all these examples in the same target class?""" + class0 = examples[0][self.dataset.target] + return all(e[self.dataset.target] == class0 for e in examples) + + def choose_attribute(self, attrs, examples): + """Choose the attribute with the highest information gain.""" + return argmax_random_tie(attrs, key=lambda a: self.information_gain(a, examples)) + + def information_gain(self, attr, examples): + """Return the expected reduction in entropy from splitting by attr.""" + raise NotImplementedError + + def split_by(self, attr, examples): + """Return a list of (val, examples) pairs for each val of attr.""" + return [(v, [e for e in examples if e[attr] == v]) for v in self.dataset.values[attr]] + + def predict(self, x): + return self.tree(x) + + def __call__(self, x): + return self.predict(x) + + +def information_content(values): + """Number of bits to represent the probability distribution in values.""" + raise NotImplementedError + + +if __name__ == "__main__": + from utils.dataset4learners import * + + iris = DataSet(name="iris") + DTL = DecisionTreeLearner(iris) + print(f'DTL.predict([5, 3, 1, 0.1]): {DTL.predict([5, 3, 1, 0.1])}') + assert DTL.predict([5, 3, 1, 0.1]) == 'setosa' + print(f'DTL.predict([6, 5, 3, 1.5]): {DTL.predict([6, 5, 3, 1.5])}') + assert DTL.predict([6, 5, 3, 1.5]) == 'versicolor' + print(f'DTL.predict([7.5, 4, 6, 2]): {DTL.predict([7.5, 4, 6, 2])}') + assert DTL.predict([7.5, 4, 6, 2]) == 'virginica' diff --git a/02_decision_tree/DecisionTreeLearner_3.py b/02_decision_tree/DecisionTreeLearner_3.py new file mode 100644 index 0000000..568fc24 --- /dev/null +++ b/02_decision_tree/DecisionTreeLearner_3.py @@ -0,0 +1,121 @@ +import sys +sys.path.insert(1, '../') +from utils.utils import * + + +class DecisionFork: + """ + A fork of a decision tree holds an attribute to test, and a dict + of branches, one for each of the attribute's values. + """ + + def __init__(self, attr, attr_name=None, default_child=None, branches=None): + """Initialize by saying what attribute this node tests.""" + self.attr = attr + self.attr_name = attr_name or attr + self.default_child = default_child + self.branches = branches or {} + + def __call__(self, example): + """Given an example, classify it using the attribute and the branches.""" + attr_val = example[self.attr] + if attr_val in self.branches: + return self.branches[attr_val](example) + else: + # return default class when attribute is unknown + return self.default_child(example) + + def add(self, val, subtree): + """Add a branch. If self.attr = val, go to the given subtree.""" + self.branches[val] = subtree + + def display(self, indent=0): + name = self.attr_name + print('Test', name) + for (val, subtree) in self.branches.items(): + print(' ' * 4 * indent, name, '=', val, '==>', end=' ') + subtree.display(indent + 1) + + def __repr__(self): + return 'DecisionFork({0!r}, {1!r}, {2!r})'.format(self.attr, self.attr_name, self.branches) + + +class DecisionLeaf: + """A leaf of a decision tree holds just a result.""" + + def __init__(self, result): + self.result = result + + def __call__(self, example): + return self.result + + def display(self): + print('RESULT =', self.result) + + def __repr__(self): + return repr(self.result) + + +class DecisionTreeLearner: + """DecisionTreeLearner: based on information gain""" + + def __init__(self, dataset): + self.dataset = dataset + self.tree = self.decision_tree_learning(dataset.examples, dataset.inputs) + + def decision_tree_learning(self, examples, attrs, parent_examples=()): + raise NotImplementedError + + def plurality_value(self, examples): + """ + Return the most popular target value for this set of examples. + (If target is binary, this is the majority; otherwise plurality). + """ + popular = argmax_random_tie(self.dataset.values[self.dataset.target], + key=lambda v: self.count(self.dataset.target, v, examples)) + return DecisionLeaf(popular) + + def count(self, attr, val, examples): + """Count the number of examples that have example[attr] = val.""" + return sum(e[attr] == val for e in examples) + + def all_same_class(self, examples): + """Are all these examples in the same target class?""" + class0 = examples[0][self.dataset.target] + return all(e[self.dataset.target] == class0 for e in examples) + + def choose_attribute(self, attrs, examples): + """Choose the attribute with the highest information gain.""" + return argmax_random_tie(attrs, key=lambda a: self.information_gain(a, examples)) + + def information_gain(self, attr, examples): + """Return the expected reduction in entropy from splitting by attr.""" + raise NotImplementedError + + def split_by(self, attr, examples): + """Return a list of (val, examples) pairs for each val of attr.""" + return [(v, [e for e in examples if e[attr] == v]) for v in self.dataset.values[attr]] + + def predict(self, x): + return self.tree(x) + + def __call__(self, x): + return self.predict(x) + + +def information_content(values): + """Number of bits to represent the probability distribution in values.""" + raise NotImplementedError + + +if __name__ == "__main__": + from utils.dataset4learners import * + + iris = DataSet(name="iris") + DTL = DecisionTreeLearner(iris) + print(f'DTL.predict([5, 3, 1, 0.1]): {DTL.predict([5, 3, 1, 0.1])}') + assert DTL.predict([5, 3, 1, 0.1]) == 'setosa' + print(f'DTL.predict([6, 5, 3, 1.5]): {DTL.predict([6, 5, 3, 1.5])}') + assert DTL.predict([6, 5, 3, 1.5]) == 'versicolor' + print(f'DTL.predict([7.5, 4, 6, 2]): {DTL.predict([7.5, 4, 6, 2])}') + assert DTL.predict([7.5, 4, 6, 2]) == 'virginica' diff --git a/02_decision_tree/DecisionTreeLearner_4.py b/02_decision_tree/DecisionTreeLearner_4.py new file mode 100644 index 0000000..c820f18 --- /dev/null +++ b/02_decision_tree/DecisionTreeLearner_4.py @@ -0,0 +1,34 @@ +import sys +sys.path.insert(1, '../') +from utils.utils import * + + +class DecisionFork: + """ + A fork of a decision tree holds an attribute to test, and a dict + of branches, one for each of the attribute's values. + """ + raise NotImplementedError + + +class DecisionLeaf: + """A leaf of a decision tree holds just a result.""" + raise NotImplementedError + + +class DecisionTreeLearner: + """DecisionTreeLearner: based on information gain""" + raise NotImplementedError + + +if __name__ == "__main__": + from utils.dataset4learners import * + + iris = DataSet(name="iris") + DTL = DecisionTreeLearner(iris) + print(f'DTL.predict([5, 3, 1, 0.1]): {DTL.predict([5, 3, 1, 0.1])}') + assert DTL.predict([5, 3, 1, 0.1]) == 'setosa' + print(f'DTL.predict([6, 5, 3, 1.5]): {DTL.predict([6, 5, 3, 1.5])}') + assert DTL.predict([6, 5, 3, 1.5]) == 'versicolor' + print(f'DTL.predict([7.5, 4, 6, 2]): {DTL.predict([7.5, 4, 6, 2])}') + assert DTL.predict([7.5, 4, 6, 2]) == 'virginica' diff --git a/02_decision_tree/decisiontree_fruit.jpg b/02_decision_tree/decisiontree_fruit.jpg new file mode 100644 index 0000000..41ac4d6 Binary files /dev/null and b/02_decision_tree/decisiontree_fruit.jpg differ diff --git a/02_decision_tree/expected_output.txt b/02_decision_tree/expected_output.txt new file mode 100644 index 0000000..b848f06 --- /dev/null +++ b/02_decision_tree/expected_output.txt @@ -0,0 +1,3 @@ +DTL.predict([5, 3, 1, 0.1]): setosa +DTL.predict([6, 5, 3, 1.5]): versicolor +DTL.predict([7.5, 4, 6, 2]): virginica \ No newline at end of file diff --git a/03_linear_regression/03_linear_regression.ipynb b/03_linear_regression/03_linear_regression.ipynb new file mode 100644 index 0000000..ceb3a0b --- /dev/null +++ b/03_linear_regression/03_linear_regression.ipynb @@ -0,0 +1,2741 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ccfb832d-9cbc-40fb-9fd7-02e9ba9d4882", + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "source": [ + "# 线性回归\n", + "\n", + "## 目录\n", + "\n", + "- 上海房价预测问题\n", + "- 线性回归的从零开始实现\n", + "- 线性回归的简洁实现" + ] + }, + { + "cell_type": "markdown", + "id": "a9ce58d2-70ce-46a0-887e-3ebc8827ca65", + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "source": [ + "## 上海房价预测问题" + ] + }, + { + "cell_type": "markdown", + "id": "536d9a6d-23ed-4c1d-b28a-e7d33e8bf25c", + "metadata": {}, + "source": [ + "数据采自统计局" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "f4e408c7-3b77-4cfe-8c1e-5bd6956c496b", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Year Price\n", + "0 2010 14213\n", + "1 2011 13448\n", + "2 2012 13870\n", + "3 2013 16192\n", + "4 2014 16415\n", + "5 2015 21501\n", + "6 2016 25910\n", + "7 2017 24866\n", + "8 2018 28981\n", + "9 2019 32926\n", + "10 2020 36741\n", + "11 2021 40974\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "price_data = {\n", + " 'Year': [str(y) for y in range(2010, 2022)],\n", + " 'Price':[14213, 13448, 13870, 16192, 16415, 21501, 25910, 24866, 28981, 32926, 36741, 40974],\n", + "}\n", + "\n", + "price_df = pd.DataFrame(price_data)\n", + "\n", + "print(price_df)" + ] + }, + { + "cell_type": "markdown", + "id": "1dd6323f-510b-4964-ba20-8762f637885b", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "首选绘制散点图,观察数据分布特征" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2e408d58-b1e6-48b7-b71c-da0dd8ca6a1f", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(12000.0, 42000.0)" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqUAAAHLCAYAAADxzuGFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAxOAAAMTgF/d4wjAAB0/UlEQVR4nO3deVxU5f4H8M8ZZtgXNVBRzOUmiGiriOaW2qKVpWhC5tLizbS0Um9Ut9tu17qJWaZ1723TNNEwu1lqueUajlnZIEJppqgoSyI7DPP9/eFvTgwzAyPOMAN83q/XvJTzPM+czwww58tZnqOIiICIiIiIyI007g5ARERERMSilIiIiIjcjkUpEREREbkdi1IiIiIicjsWpURERETkdixKiYiIiMjttO4OQI3Dx8cHYWFh7o5BREREFyE3NxcVFRXujtEoWJS2EGFhYcjOznZ3DCIiIroIERER7o7QaHj4noiIiIjcjkUpEREREbkdi1IiIiIicjsWpURERETkdixKiYiIiMjtePU9ERERUQOICPR6PdauXYvc3FyEhYUhPj4effv2dXe0JolFKREREdFFMhgMSExMRFZWFkQERqMRWq0WycnJiIyMREpKCmJiYtwds0nh4XsiIiKii2AwGNCvXz9kZGSgqqoKRqMRAGA0GlFVVYWMjAzExcUhPT3dzUmbFhalRERERA4SESQmJqKsrAwmk8lmH5PJhLKyMiQkJDRyuqaNRSkRERGRg/R6PbKysuwWpGYmkwlZWVnQ6/WNlKzpY1FKRERE5KC1a9dCRBzqKyJITU11caLmg0UpERERkYNyc3PVc0jrYzQakZeX5+JEzQeLUiIiIiIHhYWFQat1bPIirVaL0NBQFydqPliUEhERETkoPj4eiqI41FdRFIwdO9bFiZoPFqVEREREDoqNjUVkZCQ0mrpLKI1Gg8jISMTGxjZSsqavSRalt912GxRFgaIouPfee+vsu3v3bowbNw7h4eHw8fFBp06dMGXKFIfmDjMajXjrrbfQt29fhISEIDAwEFdddRXmzZuHsrKyescfP34cM2bMQLdu3eDr64u2bdvi1ltvxZdffunQ61y/fj1GjhyJtm3bwtfXF926dcPDDz+MEydOODSeiIiInEtRFKSkpMDPz89uYarRaODn54eUlJRGTtfESROzcuVKAaA+pkyZYrdvcnKyaDQaASCKokhISIg6zsfHRz799FO7Y4uLi2XAgAEW/f39/dWvo6Oj5ezZs3bH79ixQ4KDg9X+wcHBahYAMmfOnDpf52OPPab21Wg0Fs8VEhIiu3btqve9qqljx44X1Z+IiIjsMxgMEhMTIzqdTrRarQAQrVYrOp1OYmJixGAwOGU9LWn73aSK0vz8fGnbtq2EhIRIdHR0nUXp5s2bRVEUASDTpk2TvLw8ERE5ceKEjB49WgCIr6+vZGZm2hw/efJktZhMSUmR6upqERH5+uuvJTw8XADIsGHDbI7Nzc2VNm3aCAAZMGCAuo6ioiJ59tln1eJy2bJlNse///77ap/nnntOioqKRETk8OHDcv311wsACQ0Nlfz8fIffu5b0Q01ERNRY9u3bJ0lJSfLAAw9IUlKS7Nu3z6nP35K2302qKJ0yZYoAkCVLlsiQIUPqLEr79OkjAGTEiBFWbRUVFdKrVy8BIAkJCVbtBw8eVAvaTz75xKp9z549atG4YcMGq/a5c+cKAGnfvr388ccfVu0PPvigAJCIiAipqqqyaKusrJQOHTqoxXRtBQUF0r59ewEgSUlJNl+7LS3ph5qIiKi5aEnb7yZzTunmzZvx0UcfIS4uDtOmTauzb2ZmJvbv3w8AeOqpp6zavb29MXfuXADA559/juLiYov2FStWQETQrVs3m7cI69+/P2644QYAwMcff2zRJiJYsWIFAGD69Olo1aqV1XhzpuzsbGzfvt2ibcuWLTh16pTd7K1bt8ZDDz1kkZOIiIioqWsSRWlZWRmmTZsGrVaLd999t94r3rZs2QIACAoKwoABA2z2GTlyJACgvLwcu3btsmjbunUrAGDEiBF2p30wj9+8ebPF8oyMDJw+fdqiT21dunRBdHS0zfHmdffs2ROdO3euc93Z2dnIzMy02YeIiIioKWkSRemzzz6Lo0eP4rHHHsNVV11Vb/9Dhw4BAKKjo+Hl5WWzT9u2bREWFgYAFlfiiwgyMjIAAL169bK7DnPbmTNnkJ+fb7VuR8fXngXAPN6RsbbGExERETVFHl+UHjhwAAsXLsTll1+O559/3qEx5sPfHTt2rLOfud28ZxMAioqK1MP5dY2v2VZzvHndrVu3hp+f30Wt29Hs/v7+6mkBtccTERERNUUeXZRWV1fjr3/9K6qrq7F48WIEBAQ4NM5cVPr7+9fZz9xeVFRkNba+8TXbbI1vyLqdMd4sOTkZERER6qP2ebNEREREnsSji9Lk5GQcOHAAY8aMwahRo9wdp0mZPXs2srOz1UdgYKC7IxERERHZ5bFF6dGjR/H8888jKCgIb7755kWNNRdgpaWldfYztwcFBVmNrW98zTZb4xuybmeMJyIiImqKPLYonT17NkpLS/Hkk0+iVatWKC4utnhUV1cDuHArUPMyk8kEAOjQoQMA4OTJk3Wuw9weHh6uLgsKClILw7rG12yrOd687j/++KPOW5HaWrej2UtLS3Hu3Dmb44mIiIiaIo8tSo8dOwYA+Pvf/46goCCrh3kapxUrVqjLDh48CODCdErAhemZzMVrbWfPnkVubi4AICYmRl2uKIo6XZPBYLCbz9zWrl07XHbZZepy87odHV9z3TXHOzLW1ngiIiKipshji9JLMXz4cAAXLgLas2ePzT4bN24EAPj6+mLgwIE2x2/atMnu5PTm8TfeeKPF8ujoaHVvp7lPbb///rs67VTt8eZ1Z2Rk4Pjx43WuOyIiAlFRUTb7EBERETUlHluU/vjjj5ALt0G1+RgyZAgAYMqUKeqyq6++GgAQFRWFPn36AADmz59v9dxVVVVYsGABAGD06NFWFwHdc889UBQFR44cwZo1a6zGp6WlYdu2bQCASZMmWbQpioIJEyYAAJYuXYrCwkKr8a+++iqAC0Wl+c5QZsOGDUOHDh0gIjaznzt3Du+88w4AYOLEiXYn9yciIiJqSjy2KL1U8+fPh6Io+OqrrzBjxgwUFBQAuHCuZmJiIg4ePAhfX1+88MILVmN79eqlFpsPPvgg1qxZo56vumXLFsTHxwO4UEDecsstVuOTkpLQpk0bnD59GqNGjcIvv/wCACgpKcGLL76oFpWvvPIKtFqtxVidToeXX34ZAPDOO+/gxRdfRElJCQAgKysLo0aNwunTpxEaGoq//e1vl/w+EREREXkEaaKGDBkiAGTKlCl2+yxYsEAURREAoiiKtGrVSgAIAPHx8ZFPP/3U7tji4mIZMGCA2t/X11f8/f3Vr6Ojo+Xs2bN2x3/77bcSHBys9g8JCREvLy/16zlz5tT5+h599FG1r5eXl4SEhKhfBwcHy65du+p9j2rq2LHjRfUnIiIi92tJ2+9mu6cUuHAF/44dOxAfH4927dqhtLQUERERmDRpEr7//nuMHTvW7tiAgABs374dixYtQmxsLHQ6HRRFQe/evfHSSy/h+++/V29TasvgwYNx8OBBTJ8+HV26dEF5eTlat26NESNGYP369Xj99dfrzP7GG2/giy++wIgRI9C6dWuUl5ejS5cumD59On7++WcMGDCgwe8LERERkadRROxcyUPNSkREBLKzs90dg4iIiC5CS9p+N+s9pURERETUNLAoJSIiIiK3Y1FKRERERG7HopSIiIiI3I5FKRERERG5HYtSIiIiInI7FqVERERE5HYsSomIiIjI7ViUEhEREZHbsSglIiIiIrdjUUpEREREbseilIiIiIjcjkUpEREREbkdi1IiIiIicjsWpURERETkdixKiYiIiMjtWJQSERERkduxKCUiIiIit2NRSkRERERux6KUiIiIiNyORSkRERERuR2LUiIiIiJyOxalREREROR2WncHICIiIqpNRKDX67F27Vrk5uYiLCwM8fHx6Nu3r7ujkYuwKCUiIiKPYjAYkJiYiKysLIgIjEYjtFotkpOTERkZiZSUFMTExLg7JjkZD98TERGRxzAYDOjXrx8yMjJQVVUFo9EIADAajaiqqkJGRgbi4uKQnp7u5qTkbCxKiYiIyCOICBITE1FWVgaTyWSzj8lkQllZGRISEho5Hbkai1IiIiLyCHq9HllZWXYLUjOTyYSsrCzo9fpGSkaNgUUpEREReYS1a9dCRBzqKyJITU11cSJqTCxKiYiIyCPk5uaq55DWx2g0Ii8vz8WJqDGxKCUiIiKPEBYWBq3WsYmBtFotQkNDXZyIGhOLUiIiIvII8fHxUBTFob6KomDs2LEuTkSNiUUpEREReYTY2FhERkZCo6m7PNFoNIiMjERsbGwjJaPGwKKUiIiIPIKiKEhJSYGfn5/dwlSj0cDPzw8pKSmNnI5cjUUpEREReYyYmBikpaUhOjoaOp1OPcdUq9VCp9MhOjoaaWlpvKNTM8TbjBIREZFHiYmJgcFggF6vR2pqKvLy8hAaGoqxY8fykH0zxqKUiIiIPFJsbCyL0BaEh++JiIiIyO1YlBIRERGR27EoJSIiIiK3Y1FKRERERG7HopSIiIiI3I5FKRERERG5nccXpXv27MEzzzyDW265Bd27d0dISAi8vb3RoUMH3HbbbVi5ciVExObYLl26QFGUOh+PPPJInevPz89HUlISevToAX9/f7Rp0wZDhgzB8uXLHcq/e/dujBs3DuHh4fDx8UGnTp0wZcoUpKen1zvWaDTirbfeQt++fRESEoLAwEBcddVVmDdvHsrKyhxaPxEREVFToIi9is5DJCYmWtxKLDAwECaTCaWlpeqyoUOH4vPPP0dQUJDF2C5duuD3339HcHAw/Pz8bD7//fffj1deecVm2+HDhzF06FDk5OSo6y4vL4fRaAQAjB8/Hp988ondW6EtXLgQc+fOhclkgqIoCA4ORmFhIQDAx8cHK1aswNixY22OLSkpwS233ILdu3er/b28vNTXHR0djW+//RZhYWE2x9cWERGB7Oxsh/oSERGRZ2hJ22+P31M6dOhQ/Pvf/8ZPP/2EoqIiFBUVoaSkBKdPn8bLL78MLy8vbNu2DXPmzLH7HIsWLUJOTo7Nh72CtLKyEqNGjUJOTg569OgBvV6vrnvx4sXQ6XRYvXo15s2bZ3P8li1bMGfOHJhMJkybNg25ubk4d+4cTpw4gdGjR6OiogITJ05EVlaWzfEzZszA7t27ERwcjJSUFJSWlqKkpARff/01wsPDkZGRgcTExIt/Q4mIiIg8kTRxTz/9tAAQPz8/qaystGjr3LmzAJAPPvjgop938eLF6vMePXrUqv2VV14RABIQECB5eXlW7X369BEAMmLECKu2iooK6dWrlwCQhIQEq/aDBw+KoigCQD755BOr9j179ggAASAbNmxw6PV07NjRoX5ERETkOVrS9tvj95TWJy4uDgBQVlaGgoICpz2v+ZzRxMREdO3a1ap95syZCAwMRElJCT777DOLtszMTOzfvx8A8NRTT1mN9fb2xty5cwEAn3/+OYqLiy3aV6xYARFBt27dkJCQYDW+f//+uOGGGwAAH3/88cW/OCIiIiIP0+SL0l27dgG4cL5n27ZtnfKcxcXF2LdvHwBg5MiRNvsEBgZi0KBBAIDNmzdbtG3ZsgUAEBQUhAEDBtgcb37e8vJy9TWYbd26FQAwYsQIKIpS5/ja6yYiIiJqippkUVpSUoKMjAw89dRTeP311wEAs2bNslvAvf766+jQoQO8vb0RFhaG4cOHY+nSpSgvL7fZ//Dhw+oV/b169bKbw9xW+0r6Q4cOAbhwMZKXl5fNsW3btlUvUqo5XkSQkZHh8LrPnDmD/Px8u/2IiIiImgKtuwM4KicnB+Hh4VbLdTodZs6ciRdffNHu2PT0dPj5+cHf3x95eXnYunUrtm7diqVLl2L9+vW4/PLLLfqfOnVK/X/Hjh3tPq+57fTp0zbH1zXW3J6bm2sxvqioSD2c78i6zeu/7LLL6lwXERERkSdrMntKvby80K5dO7Rr1w4+Pj4AAEVR8Nhjj+GJJ56wuUdy9OjR+PTTT5Gbm4vS0lKcO3cOp06dwksvvQRvb2/8/PPPuPXWW1FZWWkxruY5nv7+/nYzmduKiopsjq9rrL3xF7tuW+sHgOTkZERERKiP2uetEhEREXmSJlOUhoWFqdM4lZaW4ujRo5g5cyaSk5PRq1cvq/MyAeCNN97A2LFjERoaqi4LDw/HM888gzVr1gC4sBf1ww8/bKyX0Whmz56N7Oxs9REYGOjuSERERER2NZmitCaNRoOuXbti0aJF+Ne//oW8vDwkJiZaTKhfnzvuuEO9UOmLL76waKtZwNX1nOa22pP2m8fXl8fW+Itdt631ExERETU1TbIorWn69Onw8fHByZMnsWHDhosaa55O6ujRoxbLO3TooP7/5MmTdseb22qf62oeX9dYe+ODgoLUwtSRddtaPxEREVFT0+SLUl9fX/UinyNHjjjlOaOjo9Ur+Q0Gg91+5raYmBiL5T179gQAZGRkoLq62ubYs2fPIjc312q8oiiIjo52eN3t2rXjRU5ERETU5DX5orS4uFgt7i72vMm0tDQAsJocPyAgQN2LunHjRptjS0pKsHPnTgDAjTfeaNE2fPhwABcuQNqzZ4/N8ebn9fX1xcCBA22O37Rpkzo1lb3xtddNRERE1BR5dFFqNBrr7bNw4UJUVVUBAAYPHqwut1fMmX355ZdqUTlq1Cir9okTJwIAVq1ahWPHjlm1v/322yguLkZAQADGjBlj0RYVFYU+ffoAAObPn281tqqqCgsWLABwYYaA2sX0PffcA0VRcOTIEfWCrJrS0tKwbds2AMCkSZPqfJ1ERERETYJbb3Jajx9++EH69esnH374oZw4cUJdbjKZJD09XR5++GH1HvHjxo2zGPvII4/IzJkz5dtvv5WSkhJ1+enTp+WVV14RX19fASAxMTFSUVFhte6Kigq54oorBID07NlT9u/fry5fsmSJeHt7CwB58cUXbWbfvHmzmm369OmSn58vIiLZ2dkSHx8vAMTX11cyMzNtjp88ebIAkJCQEFm9erVUV1erz9uhQwcBIMOGDXP4vWxJ984lIiJqLlrS9tvji1IA6sPX11dCQ0PVgtL8uOOOOywKTxGRKVOmqO2KokirVq0kJCTEYtw111wjx48ft7v+jIwMad++vdo/KChIdDqd+vX48ePVYtGWBQsWqIWpOYN5rI+Pj3z66ad2xxYXF8uAAQMsXru/v7/6dXR0tJw9e9bh97Il/VATERE1Fy1p+62I1HOc243Kysqwbt06bN26FXq9Hjk5OcjPz4evry8iIiLQt29fTJw4ETfddJPV2O+++w5r167F3r178fvvvyMvLw8mkwmXXXYZrr32Wtx11124++67odPp6syQn5+PV199FZ9//jmOHz8OX19fXHnllZg6dapDh8537dqFhQsXYs+ePSgoKEDbtm0xdOhQJCUlWV0gVZvRaMSSJUvw8ccf4/DhwzCZTOjWrRvGjx+POXPmwM/Pr971m0VERCA7O9vh/kREROR+LWn77dFFKTlPS/qhJiIiai5a0vbboy90IiIiIqKWgUUpEREREbkdi1IiIiIicjsWpURERETkdixKiYiIiMjtWJQSERERkduxKCUiIiIit2NRSkRERERup3V3ACIiImpcIgK9Xo+1a9ciNzcXYWFhiI+PR9++fd0djVowFqVEREQtiMFgQGJiIrKysiAiMBqN0Gq1SE5ORmRkJFJSUuq9DTaRK/DwPRERUQthMBjQr18/ZGRkoKqqCkajEQBgNBpRVVWFjIwMxMXFIT093c1JqSViUUpERNQCiAgSExNRVlYGk8lks4/JZEJZWRkSEhIaOR0Ri1IiIqIWQa/XIysry25BamYymZCVlQW9Xt9IyYguYFFKRETUAqxduxYi4lBfEUFqaqqLExFZYlFKRETUAuTm5qrnkNbHaDQiLy/PxYmILLEoJSIiagHCwsKg1To26Y5Wq0VoaKiLExFZYlFKRETUAsTHx0NRFIf6KoqCsWPHujgRkSUWpURERC1AbGwsIiMjodHUvenXaDSIjIxEbGxsIyUjuoBFKRERUQugKApSUlLg5+dntzDVaDTw8/NDSkpKI6cjYlFKRETUYsTExCAtLQ3R0dHQ6XTqOaZarRY6nQ7R0dFIS0vjHZ3ILXibUSIiohYkJiYGBoMBer0eqampyMvLQ2hoKMaOHctD9uRWLEqJiIhaoNjYWBah5FF4+J6IiIiI3I5FKRERERG5HYtSIiIiInI7FqVERERE5HYsSomIiIjI7ViUEhEREZHbsSglIiIiIrdjUUpEREREbseilIiIiIjcjkUpEREREbkdi1IiIiIicjsWpURERETkdixKiYiIiMjtWJQSERERkduxKCUiIiIit2NRSkRERERux6KUiIiIiNyORSkRERERuR2LUiIiIiJyOxalREREROR2Hl2U7tmzB8888wxuueUWdO/eHSEhIfD29kaHDh1w2223YeXKlRCROp9j/fr1GDlyJNq2bQtfX19069YNDz/8ME6cOFHv+ktLS/Hyyy/jqquuQmBgIEJCQhAXF4e33noL1dXV9Y43GAyYMmUKOnXqBB8fH4SHh2PcuHHYvXu3Q69/+fLlGDJkCNq0aQN/f3/06NEDSUlJKCgocGg8ERERUZMhHiwhIUEAqI/AwEDx9/e3WDZ06FA5f/68zfGPPfaY2k+j0UhwcLD6dUhIiOzatcvuunNyciQqKkrt7+/vLz4+PurXgwYNktLSUrvjU1JSxNvb22J9iqKoWRYtWmR3rNFolLFjx6pjtVqtBAYGql+Hh4dLZmam42+kiHTs2PGi+hMREZH7taTtt0fvKR06dCj+/e9/46effkJRURGKiopQUlKC06dP4+WXX4aXlxe2bduGOXPmWI394IMP8MYbbwAAnnvuORQWFqKwsBCHDx/G9ddfj8LCQowePdruXsfx48cjMzMT4eHh+Oabb1BSUoLS0lKsWrUKQUFB2LlzJx555BGbYzMyMjB58mRUVlZi9OjROHHiBM6dO4fc3FxMmzYNJpMJjz/+OL799lub41988UWkpqZCp9Nh8eLFKCkpQVFREfR6PXr06IHTp0/jjjvuQFVVVcPeWCIiIiJP4+6q+FI8/fTTAkD8/PyksrJSXV5ZWSkdOnQQADJt2jSrcQUFBdK+fXsBIElJSVbt69evV/dK7tmzx6p95cqV6h7P9PR0q/Zx48YJAOndu7dFLrNbbrlFAEhcXJxV29mzZ9W9wf/85z+t2o8cOSJ+fn4CQJYuXWr9ptjRkv7SIiIiai5a0vbbo/eU1icuLg4AUFZWZrHHc8uWLTh16hQA4KmnnrIa17p1azz00EMAgBUrVlidl7p8+XIAF/bU9u/f32p8YmIiunbtCpPJhJUrV1q0nT9/Hv/73/8AAHPnzoVOp7Mab86UlpaGX3/91aItNTUVpaWlCAwMxMyZM63GduvWDQkJCQCAjz/+2KqdiIiIqClq0kXprl27AACBgYFo27atunzr1q0AgJ49e6Jz5842x44cORIAkJ2djczMTIs283hzn9oURcGIESMAAJs3b7bKVFlZCQBqn9oGDhyIoKAgm+PN6x48eDACAgLqzL53716Ulpba7ENERETUlDS5orSkpAQZGRl46qmn8PrrrwMAZs2aBUVR1D6HDh0CAPTq1cvu89RsS09PV/+fl5eH3Nxch8eb11V73W3btrUolGvy8vJCjx49rNZ9sdlNJhMyMjLs9iMiIiJqKrTuDuCInJwchIeHWy3X6XSYOXMmXnzxRYvl5kP3HTt2tPuc/v7+aNWqFc6dO4fTp09bja1vvLmtqKgIxcXFCAwMdHjd5na9Xm+xbkfH12yrPZ6IiIioKWoSe0q9vLzQrl07tGvXDj4+PgAuHEJ/7LHH8MQTT8DLy8uif3FxMYALhWddzO1FRUVWY+sbX7PN1viGrNvR8fbWXVNycjIiIiLUR83XRURERORpmkRRGhYWhpycHOTk5KC0tBRHjx7FzJkzkZycjF69eqnnltKfZs+ejezsbPVh3pNLRERE5ImaRFFak0ajQdeuXbFo0SL861//Ql5eHhITEy0u+DEXYPVdBGRuN190VHNsfeNrttka35B1Ozre3rqJiIiImqomV5TWNH36dPj4+ODkyZPYsGGDurxDhw4AgJMnT9odW1painPnzgGAxfmq5rH1jTe3BQUFWRSyjqy7Znvtc2UdGV+zzda5tkRERERNTZMuSn19fXHZZZcBAI4cOaIu79mzJ4AL9563p2ZbTEyM+v/Q0FCEhYU5PN68rtrrPnv2rHoVf23V1dU4fPiw1bovNrtGo0F0dLTdfkRERERNRZMuSouLi9XCr+beyuHDhwO4cLvP48eP2xy7ceNGAEBERASioqIs2szjzX1qExFs2rQJAHDjjTdatA0cOFC9GMve+N27d6sXKNUeb173zp077R7CNz9v//79672gioiIiKgp8Nii1Gg01ttn4cKF6v3fBw8erC4fNmwYOnToABHB/PnzrcadO3cO77zzDgBg4sSJFnOcmpcBwLZt25CWlmY1fs2aNTh69Cg0Gg0mTJhg0RYcHIxRo0YBABYsWGDz/vTmTP369cMVV1xh0RYfHw9/f38UFRVh8eLFVmOPHTuGVatWAQAmTZpk1U5ERETUJLn5Nqd2/fDDD9KvXz/58MMP5cSJE+pyk8kk6enp8vDDD4uiKAJAxo0bZzX+/fffFwCiKIq88MILUlxcLCIimZmZMnDgQAEgoaGhkp+fb3P9gwcPFgDSsWNH2bx5s4iIVFdXy+rVqyU4OFgAyP33329z7KFDh8THx0cASHx8vGRnZ4uISH5+vkyfPl3NtX37dpvjn332WQEg3t7esmTJEqmoqBARkf3790vPnj0FgERFRUllZaWD72bLuncuEZE7mUwmSUtLk6SkJLn//vslKSlJ0tLS3B2LmqiWtP326KIUgPrw9fWV0NBQ8fX1tVh+xx13SElJic3nePTRR9V+Xl5eEhISon4dHBwsu3btsrv+nJwciYqKUvv7+/tbrHvQoEFSWlpqd3xKSop4e3ur/Vu1aqUW0RqNRhYtWmR3rNFolLFjx6pjdTqdBAUFqV+Hh4dLZmam42+mtKwfaiIid/n5558lJiZGdDqdaLVaASBarVZ0Op3ExMSIwWBwd0RqYlrS9lsREWmEHbIXraysDOvWrcPWrVuh1+uRk5OD/Px8+Pr6IiIiAn379sXEiRNx00031fk869evx9tvv439+/ejqKgI4eHhGDlyJJ588klcfvnldY4tLS1FcnIyVq9erR6u79GjByZNmoQZM2ZYTdpfm8FgwGuvvYatW7ciNzcXbdq0wfXXX4/Zs2djwIAB9b4Hy5Ytw3vvvYeDBw+ivLwcnTt3xp133omkpCS0adOm3vE1RUREIDs7+6LGEBGR4wwGA/r164eysjKYTCardo1GAz8/P6SlpVld5EpkT0vafntsUUrO1ZJ+qImIGpuIoHfv3sjIyLBZkJqZZ02pa4YVoppa0vbbYy90IiIiair0ej2ysrLqLEgBwGQyISsrC3q9vpGSETUdLEqJiIgu0dq1a+HogUcRQWpqqosTETU9Wmc8SW5uLrZt24bjx4+jtLQUzz77rDOeloiIqEnIzc11aCpD4MKUh3l5eS5ORNT0XFJRWllZiblz5+Lf//63xXycNYvSc+fO4S9/+QtKSkpw6NAhdOvW7VJWSURE5HHCwsKg1WodKky1Wi1CQ0MbIRVR09Lgw/cmkwl33nkn3n77bVRVVaFr167Qaq1r3FatWmHy5MmorKzE6tWrLyksERGRJ4qPj7e6EYs9iqJg7NixLk5E1PQ0uChdtmwZNm3ahPbt22PPnj349ddf7U5TNH78eADA1q1bG7o6IiIijxUbG4vIyEhoNHVvVjUaDSIjIxEbG9tIyYiajgYXpR999BEURcHChQsRFxdXZ99rr70WGo0Ghw4daujqiIiIPJaiKEhJSYGfn5/dwtQ8T2lKSkojpyNqGhpclB48eBCKouCOO+6ot6+Pjw9CQkJ4YjcRETVbMTExSEtLQ3R0NHQ6nXpKm1arhU6nQ3R0NCfOJ6pDgy90KikpQVBQEHx9fR3qX1VVZfOcUyIiouYiJiYGBoMBer0eqampyMvLQ2hoKMaOHctD9kT1aHCVGBYWhlOnTqGkpAQBAQF19j1y5AiKi4vRvXv3hq6OiIioyYiNjWURSnSRGnz4vn///gCAdevW1ds3OTkZiqJg8ODBDV0dERERETVjDS5Kp02bBhHB3//+dxw9etRmHxHBq6++iqVLlwIAZsyY0dDVEREREVEz1uDD98OHD8fUqVPx3//+F3369MHo0aNRUlICAHjllVdw/PhxbNy4ESdOnAAAPP7447jmmmuck5qIiIiImhVFHL1Zrw3V1dV4+umnsWDBAphMpgtPWGPyYBGBRqPBE088gXnz5jk8sTA5X0REBLKzs90dg4iIiC5CS9p+X1JRanb06FF8+OGH2Lt3L06fPo3q6mq0a9cO119/Pe69915ERkY6Iytdgpb0Q01ERNRctKTtt1OKUvJ8LemHmoiIqLloSdvvBl/oRERERETkLA0uSsvKyrBjxw7o9fp6++r1euzYsQPl5eUNXR0RERERNWMNLkpXrlyJoUOHYsWKFfX2fffddzF06FDe75eIiIiIbGpwUZqamgoAmDRpUr19p06dChHBp59+2tDVEREREVEz1uCiNCMjAzqdzqG5R2NjY6HT6ZCRkdHQ1RERERFRM9bgojQnJwfBwcHQaOp/Ci8vLwQHB+P06dMNXR0RERERNWMNLkr9/f1RWFgIo9FYb9+qqioUFhbC29u7oasjIiIiomaswUVpVFQUjEYjvv7663r7btq0CUajEd27d2/o6oiIiIioGWtwUTpq1CiICGbPno2CggK7/fLz8zFnzhwoioI77rijoasjIiIiomaswUXpww8/jHbt2uGXX37B1Vdfjffee8/inNHTp0/jP//5D6655hr88ssvaNu2LWbNmuWU0ERERETUvFzSbUbT0tJw66234o8//oCiKAAArVYLAOq5piKC1q1bY8OGDejbt68TIlNDtKTblBERETUXLWn7fUm3GY2Li8OBAweQkJAALy8viAiqqqpQVVUFEYFWq8WECRPwww8/sCAlIiIiIrsuaU9pTaWlpdDr9Thz5gwAoH379ujTpw/8/f2d8fR0iVrSX1pERETNRUvafmud9UT+/v4YMmSIs56OiIiIiFqQSzp8T0RERETkDCxKiYiIiMjtHCpKvby84OXlhZiYGKtlF/MwX5lPRERERFSTQ1Wi+VqomtdEOen6KCIiIiIix4rSbdu2AYDFlfTmZURE1DyICPR6PdauXYvc3FyEhYUhPj6eU/oRUaNw2pRQ5Nla0pQSRHTxDAYDEhMTkZWVBRGB0WiEVquFoiiIjIxESkqKxSlcRNQ4WtL2u8Eneb755psAgHHjxqFDhw5OC0RERI3LYDCgX79+KCsrg8lkUpeb78yXkZGBuLg4pKWlsTAlIpdp8J5S88VLxcXF8Pb2dnYucrKW9JcWETlORNC7d29kZGRYFKS1aTQaREdHw2AwNGI6ImpJ2+8GTwkVGhqKoKAgFqRERE2YXq9HVlZWnQUpAJhMJmRlZUGv1zdSMiJqaRpclF577bUoLCxEbm6uM/MQEVEjWrt2rcOzqYgIUlNTXZyIiFqqBhels2bNgslkwksvveTMPERE1Ihyc3PVc0frYzQakZeX5+JERNRSNbgoHTlyJF5//XW88847mDRpEn766Sdn5iIiokYQFhbm8I1NtFotQkNDXZyIiFqqBl/o1K1bNwBATk4OKioqAAB+fn647LLL4OXlZXtlioIjR440MCpdipZ0ojQROW7fvn0YOHAgqqqq6u2r0+mwe/duxMbGNkIyIgJa1va7wXtKjx07hmPHjqG8vBwiAhFBaWkpTpw4obbZelyMEydO4M0338SYMWPQtWtX+Pr6IiAgAFFRUZg2bVqdV4HecMMNUBSlzsftt99e5/pLS0vx8ssv46qrrkJgYCBCQkIQFxeHt956C9XV1fXmNxgMmDJlCjp16gQfHx+Eh4dj3Lhx2L17t0Ovf/ny5RgyZAjatGkDf39/9OjRA0lJSSgoKHBoPBFRfWJjYxEZGQmNpu7NgUajQWRkJAtSInKZBu8p/eijjxq0wilTpjjU78SJE+jcubPFCfiBgYGoqqpS98xqtVokJydj5syZVuNvuOEGfPvttwgICEBgYKDNddx0001Yvny5zbYzZ85gyJAhyMzMBHDhblbV1dXqugcNGoRNmzbBz8/P5vjVq1dj0qRJqKysBACEhITg/PnzEBFoNBosXLgQs2bNsjm2uroaCQkJ6gUFWq0Wvr6+KC4uBgCEh4dj+/btiIyMtDnelpb0lxYRXZz09HTExcVZzVNqptFo4Ofnx3lKidygRW2/xUP99ttvAkBuvvlmWbFiheTk5IiIiNFoFL1eL4MGDRIAAkA2btxoNX7IkCECQJ577rkGrX/w4MECQMLDw+Wbb74REZHq6mpZtWqVBAUFCQC5//77bY49dOiQ+Pj4CAAZPXq0nDhxQkRE8vLyZNq0aQJANBqNbN++3eb4Z599VgCITqeTxYsXS0VFhYiI6PV66dGjhwCQqKgoqaysdPj1dOzY8WJePhG1MAaDQWJiYkSn04lWqxUAotVqRafTSUxMjBgMBndHJGqRWtL2+6KL0oMHD8qMGTOkb9++EhUVJf369ZPHH39cfvnlF6cGO3funBw4cMBue0VFhVx55ZUCQIYNG2bVfilF6fr169WCd8+ePVbtK1euVAvL9PR0q/Zx48YJAOndu7fNwvGWW24RABIXF2fVdvbsWfH39xcA8s9//tOq/ciRI+Ln5ycAZOnSpQ6/ppb0Q01EDbdv3z5JSkqSBx54QJKSkmTfvn3ujkTUorWk7fdFFaXvvPOOaLVa0Wg0oiiK+tBoNOLr6yupqamuymnTa6+9JgAkODjYqu1SitKEhAQBIEOHDrXZbjKZpGvXrgJA/v73v1u0FRYWire3twCQjz76yOb47du3q0Vv7WJ+6dKlAkACAwOluLjY5vh7771XAMiAAQMcfk0t6YeaiJovk8kkaWlpkpSUJPfff78kJSVJWlqau2MRuUxL2n47fKHTwYMHMXPmTFRXV0Or1aJ///6466670LdvXyiKgoqKCkyZMqVRz3vw9fUFAIcuOroYW7duBXBh2itbFEXBiBEjAACbN2+2aNu1a5d6Hqm5T20DBw5EUFCQzfHmdQ8ePBgBAQE2x5tz7d27F6WlpfW+HiKi5sBgMKB3794YOHAgFixYgPfffx8LFizAwIED0atXL6Snp7s7IhFdAoeL0sWLF8NoNKJ79+748ccfsXv3bqSkpOC7777D3r170b59e5SWluI///mPK/Na2L59OwCgd+/edvusWLECnTt3hre3N9q0aYMBAwbgtddew/nz5232z8vLU+9S1atXL7vPa247dOiQxXLz123btkXbtm1tjvXy8kKPHj0AwOpD1DzekXWbTCZkZGTY7UdE1FwYDAb069cPGRkZqKqqUif8NxqNqKqqQkZGBuLi4liYEjVhDhelO3fuhKIoWLJkCaKjoy3aYmNj8eqrr0JEsGPHDqeHtCUtLQ3r1q0DADzwwAN2+/3666/IyclBQEAAzp07hz179iApKQm9e/e2OeH/qVOn1P937NjR7vOa24qKitSr4muOr2tszfbTp0/bXL8j67Y1noiouRERJCYm2p0dALjwR3pZWRkSEhIaOR0ROYvDRWl2dja8vLwwZMgQm+033nij2s/VCgoKMGHCBJhMJsTFxeG+++6z6nPDDTfgo48+wunTp1FeXo4//vgDeXl5WLx4MYKDg3H8+HGMHDkS+fn5FuNqFpj+/v52M9RsKyoqshpf19ia7TXHOjre3rprSk5ORkREhPqo+bqIiJoSvV6PrKwsuwWpmclkQlZWFvR6fSMlIyJncrgoLSkpQWhoqN3b0YWHhwOAy89xLCsrw5gxY3D06FGEhoZi1apVNu8g9fzzz2Py5Mlo3749FEUBALRp0wYPP/wwtm7dCp1Oh9OnT2PBggUuzesus2fPRnZ2tvqwN1crEZGnW7t2rcWc1XUREXWOZyJqWhp8Ryd7HP3gaIiKigrEx8djx44dCAkJwaZNm9ClS5eLfp7rrrsOiYmJAIAvvvjCoq1m8VZXgV2zzXzRUs3x9RXn5vaaYx0db2/dRETNUW5urnoOaX2MRiPy8vJcnIiIXMHpRamrVFZWYty4cdi4cSMCAwOxYcMGXHvttQ1+vri4OADA0aNHLZZ36NBB/f/Jkyftjje3BQUFWRSy5vF1ja3Zbt7DfDHja7bVHk9E1NyEhYXZPUpXm1arRWhoqIsTEZErOPZb/v8KCgowbNiwBvdRFAVbtmy5mFUCAKqqqnDXXXdh/fr18Pf3x5dffon+/ftf9PM4IjQ0FGFhYcjNzYXBYLA7LZTBYAAA9OzZ02K5+euzZ88iNzcXYWFhVmOrq6tx+PBhALC6ZV/Pnj2Rnp6uPn9d69ZoNFYXnRERNTfx8fFITk52qK+iKBg7dqyLExGRK1xUUVpZWalOw9SQPuZzOy9GVVUVxo8fj//973/w8/PDF198gcGDB1/089SWlpYGAOjatatV2/Dhw7Fq1Sps3LgRf/vb36zaRQSbNm0C8OcFXmYDBw6Ej48PKioqsHHjRkyaNMlq/O7du9ULlGqPHz58ONasWYOdO3eitLTU5gVPGzduBAD079+/3guqiIiautjYWERGRiIjI6POi500Gg0iIyMRGxvbiOmIyFkcLkqnTJniyhw2GY1G3H333Vi3bh18fHywbt26evfUAheKxroK4B9++AGrVq0CAIwaNcqqfeLEiVi1ahW2bduGtLQ09VC/2Zo1a3D06FFoNBpMmDDBoi04OBijRo3Cp59+igULFiAxMRE6nc6iz/z58wEA/fr1wxVXXGHRFh8fj9mzZ6OoqAiLFy/GE088YdF+7NgxNbutgpeIqLlRFAUpKSmIi4uzOy2URqOBn58fUlJS3JCQiJzCjXeTqpPRaFRv9+nj4yNfffWVw2NfeeUVuffee2Xjxo1y7tw5dXlBQYEsXbpUWrVqJQCkffv2kpeXZ/M5Bg8eLACkY8eOsnnzZhERqa6ultWrV0twcLAAkPvvv9/m2EOHDomPj48AkPj4eMnOzhYRkfz8fJk+fboAEEVRZPv27TbHP/vsswJAvL29ZcmSJVJRUSEiIvv375eePXsKAImKipLKykqH35OWdJsyImqeDAaDxMTEiE6nE61WKwBEq9WKTqeTmJgYMRgM7o5I5HQtafutiLjwcvlLsGPHDnVOVG9vb7Ru3brO/nq9Hp06dQJwYTqoF154QW0LDg6Gl5cXzp07p84O0K1bN3z22We48sorbT7fmTNnMGTIEGRmZgK4MDeoyWRCeXk5AGDQoEHYtGkT/Pz8bI5fvXo1Jk2apN5ytFWrVigsLISIQKPRYOHChZg1a5bNsdXV1UhISFCnNdHpdPD19VUP+YeHh2P79u2IjIys8z2pKSIiolFvAUtE5Cp6vR6pqanIy8tDaGgoxo4dy0P21Gy1pO23xxal27dvx9ChQx3u/9tvv6nTQ6Wnp2PVqlXYs2cPjhw5gvz8fFRUVKBNmzbo3bs3xowZgylTpti9t7xZaWkpkpOTsXr1avVwfY8ePTBp0iTMmDHD5vyoNRkMBrz22mvYunUrcnNz0aZNG1x//fWYPXs2BgwYUO9rWrZsGd577z0cPHgQ5eXl6Ny5M+68804kJSWhTZs2Dr83QMv6oSYiImouWtL222OLUnKulvRDTURE1Fy0pO13k5mnlIiIiIiaLxalREREROR2LEqJiIiIyO1YlBIRERGR27EoJSIiIiK3Y1FKRERERG7HopSIiIiI3I5FKRERERG5HYtSIiIiInI7FqVERERE5HYsSomIiIjI7ViUEhEREZHbsSglIiIiIrdjUUpEREREbseilIiIiIjcjkUpEREREbkdi1IiIiIicjsWpURERETkdixKiYiIiMjtWJQSERERkduxKCUiIiIit2NRSkRERERux6KUiIiIiNyORSkRERERuR2LUiIiIiJyOxalREREROR2WncHICJqCUQEer0ea9euRW5uLsLCwhAfH4++ffu6OxoRkUdgUUpE5GIGgwGJiYnIysqCiMBoNEKr1SI5ORmRkZFISUlBTEyMu2MSEbkVD98TEbmQwWBAv379kJGRgaqqKhiNRgCA0WhEVVUVMjIyEBcXh/T0dDcnJSJyLxalREQuIiJITExEWVkZTCaTzT4mkwllZWVISEho5HRERJ6FRSkRkYvo9XpkZWXZLUjNTCYTsrKyoNfrGykZEZHnYVFKROQia9euhYg41FdEkJqa6uJERESei0UpEZGL5ObmqueQ1sdoNCIvL8/FiYiIPBeLUiIiFwkLC4NW69gkJ1qtFqGhoS5ORETkuViUEhG5SHx8PBRFcaivoigYO3asixMREXkuFqVERC4SGxuLyMhIaDR1f9RqNBpERkYiNja2kZIREXkeFqVERC6iKApSUlLg5+dntzDVaDTw8/NDSkpKI6cjIvIsLEqJiFwoJiYGaWlpiI6Ohk6nU88x1Wq10Ol0iI6ORlpaGu/oREQtHm8zSkTkYjExMTAYDNDr9UhNTUVeXh5CQ0MxduxYHrInIvp/LEqJiBpJbGwsi1AiIjt4+J6IiIiI3I5FKRERERG5HYtSIiIiInI7jy5KT5w4gTfffBNjxoxB165d4evri4CAAERFRWHatGkwGAz1Psfu3bsxbtw4hIeHw8fHB506dcKUKVOQnp5e71ij0Yi33noLffv2RUhICAIDA3HVVVdh3rx5KCsrq3f88ePHMWPGDHTr1g2+vr5o27Ytbr31Vnz55ZcOvf7169dj5MiRaNu2LXx9fdGtWzc8/PDDOHHihEPjiYiIiJoM8VDHjx8XRVEEgPoIDAwUHx8f9WutVitvvvmm3edITk4WjUYjAERRFAkJCVHH+vj4yKeffmp3bHFxsQwYMMCiv7+/v/p1dHS0nD171u74HTt2SHBwsNo/ODhYzQJA5syZU+frf+yxx9S+Go3G4rlCQkJk165d9b+JNXTs2PGi+hMREZH7taTtt8fuKa2uroaI4Oabb8aKFSuQk5ODoqIilJSUQK/XY9CgQTAajZg1axY2bdpkNX7Lli2YM2cOTCYTpk2bhtzcXJw7dw4nTpzA6NGjUVFRgYkTJyIrK8vm+mfMmIHdu3cjODgYKSkpKC0tRUlJCb7++muEh4cjIyMDiYmJNsfm5eVh9OjROH/+PAYMGIDMzEwUFhaisLAQzz77LABgwYIFWL58uc3xH3zwAd544w0AwHPPPaeOPXz4MK6//noUFhZi9OjRKCgoaMA7S0REROSB3F0V23Pu3Dk5cOCA3faKigq58sorBYAMGzbMqr1Pnz4CQEaMGGFzbK9evQSAJCQkWLUfPHhQ3Uv7ySefWLXv2bNH3Wu5YcMGq/a5c+cKAGnfvr388ccfVu0PPvigAJCIiAipqqqyaKusrJQOHToIAJk2bZrV2IKCAmnfvr0AkKSkJKt2e1rSX1pERETNRUvafnvsntKQkBBcc801dtu9vb0xceJEAMD+/fst2jIzM9VlTz31lM2xc+fOBQB8/vnnKC4utmhfsWIFRATdunVDQkKC1fj+/fvjhhtuAAB8/PHHFm0ighUrVgAApk+fjlatWlmNN2fKzs7G9u3bLdq2bNmCU6dO2c3eunVrPPTQQxY5iYiIiJo6jy1KHeHr6wvgwqH+mrZs2QIACAoKwoABA2yOHTlyJACgvLwcu3btsmjbunUrAGDEiBFQFKXO8Zs3b7ZYnpGRgdOnT1v0qa1Lly6Ijo62Od687p49e6Jz5851rjs7OxuZmZk2+xARERE1JU26KDXvZezdu7fF8kOHDgEAoqOj4eXlZXNs27ZtERYWBgAWV+KLCDIyMgAAvXr1srtuc9uZM2eQn59vtW5Hx9eeBcA83pGxtsYTERERNUVNtihNS0vDunXrAAAPPPCARZv58HfHjh3rfA5zu3nPJgAUFRWph/PrGl+zreZ487pbt24NPz+/i1q3o9n9/f3V0wJqjyciIiJqippkUVpQUIAJEybAZDIhLi4O9913n0W7uaj09/ev83nM7UVFRVZj6xtfs83W+Ias2xnjzZKTkxEREaE+ap83S0RERORJmlxRWlZWhjFjxuDo0aMIDQ3FqlWr7B6ib8lmz56N7Oxs9REYGOjuSERERER2NamitKKiAvHx8dixYwdCQkKwadMmdOnSxaqfuQArLS2t8/nM7UFBQVZj6xtfs83W+Ias2xnjiYiIiJqiJlOUVlZWYty4cdi4cSMCAwOxYcMGXHvttTb7dujQAQBw8uTJOp/T3B4eHq4uCwoKUgvDusbXbKs53rzuP/74o85bkdpat6PZS0tLce7cOZvjiYiIiJqiJlGUVlVV4a677sL69evh7++PL7/8Ev3797fbv2fPngAuTM9Ue7oos7NnzyI3NxcAEBMToy5XFEWdrslgMNhdh7mtXbt2uOyyy6zW7ej4muuuOd6RsbbGExERETVFHl+UVlVVYfz48fjf//4HPz8/fPHFFxg8eHCdY4YPHw7gwkVAe/bssdln48aNAC7MdTpw4ECb4zdt2mR3cnrz+BtvvNFieXR0tLq309yntt9//12ddqr2ePO6MzIycPz48TrXHRERgaioKJt9iIiIiJoSjy5KjUYj7r77bqxbtw4+Pj5Yt24dhg0bVu+4qKgo9OnTBwAwf/58q/aqqiosWLAAADB69Giri4DuueceKIqCI0eOYM2aNVbj09LSsG3bNgDApEmTLNoURcGECRMAAEuXLkVhYaHV+FdffRXAhaLSfGcos2HDhqFDhw4QEZvZz507h3feeQcAMHHiRLuT+xMRERE1Ke68x2ldjEajJCQkCADx8fGRr7766qLGb968Wb1//fTp0yU/P19ERLKzsyU+Pl4AiK+vr2RmZtocP3nyZAEgISEhsnr1aqmurlaf13xv+mHDhtkcm5ubK23atBEAMmjQIMnKyhIRkeLiYnnhhRfUXMuWLbM5/v333xcAoiiKvPDCC1JcXCwiIpmZmTJw4EABIKGhoeprckRLuncuERFRc9GStt+KiGfePH3Hjh0YMmQIgAv3qm/dunWd/fV6PTp16mSxLDk5GXPnzoWIQFEUhISEqBcI+fj4YMWKFRg7dqzN5yspKcEtt9yC3bt3A7hwmF+j0ahXvUdHR+Pbb79V7wplK/+oUaNw/vx5AEBISAiKi4vVc1znzJmD119/3e7reeyxx7Bo0SIAgJeXFwIDA9W9rsHBwfjqq6/s3kLVloiICGRnZzvcn4iIiNyvJW2/PbYo3b59O4YOHepw/99++83m9FC7du3CwoULsWfPHhQUFKBt27YYOnQokpKS6r1IyGg0YsmSJfj4449x+PBhmEwmdOvWDePHj8ecOXPqvGMTcOHc0VdffRUbNmzA6dOnERQUhD59+uCRRx7BbbfdVu9rWr9+Pd5++23s378fRUVFCA8Px8iRI/Hkk0/i8ssvr3d8TS3ph5qIiKi5aEnbb48tSsm5WtIPNRERUXPRkrbfHn2hExERERG1DCxKiYiIiMjtWJQSERERkduxKCUiIiIit2NRSkRERERux6KUiIiIiNyORSkRERERuR2LUiIiIiJyOxalREREROR2LEqJiIiIyO1YlBIRERGR27EoJSIiIiK3Y1FKRERERG7HopSIiIiI3I5FKRERERG5HYtSIiIiInI7FqVERERE5HYsSomIiIjI7ViUEhEREZHbsSglIiIiIrdjUUpEREREbseilIiIiIjcjkUpEREREbkdi1IiIiIicjsWpURERETkdixKiYiIiMjtWJQSERERkdtp3R2AiOhSiQj0ej3Wrl2L3NxchIWFIT4+Hn379nV3NCIichCLUiJq0gwGAxITE5GVlQURgdFohFarRXJyMiIjI5GSkoKYmBh3xyQionqwKCWiOnnyXkiDwYB+/fqhrKwMJpNJXW40GgEAGRkZiIuLQ1paGgtTIiIPp4iIuDsEuV5ERASys7PdHYOaGHt7IRVFcfteSBFB7969kZGRYVGQ1qbRaBAdHQ2DwdCI6YiInKMlbb95oRMR2WTeC5mRkYGqqip176PRaERVVZW6FzI9Pd0t+fR6PbKysuosSAHAZDIhKysLer2+kZIREVFDsCglIisigsTERKvD4jWZTCaUlZUhISGhkdNdsHbtWjh6oEdEkJqa6uJERER0KViUEpGVprAXMjc3V917Wx+j0Yi8vDwXJyIiokvBopSIrDSFvZBhYWHQah27VlOr1SI0NNTFiYiI6FKwKCUiK01hL2R8fDwURXGor6IoGDt2rIsTERHRpWBRSkRWmsJeyNjYWERGRkKjqftjTKPRIDIyErGxsY2UjIiIGoJFKRFZaQp7IRVFQUpKCvz8/OwWphqNBn5+fkhJSWnkdEREdLFYlBKRlaayFzImJgZpaWmIjo6GTqdT9+5qtVrodDpER0dz4nwioiaCk+e3EC1p8l1yjvT0dMTFxdmdFsq8F9JTij69Xo/U1FTk5eUhNDQUY8eO5SF7ImryWtL2m0VpC9GSfqjJedLT05GQkOCRd3QiImoJWtL227ErGYioRYqJiYHBYOBeSCIicjmPPqe0tLQUGzZswMsvv4z4+Hh07twZiqJAURS8/vrrdY694YYb1L72Hrfffnu963/55Zdx1VVXITAwECEhIYiLi8Nbb72F6urqevMbDAZMmTIFnTp1go+PD8LDwzFu3Djs3r3bode/fPlyDBkyBG3atIG/vz969OiBpKQkFBQUODSeyFliY2Mxf/58/Pe//8X8+fNZkBIRkdN59J7Sffv24dZbb72k5wgICEBgYKDNttatW9sdd+bMGQwZMgSZmZkAAH9/f1RUVGDfvn3Yt28f1qxZg02bNsHPz8/m+NWrV2PSpEmorKwEAISEhODMmTNITU3FZ599hoULF2LWrFk2x1ZXVyMhIUGdkFyr1cLX1xeZmZl47bXXsHz5cmzfvh2RkZEOvw9EREREnsyj95QCFwrH4cOH429/+xs++eQTtG/f/qLGz507Fzk5OTYfy5cvtztu/PjxyMzMRHh4OL755huUlJSgtLQUq1atQlBQEHbu3IlHHnnE5tiMjAxMnjwZlZWVGD16NE6cOIFz584hNzcX06ZNg8lkwuOPP45vv/3W5vgXX3wRqamp0Ol0WLx4MUpKSlBUVAS9Xo8ePXrg9OnTuOOOO1BVVXVR7wURERGRxxIPZjQarZZ17txZAMi//vWvOscOGTJEAMhzzz130etdv369ABAAsmfPHqv2lStXCgDRaDSSnp5u1T5u3DgBIL1795bKykqr9ltuuUUASFxcnFXb2bNnxd/fXwDIP//5T6v2I0eOiJ+fnwCQpUuXOvyaOnbs6HBfIiIi8gwtafvt0XtKvby83LJe8x7UoUOHon///lbtiYmJ6Nq1K0wmE1auXGnRdv78efzvf/8DcGEvrU6nsxr/1FNPAQDS0tLw66+/WrSlpqaitLQUgYGBmDlzptXYbt26ISEhAQDw8ccfN+DVEREREXkejy5K3WXr1q0AgJEjR9psVxQFI0aMAABs3rzZom3Xrl3qeaTmPrUNHDgQQUFBNseb1z148GAEBATYHG/OtXfvXpSWltb7eoiIiIg8XbMvSlesWIHOnTvD29sbbdq0wYABA/Daa6/h/PnzNvvn5eUhNzcXANCrVy+7z2tuO3TokMVy89dt27ZF27ZtbY718vJCjx49AFyYB9LWeEfWbTKZkJGRYbcfERERUVPR7IvSX3/9FTk5OQgICMC5c+ewZ88eJCUloXfv3vjpp5+s+p86dUr9f8eOHe0+r7mtqKgIxcXFVuPrGluz/fTp0zbX78i6bY0nIiIiaoqabVF6ww034KOPPsLp06dRXl6OP/74A3l5eVi8eDGCg4Nx/PhxjBw5Evn5+RbjahaY/v7+dp+/ZltRUZHV+LrG1myvOdbR8fbWXVNycjIiIiLUR83XRURERORpmm1R+vzzz2Py5Mlo3749FEUBALRp0wYPP/wwtm7dCp1Oh9OnT2PBggVuTuoas2fPRnZ2tvqwN1crERERkSdotkVpXa677jokJiYCAL744guLtprFW10XEdVsM1+0VHN8fRcgmdtrjnV0vL11ExERETVVLbIoBYC4uDgAwNGjRy2Wd+jQQf3/yZMn7Y43twUFBVkUsubxdY2t2R4eHm5z/Y6s29Z4IiIioqaoxRal9oSGhiIsLAzAhXvX22Nu69mzp8Vy89dnz55Vr+Kvrbq6GocPHwYAxMTE2BzvyLo1Gg2io6Pt9nMlEcG+ffvw5JNP4oEHHsCTTz6Jffv2uSULERERNX0ttihNS0sDAHTt2tWqbfjw4QCAjRs32hwrIti0aRMA4MYbb7RoGzhwIHx8fOocv3v3bvUCpdrjzeveuXOn3UP45uft379/vRdUuYLBYEDv3r0xcOBALFiwAO+//z4WLFiAgQMHolevXlbTXBERERHVp1kWpSJSZ/sPP/yAVatWAQBGjRpl1T5x4kQAwLZt29TitaY1a9bg6NGj0Gg0mDBhgkVbcHCw+pwLFiyweX/6+fPnAwD69euHK664wqItPj4e/v7+KCoqwuLFi63GHjt2TM0+adKkOl+nKxgMBvTr1w8ZGRmoqqqC0WgEABiNRlRVVSEjIwNxcXEsTImIiOiieHxRap7KyfwwmUwALlzsU3N5RUWFOmb+/Pm47777sGnTJhQWFlo81zvvvINhw4ahqqoK7du3x9y5c63Wedttt2Hw4MEQEYwdOxZbtmwBcGGy+jVr1uCvf/0rAODee++1OnwPAC+++CJ8fHzw008/ITExUT0HtKCgADNmzMCGDRugKIpanNYUFhamZvrHP/6BpUuXqneI+v7773HbbbehrKwMUVFRuP/++xv0njaUiCAxMRFlZWXq96E2k8mEsrIy9VaoRERERA4RD9e5c2cBUO/jgw8+UMc899xzFm3BwcHSunVrURRFXdatWzf56aef7K43JydHoqKi1P7+/v7i6+urfj1o0CApLS21Oz4lJUW8vb3V/q1atVLXr9FoZNGiRXbHGo1GGTt2rDpWp9NJUFCQ+nV4eLhkZmZe1PvYsWPHi+pvS1pamuh0Ooe+HzqdTvbt23fJ62zuTCaTpKWlSVJSktx///2SlJQkaWlp7o5FREQewhnb76ZC6+qi1x3uuusuVFdXY8+ePThy5Ajy8/NRVlaGtm3bonfv3hgzZgymTJli997yANCuXTscOHAAycnJWL16NY4ePQqdTofevXtj0qRJmDFjBry8vOyOHz9+PHr27InXXnsNW7duRW5uLtq1a4frr78es2fPxoABA+yO9fLywqeffoply5bhvffew8GDB1FeXo6oqCjceeedSEpKQps2bS7pPWqItWvX1ntqhJmIIDU1FbGxsS5O1XQZDAYkJiYiKysLIgKj0QitVovk5GRERkYiJSXF6kI4IiKi5koRR6sMatIiIiKQnZ19Sc/xwAMP4P3337+o/v/9738vaZ3NlfncXHunQmg0Gvj5+SEtLY2FKRFRC+aM7XdT4fHnlJLnCAsLg1br2M51rVaL0NBQFydqmoTn5hIREVlhUUoOi4+PV2/ZWh9FUTB27FgXJ2qa9Ho9srKy7BakZiaTCVlZWdDr9Y2UjIiIyH2a5Tml5BqxsbGIjIxERkZGnQWVRqNBZGSk288nFRHo9XqsXbsWubm5CAsLQ3x8PPr27evWXDw3l4iIyBqLUnKYoihISUlBXFxcvedCpqSkuCHhnzz5IqLc3Fx1ftf6GI1G5OXluTgRERGR+/HwPV2UmJgYpKWlITo6GjqdTj3HVKvVQqfTITo62u0X53j6BP88N5eIiMgar75vIVxx9Z5er0dqairy8vIQGhqKsWPHuv0ws4igd+/eDp1iEB0dDYPB0IjpLti3bx8GDhxo825ftel0Ouzevdvt7ysREblHS7r6nkVpC9FSfqibQsHXFApnIiLyDC1l+w3w8D01Mw25iKixmc/N9fPzg0Zj+1fQU87NJSIiaiwsSqlZaSoXETWFc3OJiIgaE6++p2bFfBGRI4Wpuy8iiomJgcFg8Mhzc4mIiBobzyltIVrKOSlN4ZxSIiIiR7WU7TfAw/fUzJgn+Ld3rqaZp0zwT0RERBewKKVmhRcRERERNU0sSqnZ4UVERERETQ8vdKJmiRcRERERNS0sSqlZi42NZRFKRETUBPDwPRERERG5HYtSIiIiInI7FqVERERE5HYsSomIiIjI7ViUEhEREZHbsSglIiIiIrdjUUpERETkofLy8qAoCmJiYlBRUVFn3/z8fLRr1w6KouDBBx9spITOw6KUiIiIyEO1bt0awcHBOHToEJ5//vk6+86cORNnz55F586dsWDBgsYJ6EQsSomIiIg8lJeXF5KTkwEA//rXv7B//36b/T7//HN88sknUBQF7733HoKCghozplOwKCUiIiLyYA888ABGjBiB6upq3HvvvaisrLRo/+OPPzB9+nQAwLRp0zB8+HB3xLxkioiIu0OQ6/n4+CAsLMwlz11cXIzAwECXPLezMKNzMKNzMKNzMKNzMKNzuCpjbm4uKioqkJ2djV69eqGwsBBPP/005s2bp/aZMmUKli1bhi5duuDnn39Wc6SmpuLDDz+EXq9HQUEBQkJC0KdPHzz44IMYM2aMzfUZDAasXr0aO3fuxO+//45Tp07Bx8cHkZGRuPPOOzFr1iwEBwfbHKsoCgBg27Zt6N69O+bNm4eNGzfi5MmTiI6Oxo8//lj3ixWiS9SxY0d3R6gXMzoHMzoHMzoHMzoHMzpHY2R8//33BYBotVrZv3+/iIh8+eWXAkAURZGtW7eKiEhxcbHcfvvtAkB9BAcHW3x97733islkslpH586d1T5+fn7SunVri3Hdu3eXkydP2sxn7vPuu+9KaGioABB/f38JCAiQq666qt7Xx8P3RERERE3Afffdh1tvvRVGoxH33XcfcnNz1avsZ8yYgaFDhwIA7r//fqxfvx4xMTH4/PPPUVJSgsLCQpw/fx5vv/02goKC8OGHH9q8GGrIkCH46KOPcOLECZSWlqKgoABlZWX4/PPPERkZiV9++QXTpk2rM+ecOXMQHh6O3bt3o6SkBMXFxfj000/rfX08fE+XLCIiAtnZ2e6OUSdmdA5mdA5mdA5mdA5mdI7Gynjq1CnExMTg3Llz6jq7deuGgwcPIiAgANu3b8fQoUPRuXNn7N+/H6GhoVbPsWrVKtx9991o06YNcnJyoNPpHFr3b7/9hsjISFRXV+Po0aPo0qWLRbv58H2rVq1w+PBhtGvX7qJeG/eU0iWbPXu2uyPUixmdgxmdgxmdgxmdgxmdo7EydujQAYsWLQIAZGdnQ1EUvP/++wgICAAAvP/++wCAe++912ZBCgDjxo2Dj48PCgoK8P333zu87q5duyImJgYigj179tjtN3ny5IsuSAHuKSUiIiJqcq655hr8+OOPGD16ND777DN1+RVXXIEjR44gJCQEvr6+dsfn5ubCZDIhJSUF48ePt2hbt24dli9fju+//x5nz55FWVmZ1fh//etfmDt3rsUy857STz75BImJiRf9mrQXPYKIiIiI3CokJMTiX7PTp08DAAoLC1FYWFjv85SWlqr/r66uxoQJE7B69Wp1mU6nQ5s2bdRD/AUFBaiqqkJJSYnd52zobD88fE9ERETUTFRXVwMAli9fDhGp93HvvfeqY9977z2sXr0aXl5eeP755/Hrr7+ioqIC+fn5yMnJQU5ODuLi4gAAdR1o9/LyalB27iklIiIiaibatWuH48eP4/jx4xc9NiUlBQAwdepUPPfcczb7nDlz5pLy1YV7SomIiIiaieuvvx4AsH79+osee+LECQAXzle15ffff8evv/7a8HD1YFFKRERE1Ezcf//9AIC9e/fik08+qbPvH3/8YfG1+fzUn3/+2Wb/p59+us7D9peKRSkRERFRM3HTTTchISEBwIWpmZ599lmcPHlSbS8uLsbWrVsxdepUDBo0yGLsLbfcAgB499138f7776OyshIAcPz4cUyZMgWffPIJWrdu7bLsLEqJiDyAyWRydwSHmC+O8ETV1dVWe348mae+j9T0ffDBB7j77rthNBrx0ksvISIiAiEhIWjVqhWCg4MxfPhwvPfee6ioqLAYN2fOHHTv3h1GoxEPPPAA/P390bp1a3Tu3BnLli3Diy++iCuvvNJluVmUktOZN67mD9zaX9PFqf0+ehLzVZ5mnpjRzNN//jQajUe/f3l5ecjLy4OiKOpchJ7mhRdewBtvvOHuGA7z1PfRzJM/e7idqZufnx9WrlyJzZs34+6778bll1+OiooKlJeXo1OnTrj11luRnJyMb7/91mJc69atsXfvXsycOROdOnWCRqOBTqfDiBEj8OWXX+KZZ55xaW5Onk8uYTKZkJeXh1atWsHb2xsmkwkajUZtM/+fHFNdXa1OseFp719FRYV6uzk/Pz8AFzYMnrDBzcnJweHDhzFw4EBotRcmG/GUbGbp6el4++238dprryEwMNDdcWzKycnBP//5T+zfvx9z5sxBfHy8uyNZMRgM6h6cl156CU8++WSDp6VxpbKyMhw4cADff/89+vXrh06dOiE8PNzjfq/NjEaj+rtj5im/Q9zOND+cEoqc6tixY/j444+xadMmFBcX48yZM7j99tvRtWtX9OvXD0OHDlU/KDzlg81TrVmzBt988w2Ki4uh1WoxatQo3HXXXR7zQXvs2DG88847+OKLL9TzjubMmYOHHnrIY76vkydPxqlTp3Dvvfdi5MiRiImJUbN5ys/fo48+iq1btyIyMhKPPfaYu+PY9MILL+Ddd9/FwIEDERQUZLOPu9/PRx99FACg1Wrxzjvv4KqrrsLtt9/utjy2fP7551i8eDG2bNkC4MLerIkTJ2LevHl2bwfZ2MrLy7F69Wps3rwZ586dw5kzZ3DzzTejV69eiIqKwtVXX+323yFuZ5oxIXKSXbt2Sf/+/UVRFFEURQICAtT/+/n5SYcOHSQhIUH27Nmjjqmurm7UjD///LPs2LFDiouLG3W9F+Onn36SBx54QH3vaj7uueceOXbsmIiImEwmt2X87rvvZNiwYRbfX/P///a3v0l5eblb84mIrFu3Ts0UHBwsd9xxh7z//vty8uRJtY+nZOzdu7eUl5fXmamxf1fMvvzyS1EURcLCwiQrK0tdbjQa7Y5p7Kzm97FDhw4SFRUliqLIX/7yF9m7d2+j5qjLtm3bpGPHjqLRaMTX11euuuoq9efzzjvvlMLCQvV9c9fPpV6vl/j4eJufPWFhYTJo0CCZM2eO/Pjjj+qYxs7K7UzzxqKUnObaa69VP2DffvttWbdunaxZs0bi4+MlKipK/fAICAiQRx99VE6dOtXoGUNDQ6VLly6ycOFCOXTokFRWVjZ6hvrcfPPNoiiKtGnTRiZNmiQJCQly8803S1BQkAQGBsorr7zi7ojqRmH48OGycOFCWbp0qUyePFm8vb2lS5cuauHsTg8//LAoiiKhoaHi7e0tiqJI586dZerUqfLFF1/I+fPn1b61C6zG2tB269ZNFEWRlJQUERGpqqqy6lNdXS1FRUWNkseWa665RhRFkWXLlomIZUaTySS7du2S1157TT744ANZtmyZxe91Y7+P69evlx9//FHat28viqLITTfdJEeOHBER9xX1Zr179xZFUeTxxx+XnTt3Smlpqaxbt04iIiKkdevW8uuvv4rIn+9ZRUWFOrax3kfz73Xfvn3lueeekxdeeEFeeukl9f1VFEUCAwPlyiuvlNdee03y8/MbJVdN3M40byxKySkWL14siqLIoEGDbLbv3r1b/vGPf0i/fv1Eq9WKoigyePBg+fLLLxvtA/eNN96w+Mv/tttukxUrVsiJEycaZf2OWLBggSiKIldeeaX89NNP6vIff/xRJk2aJIqiiLe3t3zzzTduy5icnCyKokifPn3kjz/+UJcbDAaJjo4WRVFk48aNUlJSIj/++KMsX75c0tPTJScnp1Fznjp1SqKjo6VHjx7y6quvSkxMjPq9v/rqq+Wpp56SvXv3WhQry5Ytk+zs7EbJ9+qrr4qiKDJy5Eh1Wc3i+Ouvv5ZHH31UbrrpJrnuuuvk0UcfFb1er+59aYzfm/Xr14uiKHLdddepy8xF6YYNGyQxMdHid6p169YSFRUlCxcutFlgu4L5fRwxYoS6bOnSpere+4ceesjte8TNnz22Ph+nTp0qbdu2lbVr18qGDRvkrrvukgkTJsicOXPkk08+abT3ceHChaIoigwcOFDKysrU5SaTSX7//XeZNWuW+sed+Y+9mTNnyi+//CIide85dxZuZ5o/FqV0yUpKSmTgwIGiKIrs2LFDREQ9FFnzg6qyslK2bdsmM2bMkIiICNFoNHLDDTc0yiG2oqIidY/PgAED1A+MoKAgmTp1qnz99ddSUFDg8hx1KSgokFatWomiKOqhp5obpKKiIrn11ltFURSZPXu2iDT+3p8//vhDwsPDRVEU9ftWc4/OXXfdJYqiSHJystx2222i0WjU9/n+++9Xfz5czWg0SnV1tTz11FOiKIqkpqaK0WiUf/zjH3LZZZeJoiji6+srQ4cOlQULFsjJkydlz5496h6WwsJCl+bLzc0VnU4niqLId999JyJ//s7k5ubK3Llz1Y1qzUerVq3k6aeftigaXGnFihXi5eUlL730koiIlJaWiojIsWPHpGvXrqIoimi1WomLi5OwsDBp27atmvXuu++22vvnbDXfx3379qnLKysrZc6cOVanlLgyiz3nz5+XyMhIi9/ryspK9Xf3ww8/lLZt20psbKwEBwermXU6nYSGhspf//pXyczMdGn28+fPS/fu3UVRFElLSxORP38ezY4fPy6DBw8WRVGkU6dO4uXlJX5+fupnkatxO9MysCilS2IymaS0tFQGDBggISEhcvDgQamurrb68Kz5dV5eniQnJ6uH2K688kr1Q9dVvvzySwkPD5crrrhCMjMzJT09XW666Sb1Q6Nbt27yj3/8Q/R6vbrhtZW/vLzcZXsEFi1aJIqiyF133WWxTpE/P3T/85//qOfO1d5o1C5QXVGwmvfkjh8/3iKjeV3jxo0TRVGkS5cu0r59e7nzzjulX79+6vv8l7/8xeJcL1c7deqUXHbZZXL55ZerxXNmZqZMnjxZLZhDQ0Nl/Pjx6l7ef/zjHyJi+1C6s9x///3qoVyRCxtS8/fYfNqBv7+/3HLLLfLwww/LM888I3Fxcer7OHjwYDl+/LjL8pktXbpUFEWRuXPnisif3+fx48er5zibi8Hjx4/Lf//7XxkzZowEBQVJcHCwzJ8/36X5zO/jww8/LCIXfh7NP5MlJSUydepUURRFIiIiZN26dS7NYs+2bdskNDRU3dts/j6b/3388cfV3+kbb7xR3nnnHXnooYekT58+4uvrK61atVJ/Jl1l165dEhoaKv369RMR+58dX3zxhfj7+8usWbMsznt/4oknLAptZ+N2puVgUUqXrKysTP0re+3atQ6P27Rpk3To0MFio+cKVVVV6iG+G2+80WJj/tlnn6l7fBRFkf79+8vbb78tWVlZFh8K5g+LRx99VJYsWeL0E9jPnz8vEydOFI1Go248bX3AV1RUyF/+8hf1ELktixcvdsk5nfn5+XLbbbdZ7JWqrq5W36eDBw+q7+Orr75qkSElJUX9Xt9yyy2Nco6V+XtmPiz59NNPW+zV/frrr2XQoEEWeyIDAgLkq6++cunetP3796vr+8c//mHxXmzatEm9YGP9+vXqHtHKykopKCiQV199Vdq0aSM6nU7eeustl2U0+9///ieKosi4cePUZfv27RNFUaRdu3Y2z3XNyMiQCRMmqK8xPT3dJdnS0tLUHKdPnxaRPws98/fvwIED6h8brVu3Vj+fGnOD//3334tWq7V5rvWxY8fUQ+LLli2zOB3mxx9/lISEBPV9dOVRhoMHD4q3t7d0797d7ukr1dXV6s+u+bz2RYsWSVBQkPTs2VN+++03l+UT4XampWBRSk7xyCOPiKIo0rNnTzl48KCI2P9ru+byjz76SDQajXTo0MFlf8UajUb5+9//bnHYu3ZRNH/+fIvDpfHx8bJmzRo5deqUmnfFihWiKIqEhIRYFDfOUF5eLtdee60EBgbKtm3b7L4OEZFp06apRZbIhQ8yc9uWLVvUw0XO3tN36NAhiY6OlkGDBsnZs2et2u+44w5RFEVmzpxpc/w777wjPj4+EhYWpv6MuJL5Az4rK0vde2EuXmp68803RafTqd///v37y7PPPmtxONiZ3n//fYmMjBQfHx/RarUyaNAg9UKnESNGiKIo8u9//1tErH+H8vLy1L3R11xzjcsvgDp+/LhERESIn5+frFy5UkT+LFTNe0HNv0s1C/mSkhK5/vrrRVEUeffdd12SzXwod+HChSJiv9A0GAwSGxsriqLI7bff3uiHT/fv3y8BAQHi4+Mjixcvlry8PBEROXv2rNx+++2iKIr89a9/VfvXPhRtfp0vv/yyyzIaDAZp06aN+Pv7y+eff64uN//8mb/HGzZsUE+HqK6ulrNnz8rQoUNFURR58cUXXZbPjNuZ5o9FKTlFVlaWREdHi6+vrzz66KMWF2PUdUgnPz9frrzySlEURXbu3OmyfD/++KO8/vrrcubMGRH58wOrZuF29uxZuffee9UPjDZt2siMGTPk22+/ld9//12uuOIKl21kCwoK1A1nfefhffjhh6IoigwZMkRELDdi5g3YkiVLnJ6xvLxcXn75ZXn++eetzmk070Fp27atWijVnt4mJydHLr/8clEURXbt2uX0fHUxH4aeNm2ams18+sP8+fPVQ7zmc/pCQkKkT58+8vPPPzs9S1lZmWzZskVmzpypXtUcHBwsN954o7rBNbP1M/DDDz9IUFCQ+Pj4uHzvlNFolJkzZ4qiXLiwbffu3bJx40ZRFEXefPNNtU/tMSaTST20bu7n7L3PO3fulMcee0x9XlvPbzKZpKqqSr1ARlEUueGGG9TfscY6J/vOO+8URVHksssuk6lTp8qECRPUcw8VRZEDBw5Y5TH/sWke+89//lNd7gojR45UzxfdtGmTVXtlZaV65bv5DxQRkW+++UYURZE77rij3s/7S8XtTPPHopQuWXV1tVRVVckzzzyj/qINGDDAYi4784aqJvNfgXfccYf4+/vL119/7dKc5vXZOg+p5oY1LS1N3ctjLhLM5wVdffXVLsv322+/yf/+9796/zo+ceKE+Pv7i0ajsTgcaD7f9Morr3RZRhGxefV3bm6uLFiwwO7hUaPRKDk5OdKrVy/p2LGjxc9GY8jOzpZrrrlGQkNDLQrNzMxM9fuclZUlJ06cUM+X7NKli9Nz1D7nbdWqVTJ+/Hj14itFUWTFihUiYn/P3w8//CDBwcESFRUlubm5Ts9Y24kTJ6Rv376iKIpcddVVMm/ePOnQoYPcd999ap+ar8u8d+iJJ55QT+VwNUcOx7/77rsSFhYmwcHBaoHXWA4ePKj+0VnzMWLECOnZs6f88MMPVmPMhYz5vNh58+a5JJv5e7dz5071j6TWrVvL1KlT5aeffpJffvlFUlNT5Z577lGL+pq2bdumvhZX4namZWBRSk61aNEiad26tSjKhamLkpKSLM6TMjNvuIqKiiQgIEB8fX1dslfqYtT+QFu+fLl6pbn5opitW7e6NENZWVmde0LMf3mbP7zeeecdEblwVbx5L6SrMta3B6Rm7pr/N4/7/fffRafTSVRUlDp3ZGNasmSJKIoiDz74oLrMfM7elClTLPp++umnotfrXZKj9vf3119/lbfeekuGDBki3bt3t3vuoPl9zMjIEJ1OJ9ddd12jTTNT87y8mo8333zTYq+5eaNbXFwsoaGhoiiK7N+/v1Ey2mN+306dOiVjxoxRs7/yyiuNOn9keXm5fPDBB3LnnXfKE088IWvWrFFnfPjss89E5M+Cxvw+FhQUqFNbff/99y7P+PHHH0u/fv0spn4yn8NuLpbMRznMRxr27dsnWq1WJk6cKCUlJS7PKMLtTHPGopScwvwhWllZKStXrpQhQ4aoH2SdOnWSN954Q4qKiiyuODQajTJjxgxRFEUmTpzoruhWap+LaT7MNnr0aDcl+pP5fX7uuedEURSZPHmyiPy5V2rs2LHujGeT+QPYfJWx+UrpxmY0GmXw4MHi5eUlP/zwg3rY0cvLSz3cVntGA1eqXZzu27dPlixZYrfQrHkRhKIo8uijj7o6ooWMjAx1yi/zBrRHjx7y97//3WK6nbNnz6qH7idMmNCoGetTXl6uzq3ao0ePRpuizN4fmuafwUmTJtlsN8/EcPfdd7synoXNmzfLQw89ZHHXpMDAQLnnnnssZs6oPVvEs88+6/Js3M40fyxKyekqKipk+/btMnPmTPVqSfOHxvTp0+Whhx6SZ555Rj0/KSwsrFGmt7kY5g+/nTt3qvk94S5FZuYLDvr16yfff/+9OlejJ2UU+XMv1cGDB9W9AO74Xpu/n+ZzC//617+qt3k0X7DjrilYbB36tten5gwHjfk+mtf/yy+/yOuvvy5XX321miMkJESuvfZaGT58uNx3333qHvvOnTt71ITh5u/v9u3b1XlDL7/8cptT87hK7fOsz5w5o85NHBsbK1999ZVkZ2fLhg0b1D/i/Pz85Pfff3d5tpo/h0VFRXLo0CHZt2+frFq1So4dO2ZziroffvhB/cPO2d/r2kdmbB2Wd/d2pr6M9fH07Yw7sCili+boiexnzpyRtWvXyqxZsyxuU1fzMXz4cFm9erXbMtb3HObJol0xT+ClZCwtLVUPp5o3sE899ZQT013Q0Iy1r8Q2TyTtiiuILyZjze9p7fNGXTkNlDPex3Pnzqnvo3kye2dyNGNRUZHo9XqZP3+++rNXe6L/8ePHy4YNG9yWsT7bt28XrVYrjzzyiFOer6aLzbhs2TLp2LGjxeFy8yH7sLAwWbp0aaNmtPd7UHte0OzsbPWcSFf8XldVVcmJEydk48aNkpmZafOUIBH3bmfqyugoV29nmhoWpXRRau7JsffBVvsXs6CgQI4cOSLvvfee3HPPPTJ16lS555575L///a9L5mFzJKMjPvroI3XD4Oy9aJeS0Zzl7rvvVvc+elpG88/Atm3b1Dkrb7rpJqdfmXsxGc2Z1q1bJ126dBFFUWTNmjUi4tqJ8p3xPn7zzTfq+a+DBw926/tYM5vJZJK1a9fKvHnz5LnnnpNnnnlGvvrqK6dmu5SMtdX8bHrvvfc84n0sLCyU//znP3LrrbeKv7+/xYUvy5cvd/ofSw35nanNYDDI6NGjRVEu3CbX2e/jgQMHZOLEiRIaGiq+vr6iKBduKrJ9+3a72Rt7O+NIRke4cjvTFLEoJYctX75cbrrpJqv7rjtanNrr68wP3YvNWJeysjKZMGGCfPTRR86KJyKXntH8fr311lvqBszTMopc2KNmvnL4lltucfoFL5eSsbS0VP7zn/84NY8tzngfy8vL1fP2+vXr5/TbJTrzd6YmT/29dpVLyVhdXS3p6eny7bffSnJysmzfvt3mnLruzFiTwWCQv/3tb/LYY4/JTz/95MyI8tVXX6nTNynKhanazP/38/OT9957z6K/O7YzF5uxLq7azjRVLErJIeXl5RIYGKgeLn788cct7tRi65ZvtZk/KFy1IXFGRjNXHcp1ZsaCggKZOXOmeltST8tYVVUlP/zwg6xZs0YOHTrkMRlr743w9J/H6upqdbow833JPSlj7VvNOpszf2fMnP37fSkZL3a5OzLaUlJSIufOnXN6RvP5oWPHjpVPPvlE9u7dK1988YV66spf/vIXOXr0qN3naIztzKVmNHPlKUNNFYtScsgrr7wiinJhkm9vb2/R6XRy7bXXyuuvv24xV6I791w4O6MrXouzM+bn50thYaFHZ3QFZmTG5pjRla/Bme+jq3I+9dRToigXJuKv7cCBA9KzZ09RFEVmzZqlnj7S2Jyd0ZP29nsCFqVUr/z8fPVE7GnTpsnUqVOla9euotFoJDAwUG6++WZZtWqVxR6ohp6rxIwXl9HZWur7yIzMyIzuzZidna3OPZqVlSUi1rNRvP322+Lt7S0xMTGNcuOIppixqWNRSvXasmWLXHHFFRIRESE7d+4Uo9Eoq1evljFjxkhYWJgoiiLt2rWTyZMnW93CreYhoUOHDlnMc8eMzMiMzMiMzCjy580tzHPb2iqKT5w4IZ06dRJFUWTjxo0WbeaC+ty5c+q8wy0xY1PHopTqZDKZ5M033xRFUaR///5y8uRJte3UqVOyaNEiGTRokAQEBIhGo5Hu3bvLk08+Kb/88ovF85SVlcn1118v7du3l++++44ZmZEZmZEZmVFELuxtnDNnjnh5eal3qbPnkUceEUVR5IknnhAR68Jw8uTJMmnSJKfPSdoUMjYHLEqpXlu3bhVF+fNOPFVVVRaHbn7++Wd56qmnpFevXqLT6cTb21vi4uJk8eLFkp+fLyIXDmmYJzZmRmZkRmZkRmasaejQoaIoiqxcudJmu7mw++yzz9QC23xbU/OUbp9++qm617elZmzqWJSSw8y3YLR3dePmzZvVO7poNBoJCQmRO+64Q9577z257LLLRFEUWbt2LTMyIzMyIzMyo4j8eW7q66+/LlFRUfL999/bzGZ27Ngxufzyy8Xf318OHDigLq+srJSYmBhRFEVSU1NbXMbmgkUp1ct8Inddd/owKyoqkuXLl8ttt90mbdq0EUVR1BPDhw0bxozMyIzMyIzMaFNhYaEUFBTU2cdkMsmYMWNEURRZvHixuvzVV18VRVFk6NChLT5jU8ailJym5gfeb7/9Jm+88YZ6f3FFUeTHH390Y7oLmNE5mNE5mNE5mNE53JXRZDJd1N2MkpOTRVEUdY7mo0ePSqtWrURRFKdP5t+UMjYHLErJIY7+Mtael23QoEGiKIo8+OCDroqmYkbnYEbnYEbnYEbnaA4Zzbn27NkjXl5e0q1bN6mqqlLveDZt2jRmbOJYlJJNJpNJqqur7U7MXtccdeZDQeYTuv39/Z1+5w9mZEZmZEZmbJkZ8/PzJSYmRoKCgtTJ7FtyxuZEC6IaSktL8d133+GTTz7B3r17cdlll6G6uhqJiYmIjo5Gjx490LFjRyiKAhEBACiKYvEcGo0GZWVleOqppwAAr776KkJCQpiRGZmRGZmRGS85Y5s2bTBw4ED8+9//xuLFiwEAL7/8covL2Cy5pxYmTzVt2jT15HZFUcTX11f9/xVXXCETJkyQDz74QPLy8tQxtv5S3LNnj7Rt21auueYaZmRGZmRGZmRGp2Q0/7tixQp1TFRUVIvM2BwpIv9f4lOLt3DhQsyZMwcREREYNWoU+vfvj5KSEpw8eRL//ve/cfbsWQBAREQEhg4dismTJ2Po0KHQaDQ2n2/btm1o3749oqOjmZEZmZEZmZEZnZbxl19+wfXXX4/8/Hx88cUXuO2221pUxmbL3VUxeYa8vDwJDAwURVFk165d6nLzuUWlpaWSnJysXomp1Wpl4MCB8tFHH0lpaalFX2ffE5kZmZEZmZEZmbFmXxGR8+fPy9KlS1tcxuaMRSmJiMjChQtFURS55557RMTyfsc1rzY8ceKEJCUlSWhoqCiKIp07d5aPP/6YGZmRGZmRGZmx0TKa75DUEjM2ZyxKSUREXnvtNVEURV544QUR+XOyZbPa04R8++23FvPXLVmyRO3HjMzIjMzIjMzo6oyuyNoUMjZntk8koRbHZDIBANLT0wEAOp3Ool1RFCiKovYbPHgwdu3ahXvvvRdarRYrVqzAuXPnrK7iZEZmZEZmZEZmdEVGV2RtChmbNXdXxeQZvv76a1EURUJCQuTLL78UkT/nZ7PFfBhj9+7d0q5dO1EURVasWMGMzMiMzMiMzMiM1CAsSklELhyiMN+r9+qrrxa9Xq+22ftlNHvyySdFURSZPn06MzIjMzIjMzIjM1KD8PA9AbhwiGLmzJkIDw/HTz/9hBEjRuDNN99EcXGxOs2F+XCFWVVVFQAgOjoaGo0Gvr6+zMiMzMiMzMiMzEgNwqKUVEOHDsXKlSsxYMAAFBQU4KWXXsKMGTOwefNmAFB/Iaurqy3G7d27FyaTqVHuVMGMzMiMzMiMzNjSMzZb7t5VS+5R+4rA6upqMRqNUllZKZ9//rncfPPN4uPjIzqdTqKjo+Xhhx+2mLPNbNeuXaIoiuh0Ojl16hQzMiMzMiMzMiMzUoPwjk4tlIigqKgIJSUlAIDw8HCL9iNHjmDJkiVITU1FdnY2TCYTdDodBg8ejBEjRqCwsBDHjx/HN998g9OnT2P+/Pl44oknmJEZmZEZmZEZmZEaxh2VMLnX6dOn5YknnpArr7xSunfvLl27dpXHH39cCgsLrfpu375dHnzwQbnuuuskICBAnYvN/GjdurU8/vjjTp+LjRmZkRmZkRmZsaVnbGm4p7SF+frrr/HKK69gx44dVm3dunXDf/7zHwwdOhSVlZXw9vYGcOGk7n379iE7Oxs7d+7EwYMHERoaii5duiA+Ph7XXXed2pcZmZEZmZEZmZEZqSFYlLYg5eXluPbaa3H48GH07dsXEyZMgE6nw6lTp7BmzRpkZWUhMTERK1asUCf8NZlM6kndZhUVFfDx8WFGZmRGZmRGZmRGch737qilxvTEE0+IoigycuRIKSoqUpfn5eXJm2++aXWbNFtcfes0ZmRGZmRGZmTGlp6xpWJR2kIcO3ZMfH19RVEU+eWXX0TE+p6+jz/+uCiKIrfddpvV/X2ZkRmZkRmZkRmZkVyJRWkL8dprr4miKDJ16lQRsbwzhfn/27ZtE0VRJCAgQLKzsy3Gm2+lZv6XGZmRGZmRGZmRGcmZOHl+C1BSUoKMjAx4e3tjxIgRAKCeJwP8ORHwDTfcgH79+qG0tBTffPONxXN4eXkBAKZPn453333X6o4WzMiMzMiMzMiMzEiXxN1VMbmeyWSSqKgoURRFvvnmG5t9zH/1zZ8/XxRFkYSEBPWvxqqqKhERWb16tSiKIpdddhkzMiMzMiMzMiMzklNxT2kLUFVVhcGDByMoKAidOnUCcGHC4JrMf/31798fgYGBOHDgAM6fPw8RgVarRWVlJZ5//nkAwH//+19mZEZmZEZmZEZmJOdqvPqX3O3IkSNSWlpaZ59z585Jt27dRFEU+e6779Tl8+bNE0VR5MYbb2RGZmRGZmRGZmRGcjoWpWRl0qRJoiiKPPPMMyIicvToUQkODhZFUcRgMLg53QXM6BzM6BzM6BzM6BzM6BxNIWNzw8P3pDKfsH3TTTcBgHqni3nz5qGoqAgzZsxATEyM2/IBzOgszOgczOgczOgczOgcTSFjs+Xuqpg8z08//SStWrWSyMhIWbZsmTo1xvnz590dTcWMzsGMzsGMzsGMzsGMztEUMjY3LErJSnV1tVx33XXq1YWKosiiRYvcHcsCMzoHMzoHMzoHMzoHMzpHU8jY3LAoJQvmO1ckJSWJoly41Vp0dLSbU1liRudgRudgRudgRudgRudoChmbI55TShaU/59IePjw4QgODgYALFy40J2RrDCjczCjczCjczCjczCjczSFjM2RIlJrki6i/5eamorDhw/j73//u7uj2MWMzsGMzsGMzsGMzsGMztEUMjYXLEqpTiJicRs2T8SMzsGMzsGMzsGMzsGMztEUMjYHLEqJiIiIyO14TikRERERuR2LUiIiIiJyOxalREREROR2LEqJiIiIyO1YlBIRERGR27EoJSIiIiK3Y1FKRERERG7HopSIiIiI3O7/AGIIFih2cKFLAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['font.size'] = 22\n", + "\n", + "fig = plt.figure(figsize=(8, 6), dpi=80)\n", + "ax = fig.add_subplot(111)\n", + "ax.plot(price_df.index, price_data['Price'], 'ko', markersize=10)\n", + "ax.set_xlabel(\"Year\")\n", + "ax.set_xticks(price_df.index)\n", + "ax.set_xticklabels(price_data['Year'], rotation=60)\n", + "ax.xaxis.set_label_coords(1.05, 0.015)\n", + "ax.set_ylabel(\"Price\")\n", + "ax.set_ylim(12000, 42000)" + ] + }, + { + "cell_type": "markdown", + "id": "6d94511c-d2ac-4f74-9820-08bd3dd41554", + "metadata": {}, + "source": [ + "从图形上看,房价大致与年份成线性关系,特别是2017年-2021年间。\n", + "\n", + "因此,选用线性回归模型进行预测大体上是合理的。" + ] + }, + { + "cell_type": "markdown", + "id": "e59521ef-f904-4d6d-b098-7f97f539151c", + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "source": [ + "### 最小二乘解" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "2e118155-96e6-4a04-beef-d838da36010d", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "

\n", + "\n", + "
class LinearRegressionLS(LinearRegression):\n",
+       "    """\n",
+       "    solve linear regression problem via least squares\n",
+       "    """\n",
+       "    X: np.array\n",
+       "    Y: np.array\n",
+       "    w: np.array\n",
+       "    \n",
+       "    def __init__(self, ylist):\n",
+       "        num_data = len(ylist)\n",
+       "        self.X = self.homogeneous([x for x in range(num_data)])\n",
+       "        self.Y = np.array(ylist).reshape(num_data, 1)\n",
+       "        self.w = np.random.rand(2)\n",
+       "        \n",
+       "    def homogeneous(self, xlist):\n",
+       "        """ build homogeneous coordinates """\n",
+       "        raise NotImplementedError\n",
+       "    \n",
+       "    def linout(self, xlist):\n",
+       "        """ linear output for given data """\n",
+       "        raise NotImplementedError\n",
+       "    \n",
+       "    def loss_sq(self, X, Y):\n",
+       "        """ loss function: (half) sum of square errors """\n",
+       "        raise NotImplementedError\n",
+       "        \n",
+       "    def solve(self, lr, nepoch):\n",
+       "        """ form normal equation """\n",
+       "        raise NotImplementedError\n",
+       "
\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import sys\n", + "sys.path.insert(1, '../')\n", + "from utils.utils import *\n", + "from LinearRegression_2 import *\n", + "\n", + "\n", + "psource(LinearRegressionLS)" + ] + }, + { + "cell_type": "markdown", + "id": "be5afa50-d1e3-4519-ae98-300d5f526077", + "metadata": {}, + "source": [ + "#### 实现要点" + ] + }, + { + "cell_type": "markdown", + "id": "ca886ab0-47e1-4f43-a5e6-4c8b908666b4", + "metadata": {}, + "source": [ + "```py\n", + "def linout(self, xlist):\n", + " \"\"\" linear output for given data \"\"\"\n", + " return 自变量 * 权重 + 偏置\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "6feb8387-9f9d-40b3-90fe-a468eb7d6a76", + "metadata": {}, + "source": [ + "```py\n", + "def homogeneous(self, xlist):\n", + " \"\"\" build homogeneous coordinates \"\"\"\n", + " xlist 转换成列向量\n", + " return xlist + 全是1的列向量\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "6207cb3b-c29e-4696-bb16-cdf810bd27f3", + "metadata": {}, + "source": [ + "```py\n", + "def loss_sq(self, X, Y):\n", + " \"\"\" loss function: (half) sum of square errors \"\"\"\n", + " return ((预测值 - 真实值)的平方)求和后除以2\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "9aaac662-d290-43af-8375-c173216ba30e", + "metadata": {}, + "source": [ + "```py\n", + "def solve(self, lr, nepoch):\n", + " \"\"\" form normal equation \"\"\"\n", + " XtX = X转置 * X\n", + " XtY = X转置 * Y\n", + " 权重W = XtX求逆 * XtY\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "1ef5e483-6592-4991-80f5-1a3b4ff007c8", + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "source": [ + "### 梯度下降法" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e6589f47-5922-42c8-8221-c0c34347d021", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "

\n", + "\n", + "
class LinearRegressionGD1(LinearRegression):\n",
+       "    """\n",
+       "    solve linear regression problem via gradient descent,\n",
+       "    using single weight vector: homogeneous coordinates\n",
+       "    """\n",
+       "    X: np.array\n",
+       "    Y: np.array\n",
+       "    w: np.array\n",
+       "    \n",
+       "    def __init__(self, ylist):\n",
+       "        num_data = len(ylist)\n",
+       "        self.X = self.homogeneous([x for x in range(num_data)])\n",
+       "        self.Y = np.array(ylist).reshape(num_data, 1)\n",
+       "        self.w = np.random.rand(2)\n",
+       "        \n",
+       "    def homogeneous(self, xlist):\n",
+       "        """ build homogeneous coordinates """\n",
+       "        raise NotImplementedError\n",
+       "    \n",
+       "    def linout(self, xlist):\n",
+       "        """ linear output for given data """\n",
+       "        raise NotImplementedError\n",
+       "    \n",
+       "    def loss_sq(self, X, Y):\n",
+       "        """ loss function: (half) sum of square errors """\n",
+       "        raise NotImplementedError\n",
+       "\n",
+       "    def gd(self, lr):\n",
+       "        """ gradient descent update """\n",
+       "        raise NotImplementedError\n",
+       "        \n",
+       "    def solve(self, lr, nepoch):\n",
+       "        """ iterative solver """\n",
+       "        for epoch in range(num_epochs):\n",
+       "            self.gd(lr)\n",
+       "            print(f'epoch {epoch + 1}, loss {self.loss_sq(self.X, self.Y)}')\n",
+       "
\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "psource(LinearRegressionGD1)" + ] + }, + { + "cell_type": "markdown", + "id": "9b4ae99c-d68c-451b-836e-edb463ffea02", + "metadata": {}, + "source": [ + "#### 实现要点" + ] + }, + { + "cell_type": "markdown", + "id": "910185b9-0fd7-4911-af7a-9c803bb2c44a", + "metadata": {}, + "source": [ + "```py\n", + "def gd(self, lr):\n", + " \"\"\" gradient descent update \"\"\"\n", + " def gradient(Y_hat, Y, X):\n", + " return ((预测值 - 真实值Y) * 自变量X)求和\n", + "\n", + " 预测值 = linout(X)\n", + " 梯度值 = gradient(预测值, Y, X)\n", + " 梯度下降更新:权重W -= 学习率lr * 梯度值\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "d4845484-98dc-40f1-8ffe-0c31580f1970", + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "source": [ + "### 学习率测试\n", + "\n", + "学习率的取值可以决定优化过程是否收敛。\n", + "\n", + "- 尝试将学习率改成$0.03$并重新运行,观察误差的变化并解释原因" + ] + }, + { + "cell_type": "markdown", + "id": "a2c6a000-43ee-4d5c-bf31-1bdd642f9c6f", + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "source": [ + "## 线性回归的从零开始实现" + ] + }, + { + "cell_type": "markdown", + "id": "939eb9d6-7b43-4703-94a5-4d9488df3c05", + "metadata": {}, + "source": [ + "我们将从零开始实现整个方法,\n", + "包括数据流水线、模型、损失函数和小批量随机梯度下降优化器" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "de28883e-7e4d-45bf-b7fa-ce618d1b131b", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import random\n", + "import torch\n", + "from d2l import torch as d2l" + ] + }, + { + "cell_type": "markdown", + "id": "20bb6de7-a972-4798-b521-c05b7a6f3e43", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "根据带有噪声的线性模型构造一个人造数据集。\n", + "我们使用线性模型参数$\\mathbf{w} = [2, -3.4]^\\top$、$b = 4.2$\n", + "和噪声项$\\epsilon$生成数据集及其标签:\n", + "\n", + "$$\\mathbf{y}= \\mathbf{X} \\mathbf{w} + b + \\mathbf\\epsilon$$" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "e26dcefc-d894-4e45-9ae2-e1f5004d0645", + "metadata": { + "origin_pos": 7, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def synthetic_data(w, b, num_examples): \n", + " \"\"\"生成y=Xw+b+噪声\"\"\"\n", + " X = torch.normal(0, 1, (num_examples, len(w)))\n", + " y = torch.matmul(X, w) + b\n", + " y += torch.normal(0, 0.01, y.shape)\n", + " return X, y.reshape((-1, 1))\n", + "\n", + "true_w = torch.tensor([2, -3.4])\n", + "true_b = 4.2\n", + "features, labels = synthetic_data(true_w, true_b, 1000)" + ] + }, + { + "cell_type": "markdown", + "id": "54118bf5-96e2-412c-8ea1-4d17ff07389a", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "`features`中的每一行都包含一个二维数据样本,\n", + "`labels`中的每一行都包含一维标签值(一个标量)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "25b8aaf8-9e8e-4ec6-9fb2-bd24587e629a", + "metadata": { + "origin_pos": 9, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "features: tensor([-1.9279, -0.3628]) \n", + "label: tensor([1.5922])\n" + ] + } + ], + "source": [ + "print('features:', features[0],'\\nlabel:', labels[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "cf24318d-203e-47b6-a508-47fc5cb38403", + "metadata": { + "origin_pos": 11, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-26T14:01:17.224479\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "d2l.set_figsize()\n", + "d2l.plt.scatter(features[:, (1)].detach().numpy(), labels.detach().numpy(), 1);" + ] + }, + { + "cell_type": "markdown", + "id": "9810a549-14d5-45a6-b766-5a6dbf50fa18", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "定义一个`data_iter`函数,\n", + "该函数接收批量大小、特征矩阵和标签向量作为输入,生成大小为`batch_size`的小批量" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "6ca5e14c-f1d6-4f2b-bd57-e72c9252f452", + "metadata": { + "origin_pos": 16, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def data_iter(batch_size, features, labels):\n", + " num_examples = len(features)\n", + " indices = list(range(num_examples))\n", + " random.shuffle(indices) # 这些样本是随机读取的,没有特定的顺序\n", + " for i in range(0, num_examples, batch_size):\n", + " batch_indices = torch.tensor(\n", + " indices[i: min(i + batch_size, num_examples)])\n", + " yield features[batch_indices], labels[batch_indices]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "89ed6ef0-e020-4f6f-b788-9f0625ec0161", + "metadata": { + "origin_pos": 16, + "slideshow": { + "slide_type": "subslide" + }, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[ 0.4983, -0.6839],\n", + " [ 0.9086, 0.1984],\n", + " [-0.9412, 1.7741],\n", + " [ 0.5633, 0.5367],\n", + " [-0.5475, 1.1045],\n", + " [-0.4858, 0.1637],\n", + " [-0.6546, 0.7451],\n", + " [ 0.0917, -0.5540],\n", + " [ 1.0365, 0.3422],\n", + " [-1.3411, 1.1863]]) \n", + " tensor([[ 7.5235],\n", + " [ 5.3381],\n", + " [-3.7026],\n", + " [ 3.5037],\n", + " [-0.6498],\n", + " [ 2.6669],\n", + " [ 0.3536],\n", + " [ 6.2596],\n", + " [ 5.1234],\n", + " [-2.5214]])\n" + ] + } + ], + "source": [ + "batch_size = 10\n", + "\n", + "for X, y in data_iter(batch_size, features, labels):\n", + " print(X, '\\n', y)\n", + " break" + ] + }, + { + "cell_type": "markdown", + "id": "a23201e5-a1e7-460e-83e9-fac956fb7ade", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "在我们开始用小批量随机梯度下降优化我们的模型参数之前\n", + "我们需要先有一些参数" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ebe326c7-d78f-42d2-abc3-57741be363a5", + "metadata": { + "origin_pos": 19, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [], + "source": [ + "w = torch.normal(0, 0.01, size=(2,1), requires_grad=True)\n", + "b = torch.zeros(1, requires_grad=True)" + ] + }, + { + "cell_type": "markdown", + "id": "a4d10480-4182-4c33-9cd7-54320f2f3453", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "定义模型,将模型的输入和参数同模型的输出关联起来" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f2779003-d348-4712-afcc-b0e5a2f588b9", + "metadata": { + "origin_pos": 22, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def linreg(X, w, b): \n", + " \"\"\"线性回归模型\"\"\"\n", + " return torch.matmul(X, w) + b" + ] + }, + { + "cell_type": "markdown", + "id": "e817d119-d0d3-4ced-b559-8fcd7286615e", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "定义损失函数" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "a2160898-f709-4b0d-a4c2-d30f47775eff", + "metadata": { + "origin_pos": 24, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def squared_loss(y_hat, y): \n", + " \"\"\"均方损失\"\"\"\n", + " return (y_hat - y.reshape(y_hat.shape)) ** 2 / 2" + ] + }, + { + "cell_type": "markdown", + "id": "3c1eab90-6484-409c-b01f-1547ae043d9a", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "定义优化算法" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "20cc3c52-4538-43e6-acf8-130846cba391", + "metadata": { + "origin_pos": 27, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def sgd(params, lr, batch_size): \n", + " \"\"\"小批量随机梯度下降\"\"\"\n", + " with torch.no_grad():\n", + " for param in params:\n", + " param -= lr * param.grad / batch_size\n", + " param.grad.zero_()" + ] + }, + { + "cell_type": "markdown", + "id": "9e3b351c-4673-4278-afd6-4a7ab6262757", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "训练过程" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a3d0fbca-3007-4d70-a529-bcc3c89d140a", + "metadata": { + "origin_pos": 32, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch 1, loss 0.035369\n", + "epoch 2, loss 0.000128\n", + "epoch 3, loss 0.000051\n" + ] + } + ], + "source": [ + "lr = 0.03\n", + "num_epochs = 3\n", + "net = linreg\n", + "loss = squared_loss\n", + "\n", + "for epoch in range(num_epochs):\n", + " for X, y in data_iter(batch_size, features, labels):\n", + " l = loss(net(X, w, b), y) # X和y的小批量损失\n", + " # 因为l形状是(batch_size,1),而不是一个标量。\n", + " l.sum().backward() # l中的所有元素被加到一起,并以此计算关于[w,b]的梯度\n", + " sgd([w, b], lr, batch_size)\n", + " with torch.no_grad():\n", + " train_l = loss(net(features, w, b), labels)\n", + " print(f'epoch {epoch + 1}, loss {float(train_l.mean()):f}')" + ] + }, + { + "cell_type": "markdown", + "id": "e92e5c1e-cfbf-4fb1-8593-bc53e531976c", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "比较真实参数和通过训练学到的参数来评估训练的成功程度" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "1822da05-a22e-43cd-a443-d09f640daafe", + "metadata": { + "origin_pos": 35, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "w的估计误差: tensor([ 0.0011, -0.0005], grad_fn=)\n", + "b的估计误差: tensor([0.0004], grad_fn=)\n" + ] + } + ], + "source": [ + "print(f'w的估计误差: {true_w - w.reshape(true_w.shape)}')\n", + "print(f'b的估计误差: {true_b - b}')" + ] + }, + { + "cell_type": "markdown", + "id": "a9ea5874-18f0-4c81-b536-1f3ffabfe38c", + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "source": [ + "## 线性回归的简洁实现" + ] + }, + { + "cell_type": "markdown", + "id": "48641c86-e087-4daa-a591-0c8c41fe3fd3", + "metadata": {}, + "source": [ + "通过使用深度学习框架来简洁地实现线性回归模型\n", + "\n", + "首先生成数据集" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "713a636a-30b5-473d-8614-70f8cabf080e", + "metadata": { + "origin_pos": 4, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import torch\n", + "from torch.utils import data\n", + "from d2l import torch as d2l\n", + "\n", + "true_w = torch.tensor([2, -3.4])\n", + "true_b = 4.2\n", + "features, labels = d2l.synthetic_data(true_w, true_b, 1000)" + ] + }, + { + "cell_type": "markdown", + "id": "ebbf6c90-48e9-4dd1-98d4-dba177b4d84a", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "调用框架中现有的API来读取数据" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "f73bdcc1-8586-491b-880f-6812bf6313b9", + "metadata": { + "origin_pos": 11, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[tensor([[-0.7491, 1.8152],\n", + " [ 2.4920, 0.1592],\n", + " [ 1.9233, 0.3220],\n", + " [ 0.8503, 1.4565],\n", + " [ 0.4859, 0.6209],\n", + " [-0.9299, -1.5807],\n", + " [ 1.8116, 1.2680],\n", + " [ 0.7691, 0.2507],\n", + " [ 0.8320, -0.9070],\n", + " [ 0.5925, 0.2233]]),\n", + " tensor([[-3.4789],\n", + " [ 8.6513],\n", + " [ 6.9465],\n", + " [ 0.9517],\n", + " [ 3.0479],\n", + " [ 7.7058],\n", + " [ 3.5047],\n", + " [ 4.8805],\n", + " [ 8.9502],\n", + " [ 4.6297]])]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def load_array(data_arrays, batch_size, is_train=True): \n", + " \"\"\"构造一个PyTorch数据迭代器\"\"\"\n", + " dataset = data.TensorDataset(*data_arrays)\n", + " return data.DataLoader(dataset, batch_size, shuffle=is_train)\n", + "\n", + "batch_size = 10\n", + "data_iter = load_array((features, labels), batch_size)\n", + "\n", + "next(iter(data_iter))" + ] + }, + { + "cell_type": "markdown", + "id": "cf07e6f2-f783-4187-a0ff-70e35643dfcd", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "使用框架的预定义好的层" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "474fee23-20ce-45ad-88cb-1cf4d1eacf12", + "metadata": { + "origin_pos": 17, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "from torch import nn # nn是神经网络的缩写\n", + "\n", + "net = nn.Sequential(nn.Linear(2, 1))" + ] + }, + { + "cell_type": "markdown", + "id": "f6bd2056-9dd5-47c1-8c77-9e0626484b1b", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "初始化模型参数" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "f197ee0c-f05b-4470-9430-46127dd15228", + "metadata": { + "origin_pos": 24, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0.])" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "net[0].weight.data.normal_(0, 0.01)\n", + "net[0].bias.data.fill_(0)" + ] + }, + { + "cell_type": "markdown", + "id": "e0758d6e-a034-443d-9002-5c5ac5178e0a", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "计算均方误差使用的是`MSELoss`类,也称为平方$L_2$范数" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "441fabfd-b551-4d3b-b2b6-aa1749d76e9e", + "metadata": { + "origin_pos": 34, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [], + "source": [ + "loss = nn.MSELoss()" + ] + }, + { + "cell_type": "markdown", + "id": "24c5b7e6-19fd-40bf-a554-13da0c3c450b", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "实例化一个`SGD`实例" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "aad90c0a-d5b9-4486-8baa-e001b4ef9739", + "metadata": { + "origin_pos": 41, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "trainer = torch.optim.SGD(net.parameters(), lr=0.03)" + ] + }, + { + "cell_type": "markdown", + "id": "c43537c5-ce8b-4587-8d77-9202abd47bf9", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "训练过程代码与我们从零开始实现时所做的非常相似" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "92b54dcc-bc68-4463-978b-ea86b285e742", + "metadata": { + "origin_pos": 45, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch 1, loss 0.000356\n", + "epoch 2, loss 0.000104\n", + "epoch 3, loss 0.000105\n" + ] + } + ], + "source": [ + "num_epochs = 3\n", + "for epoch in range(num_epochs):\n", + " for X, y in data_iter:\n", + " l = loss(net(X) ,y)\n", + " trainer.zero_grad()\n", + " l.backward()\n", + " trainer.step()\n", + " l = loss(net(features), labels)\n", + " print(f'epoch {epoch + 1}, loss {l:f}')" + ] + }, + { + "cell_type": "markdown", + "id": "764227db-eac3-483c-827f-a3f009a4049d", + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "source": [ + "比较生成数据集的真实参数和通过有限数据训练获得的模型参数" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "cc70dacd-d411-47c3-923e-59fda6975788", + "metadata": { + "origin_pos": 49, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "w的估计误差: tensor([-0.0005, 0.0009])\n", + "b的估计误差: tensor([-0.0008])\n" + ] + } + ], + "source": [ + "w = net[0].weight.data\n", + "print('w的估计误差:', true_w - w.reshape(true_w.shape))\n", + "b = net[0].bias.data\n", + "print('b的估计误差:', true_b - b)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/03_linear_regression/LinearRegression_1.py b/03_linear_regression/LinearRegression_1.py new file mode 100644 index 0000000..38b7bb1 --- /dev/null +++ b/03_linear_regression/LinearRegression_1.py @@ -0,0 +1,147 @@ +import numpy as np + + +class LinearRegression: + def solve(self, lr, nepoch): + raise NotImplementedError + + +class LinearRegressionLS(LinearRegression): + """ + solve linear regression problem via least squares + """ + X: np.array + Y: np.array + w: np.array + + def __init__(self, ylist): + num_data = len(ylist) + self.X = self.homogeneous([x for x in range(num_data)]) + self.Y = np.array(ylist).reshape(num_data, 1) + self.w = np.random.rand(2) + + def homogeneous(self, xlist): + """ build homogeneous coordinates """ + raise NotImplementedError + + def linout(self, xlist): + """ linear output for given data """ + raise NotImplementedError + + def loss_sq(self, X, Y): + """ loss function: (half) sum of square errors """ + raise NotImplementedError + + def solve(self, lr, nepoch): + """ form normal equation """ + XtX = np.dot(self.X.T, self.X) + XtY = np.dot(self.X.T, self.Y) + # print(XtX.shape, XtY.shape) + self.w = np.dot(np.linalg.inv(XtX), XtY).flatten() + # print(self.w.shape, self.w) + + +class LinearRegressionGD1(LinearRegression): + """ + solve linear regression problem via gradient descent, + using single weight vector + """ + X: np.array + Y: np.array + w: np.array + + def __init__(self, ylist): + num_data = len(ylist) + self.X = self.homogeneous([x for x in range(num_data)]) + self.Y = np.array(ylist).reshape(num_data, 1) + self.w = np.random.rand(2) + + def homogeneous(self, xlist): + """ build homogeneous coordinates """ + raise NotImplementedError + + def linout(self, xlist): + """ linear output for given data """ + raise NotImplementedError + + def loss_sq(self, X, Y): + """ loss function: (half) sum of square errors """ + raise NotImplementedError + + def gd(self, lr): + """ gradient descent update """ + def gradient(Y_hat, Y, X): + return np.sum((Y_hat - Y) * X, axis=0) + + Y_hat = self.linout(self.X) + # print(Y_hat) + grad = gradient(Y_hat, self.Y, self.X) + self.w -= lr * grad + + def solve(self, lr, nepoch): + """ iterative solver """ + for epoch in range(num_epochs): + self.gd(lr) + print(f'epoch {epoch + 1}, loss {self.loss_sq(self.X, self.Y)}') + + +class LinearRegressionGD2(LinearRegression): + X: np.array + Y: np.array + w: np.array + b: np.array + + def __init__(self, ylist): + num_data = len(ylist) + self.X = np.array([x for x in range(num_data)]).reshape(-1, 1) + self.Y = np.array(ylist).reshape(num_data, 1) + self.w = np.random.rand(1) + self.b = np.min(ylist) + + def linout(self, xlist): + """ linear output for given data """ + raise NotImplementedError + + def loss_sq(self, X, Y): + """ loss function: (half) sum of square errors """ + raise NotImplementedError + + def gd(self, lr): + """ gradient descent update """ + def gradient(Y_hat, Y, X): + return np.array( + [np.sum((Y_hat - Y) * X, axis=0), + np.sum((Y_hat - Y), axis=0) + ]) + + Y_hat = self.linout(self.X) + # print(Y_hat) + grad = gradient(Y_hat, self.Y, self.X) + self.w -= lr * grad[0] + self.b -= lr * grad[1] + + def solve(self, lr, nepoch): + """ iterative solver """ + for epoch in range(num_epochs): + self.gd(lr) + print(f'epoch {epoch + 1}, loss {self.loss_sq(self.X, self.Y)}') + + +if __name__ == "__main__": + hp = [14213, 13448, 13870, 16192, 16415, 21501, 25910, 24866, 28981, 32926, 36741, 40974] + hp = [x / 10000. for x in hp] + + lr = 0.001 + num_epochs = 10 + + ls = LinearRegressionLS(hp) + ls.solve(lr, num_epochs) + print(f'next prediction (LS): {ls.linout([len(hp)])}') + + gd1 = LinearRegressionGD1(hp) + gd1.solve(lr, num_epochs) + print(f'next prediction year (GD1): {gd1.linout([len(hp)])}') + + gd2 = LinearRegressionGD1(hp) + gd2.solve(lr, num_epochs) + print(f'next prediction year (GD2): {gd2.linout([len(hp)])}') diff --git a/03_linear_regression/LinearRegression_2.py b/03_linear_regression/LinearRegression_2.py new file mode 100644 index 0000000..58065fe --- /dev/null +++ b/03_linear_regression/LinearRegression_2.py @@ -0,0 +1,131 @@ +import numpy as np + + +class LinearRegression: + def solve(self, lr, nepoch): + raise NotImplementedError + + +class LinearRegressionLS(LinearRegression): + """ + solve linear regression problem via least squares + """ + X: np.array + Y: np.array + w: np.array + + def __init__(self, ylist): + num_data = len(ylist) + self.X = self.homogeneous([x for x in range(num_data)]) + self.Y = np.array(ylist).reshape(num_data, 1) + self.w = np.random.rand(2) + + def homogeneous(self, xlist): + """ build homogeneous coordinates """ + raise NotImplementedError + + def linout(self, xlist): + """ linear output for given data """ + raise NotImplementedError + + def loss_sq(self, X, Y): + """ loss function: (half) sum of square errors """ + raise NotImplementedError + + def solve(self, lr, nepoch): + """ form normal equation """ + raise NotImplementedError + + +class LinearRegressionGD1(LinearRegression): + """ + solve linear regression problem via gradient descent, + using single weight vector: homogeneous coordinates + """ + X: np.array + Y: np.array + w: np.array + + def __init__(self, ylist): + num_data = len(ylist) + self.X = self.homogeneous([x for x in range(num_data)]) + self.Y = np.array(ylist).reshape(num_data, 1) + self.w = np.random.rand(2) + + def homogeneous(self, xlist): + """ build homogeneous coordinates """ + raise NotImplementedError + + def linout(self, xlist): + """ linear output for given data """ + raise NotImplementedError + + def loss_sq(self, X, Y): + """ loss function: (half) sum of square errors """ + raise NotImplementedError + + def gd(self, lr): + """ gradient descent update """ + raise NotImplementedError + + def solve(self, lr, nepoch): + """ iterative solver """ + for epoch in range(num_epochs): + self.gd(lr) + print(f'epoch {epoch + 1}, loss {self.loss_sq(self.X, self.Y)}') + + +class LinearRegressionGD2(LinearRegression): + """ + solve linear regression problem via gradient descent, + using two weights: w, b + """ + X: np.array + Y: np.array + w: np.array + b: np.array + + def __init__(self, ylist): + num_data = len(ylist) + self.X = np.array([x for x in range(num_data)]).reshape(-1, 1) + self.Y = np.array(ylist).reshape(num_data, 1) + self.w = np.random.rand(1) + self.b = np.min(ylist) + + def linout(self, xlist): + """ linear output for given data """ + raise NotImplementedError + + def loss_sq(self, X, Y): + """ loss function: (half) sum of square errors """ + raise NotImplementedError + + def gd(self, lr): + """ gradient descent update """ + raise NotImplementedError + + def solve(self, lr, nepoch): + """ iterative solver """ + for epoch in range(num_epochs): + self.gd(lr) + print(f'epoch {epoch + 1}, loss {self.loss_sq(self.X, self.Y)}') + + +if __name__ == "__main__": + hp = [14213, 13448, 13870, 16192, 16415, 21501, 25910, 24866, 28981, 32926, 36741, 40974] + hp = [x / 10000. for x in hp] + + lr = 0.001 + num_epochs = 10 + + ls = LinearRegressionLS(hp) + ls.solve(lr, num_epochs) + print(f'next prediction (LS): {ls.linout([len(hp)])}') + + gd1 = LinearRegressionGD1(hp) + gd1.solve(lr, num_epochs) + print(f'next prediction year (GD1): {gd1.linout([len(hp)])}') + + gd2 = LinearRegressionGD1(hp) + gd2.solve(lr, num_epochs) + print(f'next prediction year (GD2): {gd2.linout([len(hp)])}') diff --git a/03_linear_regression/expected_output.txt b/03_linear_regression/expected_output.txt new file mode 100644 index 0000000..6c57b30 --- /dev/null +++ b/03_linear_regression/expected_output.txt @@ -0,0 +1,38 @@ +=============================================== +next prediction (LS): [[4.04524848]] +=============================================== +epoch 1, loss [2.25284504] +epoch 2, loss [1.33208397] +epoch 3, loss [1.11136605] +epoch 4, loss [1.05556832] +epoch 5, loss [1.03864276] +epoch 6, loss [1.03089871] +epoch 7, loss [1.02534236] +epoch 8, loss [1.02032617] +epoch 9, loss [1.01546189] +epoch 10, loss [1.01065792] +next prediction year (GD1): [[4.42035817]] +=============================================== +epoch 1, loss [5.13717936] +epoch 2, loss [1.45618137] +epoch 3, loss [0.58898843] +epoch 4, loss [0.38458619] +epoch 5, loss [0.33630472] +epoch 6, loss [0.32479828] +epoch 7, loss [0.32195507] +epoch 8, loss [0.32115336] +epoch 9, loss [0.3208334] +epoch 10, loss [0.32062779] +next prediction year (GD2): [[4.11825378]] +=============================================== +epoch 1, loss [0.87523181] +epoch 2, loss [0.88078976] +epoch 3, loss [24.9157167] +epoch 4, loss [5055.03526329] +epoch 5, loss [1053914.58956747] +epoch 6, loss [2.19754618e+08] +epoch 7, loss [4.5821661e+10] +epoch 8, loss [9.55440499e+12] +epoch 9, loss [1.99221619e+15] +epoch 10, loss [4.15402668e+17] +next prediction year (GD1): [[4.83262041e+08]] \ No newline at end of file diff --git a/04_linear_classifier/04_linear_classifier.ipynb b/04_linear_classifier/04_linear_classifier.ipynb new file mode 100644 index 0000000..cc89639 --- /dev/null +++ b/04_linear_classifier/04_linear_classifier.ipynb @@ -0,0 +1,5498 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8467bf5b-8c23-4084-adb3-a0e0c3b3e7a6", + "metadata": {}, + "source": [ + "## 目录\n", + "\n", + "- 感知机分类器\n", + "- 逻辑回归分类器\n", + "- 图像分类数据集\n", + "- softmax回归的从零开始实现\n", + "- softmax回归的简洁实现" + ] + }, + { + "cell_type": "markdown", + "id": "412d121a-f1f9-4150-ad97-39c5b0edc498", + "metadata": { + "tags": [] + }, + "source": [ + "## 感知机分类器" + ] + }, + { + "cell_type": "markdown", + "id": "13474e54-4e85-4abf-abc0-bead50177068", + "metadata": {}, + "source": [ + "Perceptron是一个线性分类器。它的工作方式与没有隐藏层的神经网络相同(只有输入和输出)。\n", + "\n", + "![perceptron](perceptron.png)\n", + "\n", + "首先,它对给定的数据集进行权重训练,然后通过网络对一个新项目进行分类。\n", + "\n", + "> 注意,在分类问题中,每个节点代表一个类别。最终的分类是具有最大输出值的类/节点。" + ] + }, + { + "cell_type": "markdown", + "id": "d05ceed2-8d2e-4bc6-ac1c-da2bb67ecb89", + "metadata": { + "tags": [] + }, + "source": [ + "### 实现\n", + "\n", + "`PerceptionLinearLearner` 用于训练(计算)给定数据集的权重。\n", + "\n", + "函数`predict`用于对一个新的项目进行分类:该函数计算项目与外层每个节点的计算权重的(代数)点积。然后,它选择一个最大的值,将项目归入相应的类别。" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c3423455-3a8c-44e5-a9aa-592b7c6e285d", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "

\n", + "\n", + "
class PerceptionLinearLearner(LinearClassifier):\n",
+       "    """\n",
+       "    Perception linear classifier: hard threshold\n",
+       "    """\n",
+       "    def __init__(self, dataset, learning_rate=0.01, epochs=100):\n",
+       "        self.idx_i = dataset.inputs\n",
+       "        self.idx_t = dataset.target\n",
+       "        self.examples = dataset.examples\n",
+       "        self.num_examples = len(self.examples)\n",
+       "        # initialize random weights\n",
+       "        self.w = random_weights(min_value=-0.5, max_value=0.5, num_weights=len(self.idx_i) + 1)\n",
+       "        # learning loop\n",
+       "        self.learn(learning_rate, epochs)\n",
+       "        \n",
+       "    def learn(self, learning_rate, epochs):\n",
+       "        """ learning loop """\n",
+       "        def loss(example, w, idx_i, idx_t):\n",
+       "            """ error: difference between estimation and true value """\n",
+       "            raise NotImplementedError\n",
+       "        \n",
+       "        def update(w, learning_rate, err, X_col, num_examples):\n",
+       "            """ update weights """\n",
+       "            raise NotImplementedError\n",
+       "\n",
+       "        def homogeneous(num_examples):\n",
+       "            """ build homogeneous coordinates """\n",
+       "            raise NotImplementedError\n",
+       "\n",
+       "        raise NotImplementedError\n",
+       "\n",
+       "    def predict(self, x):\n",
+       "        """ make prediction """\n",
+       "        return int(np.dot(self.w, [1] + x))\n",
+       "
\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import sys\n", + "sys.path.insert(1, '../')\n", + "from utils.dataset4learners import *\n", + "from LinearClassifier_3 import *\n", + "\n", + "psource(PerceptionLinearLearner)" + ] + }, + { + "cell_type": "markdown", + "id": "8c4fac32-dfa4-46d4-a2c4-8e5724f1def3", + "metadata": {}, + "source": [ + "#### 实现要点\n", + "\n", + "```py\n", + "def learn(self, learning_rate, epochs):\n", + " \"\"\" learning loop \"\"\"\n", + " def loss(example, w, idx_i, idx_t):\n", + " \"\"\" error: difference between estimation and true value \"\"\"\n", + " x = example的齐次坐标表示\n", + " y = w * x,即预测值\n", + " t = 真实值\n", + " return y - t\n", + "\n", + " def update(w, learning_rate, err, X_col, num_examples):\n", + " \"\"\" update weights \"\"\"\n", + " 遍历w[i]:\n", + " w[i] = w[i] - 学习率 * 误差值 * 自变量X[i] / 样本数量)\n", + "\n", + " def homogeneous(num_examples):\n", + " \"\"\" build homogeneous coordinates \"\"\"\n", + " return 齐次坐标表示\n", + "\n", + " X_col = 齐次坐标表示\n", + " for epoch in range(epochs):\n", + " err = []\n", + " # pass over all examples\n", + " for example in self.examples:\n", + " err.append(loss(example, w, idx_i, idx_t))\n", + "\n", + " # update weights\n", + " update(w, 学习率, err, X_col, 样本数量)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "d8f78b41-5136-4563-8359-67c1dae52642", + "metadata": {}, + "source": [ + "> 注意,Perceptron是一个单层的神经网络,在后面课程中讲授。" + ] + }, + { + "cell_type": "markdown", + "id": "ca1d7196-b054-4718-9402-c0a377fc53a6", + "metadata": {}, + "source": [ + "### 例子\n", + "\n", + "我们将在`iris`数据集上训练感知机。\n", + "\n", + "尽管`BackPropagationLearner`使用的是整数索引而不是字符串,我们需要将类名转换成整数。" + ] + }, + { + "cell_type": "markdown", + "id": "c37147ed-40e7-4b05-83e3-5c4db4c0a039", + "metadata": { + "tags": [] + }, + "source": [ + "```py\n", + "iris = DataSet(name=\"iris\")\n", + "iris.classes_to_numbers()\n", + "\n", + "perceptron = PerceptionLinearLearner(iris)\n", + "print(perceptron.predict([5, 3, 1, 0.1]))\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "debca21b-5262-49ac-8502-8e692e9d26af", + "metadata": {}, + "source": [ + "正确的输出是0,这意味着该物品属于第一类,\"setosa\"。注意Perceptron算法并不完美,可能会产生错误的分类。" + ] + }, + { + "cell_type": "markdown", + "id": "c6ab0a90-bf04-497b-9795-31e62155054d", + "metadata": { + "tags": [] + }, + "source": [ + "## 逻辑回归分类器" + ] + }, + { + "cell_type": "markdown", + "id": "55e14e99-4e16-4ef2-a652-43a51364c144", + "metadata": {}, + "source": [ + "逻辑回归将感知机采用的硬性阈值函数改为柔性阈值,可以更好地对分界区域的不确定性建模。\n", + "\n", + "### 实现\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "360a50d6-647e-4e48-8b66-5feb22db7dd0", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "

\n", + "\n", + "
class LogisticLinearLeaner(LinearClassifier):\n",
+       "    def __init__(self, dataset, learning_rate=0.01, epochs=100):\n",
+       "        self.idx_i = dataset.inputs\n",
+       "        self.idx_t = dataset.target\n",
+       "        self.examples = dataset.examples\n",
+       "        self.num_examples = len(self.examples)\n",
+       "        # initialize random weights\n",
+       "        self.w = random_weights(min_value=-0.5, max_value=0.5, num_weights=len(self.idx_i) + 1)\n",
+       "        # learning loop\n",
+       "        self.learn(learning_rate, epochs)\n",
+       "        \n",
+       "    def learn(self, learning_rate, epochs):\n",
+       "        """ learning loop """\n",
+       "        def loss(example, w, idx_i, idx_t, h):\n",
+       "            """ error: difference between estimation and true value """\n",
+       "            raise NotImplementedError\n",
+       "        \n",
+       "        def update(w, learning_rate, err, h, X_col, num_examples):\n",
+       "            """ update weights """\n",
+       "            raise NotImplementedError\n",
+       "\n",
+       "        def homogeneous(num_examples):\n",
+       "            """ build homogeneous coordinates """\n",
+       "            raise NotImplementedError\n",
+       "\n",
+       "        raise NotImplementedError\n",
+       "\n",
+       "    def predict(self, x):\n",
+       "        """ make prediction """\n",
+       "        return int(np.dot(self.w, [1] + x))\n",
+       "
\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "psource(LogisticLinearLeaner)" + ] + }, + { + "cell_type": "markdown", + "id": "64d23b34-7203-4abe-bb3b-b3e65f192d62", + "metadata": {}, + "source": [ + "#### 实现要点\n", + "\n", + "```py\n", + "def learn(self, learning_rate, epochs):\n", + " \"\"\" learning loop \"\"\"\n", + " def loss(example, w, idx_i, idx_t, h):\n", + " \"\"\" error: difference between estimation and true value \"\"\"\n", + " x = example的齐次坐标表示\n", + " y = Sigmoid(w * x),即预测值\n", + " h.append(Sigmoid().derivative(y)):收集梯度值\n", + " t = 真实值\n", + " return y - t\n", + "\n", + " def update(w, learning_rate, err, h, X_col, num_examples):\n", + " \"\"\" update weights \"\"\"\n", + " 遍历w[i]:\n", + " w[i] = w[i] - 学习率 * 误差值err[i] * 梯度值h[i] / 样本数量)\n", + "\n", + " def homogeneous(num_examples):\n", + " \"\"\" build homogeneous coordinates \"\"\"\n", + " 同感知机\n", + "\n", + " 同感知机\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "34dde5da-d093-4021-ae8b-7a38cd5b11dc", + "metadata": {}, + "source": [ + "该算法首先为输入变量分配一些随机权重,然后根据计算的误差更新每个变量的权重。最后用更新后的权重进行预测。" + ] + }, + { + "cell_type": "markdown", + "id": "2b3f1358-fe11-4499-8fc1-f8a7fb6ef826", + "metadata": { + "tags": [] + }, + "source": [ + "```py\n", + "iris = DataSet(name=\"iris\")\n", + "iris.classes_to_numbers()\n", + "\n", + "logisticer = LogisticLinearLeaner(iris)\n", + "print(logisticer.predict([5, 3, 1, 0.1]))\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "b0e066c5-554b-4286-ae5f-1f71dd7b95a8", + "metadata": { + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 图像分类数据集" + ] + }, + { + "cell_type": "markdown", + "id": "19b612e3-6ede-4ffa-a9dd-14a0480a2c8f", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "MNIST数据集是图像分类中广泛使用的数据集之一,但作为基准数据集过于简单。\n", + "\n", + "我们将使用类似但更复杂的Fashion-MNIST数据集" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "451140a0-6c93-45e0-8e19-8ffb51099e77", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import torch\n", + "import torchvision\n", + "from torch.utils import data\n", + "from torchvision import transforms\n", + "from d2l import torch as d2l\n", + "\n", + "d2l.use_svg_display()" + ] + }, + { + "cell_type": "markdown", + "id": "b5386450-4cdd-4c05-9524-25505b67f22c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "通过框架中的内置函数将Fashion-MNIST数据集下载并读取到内存中" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "af7b66a6-a2ca-4ec3-981e-8acaf5880cbe", + "metadata": { + "origin_pos": 9, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz to ../data/FashionMNIST/raw/train-images-idx3-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100.0%\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ../data/FashionMNIST/raw/train-images-idx3-ubyte.gz to ../data/FashionMNIST/raw\n", + "\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz to ../data/FashionMNIST/raw/train-labels-idx1-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100.0%\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ../data/FashionMNIST/raw/train-labels-idx1-ubyte.gz to ../data/FashionMNIST/raw\n", + "\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz to ../data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100.0%\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ../data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz to ../data/FashionMNIST/raw\n", + "\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz to ../data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100.0%\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ../data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz to ../data/FashionMNIST/raw\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "(60000, 10000)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 通过ToTensor实例将图像数据从PIL类型变换成32位浮点数格式,\n", + "# 并除以255使得所有像素的数值均在0到1之间\n", + "trans = transforms.ToTensor()\n", + "mnist_train = torchvision.datasets.FashionMNIST(\n", + " root=\"../data\", train=True, transform=trans, download=True)\n", + "mnist_test = torchvision.datasets.FashionMNIST(\n", + " root=\"../data\", train=False, transform=trans, download=True)\n", + "\n", + "len(mnist_train), len(mnist_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "e5de50d6-1fe0-407d-87a9-8eec3c6c475d", + "metadata": { + "origin_pos": 12, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 28, 28])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mnist_train[0][0].shape" + ] + }, + { + "cell_type": "markdown", + "id": "87d0348b-ec5d-4299-9d8c-08987de460f1", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "两个可视化数据集的函数" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "0557638e-71b8-40ce-95e6-5d61038f1ffc", + "metadata": { + "origin_pos": 17, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def get_fashion_mnist_labels(labels): \n", + " \"\"\"返回Fashion-MNIST数据集的文本标签\"\"\"\n", + " text_labels = ['t-shirt', 'trouser', 'pullover', 'dress', 'coat',\n", + " 'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']\n", + " return [text_labels[int(i)] for i in labels]\n", + "\n", + "def show_images(imgs, num_rows, num_cols, titles=None, scale=1.5): \n", + " \"\"\"绘制图像列表\"\"\"\n", + " figsize = (num_cols * scale, num_rows * scale)\n", + " _, axes = d2l.plt.subplots(num_rows, num_cols, figsize=figsize)\n", + " axes = axes.flatten()\n", + " for i, (ax, img) in enumerate(zip(axes, imgs)):\n", + " if torch.is_tensor(img):\n", + " ax.imshow(img.numpy()) # 图片张量\n", + " else:\n", + " ax.imshow(img) # PIL图片\n", + " ax.axes.get_xaxis().set_visible(False)\n", + " ax.axes.get_yaxis().set_visible(False)\n", + " if titles:\n", + " ax.set_title(titles[i])\n", + " return axes" + ] + }, + { + "cell_type": "markdown", + "id": "0c9255c0-c447-4232-a1e8-9cbdd63f2a62", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "几个样本的图像及其相应的标签" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "6bd5a31f-a289-4ab2-9ee5-bbe8322ac216", + "metadata": { + "origin_pos": 20, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-26T14:13:22.029419\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "X, y = next(iter(data.DataLoader(mnist_train, batch_size=18)))\n", + "show_images(X.reshape(18, 28, 28), 2, 9, titles=get_fashion_mnist_labels(y));" + ] + }, + { + "cell_type": "markdown", + "id": "9ce47a48-d8e3-46a6-8e2b-009c4e299341", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "读取一小批量数据,大小为`batch_size`" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "b15b88d6-d561-4971-9c20-b30d2618da04", + "metadata": { + "origin_pos": 27, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'20.21 sec'" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "batch_size = 256\n", + "\n", + "def get_dataloader_workers(): \n", + " \"\"\"使用4个进程来读取数据\"\"\"\n", + " return 4\n", + "\n", + "train_iter = data.DataLoader(mnist_train, batch_size, shuffle=True,\n", + " num_workers=get_dataloader_workers())\n", + "\n", + "timer = d2l.Timer()\n", + "for X, y in train_iter:\n", + " continue\n", + "f'{timer.stop():.2f} sec'" + ] + }, + { + "cell_type": "markdown", + "id": "1e0a3486-8788-4a87-b9b9-cb327ba7d2ca", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "定义`load_data_fashion_mnist`函数" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "5f88e2a1-88b6-49c3-a8b8-2ac9602a71a8", + "metadata": { + "origin_pos": 33, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def load_data_fashion_mnist(batch_size, resize=None): \n", + " \"\"\"下载Fashion-MNIST数据集,然后将其加载到内存中\"\"\"\n", + " trans = [transforms.ToTensor()]\n", + " if resize:\n", + " trans.insert(0, transforms.Resize(resize))\n", + " trans = transforms.Compose(trans)\n", + " mnist_train = torchvision.datasets.FashionMNIST(\n", + " root=\"../data\", train=True, transform=trans, download=True)\n", + " mnist_test = torchvision.datasets.FashionMNIST(\n", + " root=\"../data\", train=False, transform=trans, download=True)\n", + " return (data.DataLoader(mnist_train, batch_size, shuffle=True,\n", + " num_workers=get_dataloader_workers()),\n", + " data.DataLoader(mnist_test, batch_size, shuffle=False,\n", + " num_workers=get_dataloader_workers()))" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "b60ce5c6-c1e3-4d51-9d2a-d35e114773ed", + "metadata": { + "origin_pos": 33, + "slideshow": { + "slide_type": "slide" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([32, 1, 64, 64]) torch.float32 torch.Size([32]) torch.int64\n" + ] + } + ], + "source": [ + "train_iter, test_iter = load_data_fashion_mnist(32, resize=64)\n", + "for X, y in train_iter:\n", + " print(X.shape, X.dtype, y.shape, y.dtype)\n", + " break" + ] + }, + { + "cell_type": "markdown", + "id": "2b3bc703-261c-4f28-8b13-10599cdfb648", + "metadata": { + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## softmax回归的从零开始实现" + ] + }, + { + "cell_type": "markdown", + "id": "461c158e-d492-4138-a1dc-a6dee27856c3", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "就像我们从零开始实现线性回归一样,你应该知道实现softmax回归的细节" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "7c6ea6a5-00d4-4032-8b69-a23295a1c3b3", + "metadata": { + "origin_pos": 4, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from IPython import display\n", + "from d2l import torch as d2l\n", + "\n", + "batch_size = 256\n", + "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)" + ] + }, + { + "cell_type": "markdown", + "id": "79517ad9-cddb-4ee2-9f61-43e0b6bb3341", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "将展平每个图像,把它们看作长度为784的向量。\n", + "因为我们的数据集有10个类别,所以网络输出维度为10" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "bf6ec527-910e-42ce-b24c-c72da6fa9c74", + "metadata": { + "origin_pos": 7, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "num_inputs = 784\n", + "num_outputs = 10\n", + "\n", + "W = torch.normal(0, 0.01, size=(num_inputs, num_outputs), requires_grad=True)\n", + "b = torch.zeros(num_outputs, requires_grad=True)" + ] + }, + { + "cell_type": "markdown", + "id": "2499dc12-833a-4851-b5de-859d86f428c5", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "给定一个矩阵`X`,我们可以对所有元素求和" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "e7fbccdf-9d88-4244-963a-33ebe05ba5ae", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([[5., 7., 9.]]),\n", + " tensor([[ 6.],\n", + " [15.]]))" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n", + "X.sum(0, keepdim=True), X.sum(1, keepdim=True)" + ] + }, + { + "cell_type": "markdown", + "id": "7338e047-94ed-438d-a0b0-2477692b0c45", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "实现softmax\n", + "\n", + "$$\n", + "\\mathrm{softmax}(\\mathbf{X})_{ij} = \\frac{\\exp(\\mathbf{X}_{ij})}{\\sum_k \\exp(\\mathbf{X}_{ik})}.\n", + "$$" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "073aadb5-9a24-44e3-ac4b-3bae84a3a9e2", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def softmax(X):\n", + " X_exp = torch.exp(X)\n", + " partition = X_exp.sum(1, keepdim=True)\n", + " return X_exp / partition # 广播机制" + ] + }, + { + "cell_type": "markdown", + "id": "17706f99-24f6-446f-9f4a-a17368170e3f", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "我们将每个元素变成一个非负数。\n", + "此外,依据概率原理,每行总和为1" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "2ff371fa-6e82-4540-8c7d-0bba75225e1f", + "metadata": { + "origin_pos": 16, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([[0.3532, 0.0347, 0.0805, 0.0395, 0.4921],\n", + " [0.6108, 0.0814, 0.1468, 0.0484, 0.1126]]),\n", + " tensor([1., 1.]))" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.normal(0, 1, (2, 5))\n", + "X_prob = softmax(X)\n", + "X_prob, X_prob.sum(1)" + ] + }, + { + "cell_type": "markdown", + "id": "02ba4bf7-0785-4958-afbf-9519f33ed8af", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "实现softmax回归模型" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "ec1c7adf-b253-48cb-8abc-395098c3635e", + "metadata": { + "origin_pos": 19, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def net(X):\n", + " return softmax(torch.matmul(X.reshape((-1, W.shape[0])), W) + b)" + ] + }, + { + "cell_type": "markdown", + "id": "e1ee288d-3e6e-41de-b4f5-c020e83bd14e", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "创建一个数据样本`y_hat`,其中包含2个样本在3个类别的预测概率,\n", + "以及它们对应的标签`y`。\n", + "使用`y`作为`y_hat`中概率的索引" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "83632ade-00af-4692-887f-cd13041c814d", + "metadata": { + "origin_pos": 21, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0.1000, 0.5000])" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "y = torch.tensor([0, 2]) # 在第二个样本中,第三类是正确的预测\n", + "y_hat = torch.tensor([[0.1, 0.3, 0.6], [0.3, 0.2, 0.5]])\n", + "y_hat[[0, 1], y] # 使用y作为y_hat中概率的索引" + ] + }, + { + "cell_type": "markdown", + "id": "c40efbc6-376d-4557-8b46-35c6aff4f979", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "实现交叉熵损失函数" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "f070bea1-2b36-402e-828f-29d2446b0c56", + "metadata": { + "origin_pos": 24, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([2.3026, 0.6931])" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def cross_entropy(y_hat, y):\n", + " \"\"\" 避免低效的for循环 \"\"\"\n", + " return - torch.log(y_hat[range(len(y_hat)), y])\n", + "\n", + "cross_entropy(y_hat, y) # 损失函数,越低越好" + ] + }, + { + "cell_type": "markdown", + "id": "9374a06e-27ed-42fa-a1c9-0c6d90dd4a48", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "将预测类别与真实`y`元素进行比较" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "97a56aa3-f780-4f32-98ca-086d6e1024f0", + "metadata": { + "origin_pos": 29, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0.5" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def accuracy(y_hat, y): \n", + " \"\"\"计算预测正确的数量\"\"\"\n", + " if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:\n", + " y_hat = y_hat.argmax(axis=1)\n", + " cmp = y_hat.type(y.dtype) == y # 数据类型转成一致\n", + " return float(cmp.type(y.dtype).sum())\n", + "\n", + "accuracy(y_hat, y) / len(y)" + ] + }, + { + "cell_type": "markdown", + "id": "585af5a4-d317-42ee-a5f1-255a594856fa", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "我们可以评估在任意模型`net`的精度" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "540c6f10-7592-4fd3-8221-0ef56f88a0bf", + "metadata": { + "origin_pos": 32, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def evaluate_accuracy(net, data_iter): \n", + " \"\"\"计算在指定数据集上模型的精度\"\"\"\n", + " if isinstance(net, torch.nn.Module):\n", + " net.eval() # 将模型设置为评估模式\n", + " metric = Accumulator(2) # 正确预测数、预测总数\n", + " with torch.no_grad():\n", + " for X, y in data_iter:\n", + " metric.add(accuracy(net(X), y), y.numel())\n", + " return metric[0] / metric[1]" + ] + }, + { + "cell_type": "markdown", + "id": "d7593436-0ca0-4fc3-ae2c-7d7311ddb9f6", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "`Accumulator`实例中创建了2个变量,\n", + "分别用于存储正确预测的数量和预测的总数量" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "bb38e91c-b8aa-4912-862e-9080a0bbda66", + "metadata": { + "origin_pos": 36, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0.1004" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class Accumulator: \n", + " \"\"\"在n个变量上累加\"\"\"\n", + " def __init__(self, n):\n", + " self.data = [0.0] * n\n", + "\n", + " def add(self, *args):\n", + " self.data = [a + float(b) for a, b in zip(self.data, args)]\n", + "\n", + " def reset(self):\n", + " self.data = [0.0] * len(self.data)\n", + "\n", + " def __getitem__(self, idx):\n", + " return self.data[idx]\n", + "\n", + "evaluate_accuracy(net, test_iter) # 随机权重初始化,随机猜测应该接近0.1" + ] + }, + { + "cell_type": "markdown", + "id": "9ca7e9be-2270-481f-b0c9-882b79efed78", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Softmax回归的训练" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "c4c903ea-72eb-49a9-9888-8e8d5fd6f8a6", + "metadata": { + "origin_pos": 39, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def train_epoch_ch3(net, train_iter, loss, updater): \n", + " \"\"\"训练模型一个迭代周期(定义见第3章)\"\"\"\n", + " if isinstance(net, torch.nn.Module):\n", + " net.train() # 将模型设置为训练模式\n", + " metric = Accumulator(3) # 训练损失总和、训练准确度总和、样本数\n", + " for X, y in train_iter:\n", + " # 计算梯度并更新参数\n", + " y_hat = net(X)\n", + " l = loss(y_hat, y)\n", + " if isinstance(updater, torch.optim.Optimizer):\n", + " # 使用PyTorch内置的优化器和损失函数\n", + " updater.zero_grad()\n", + " l.mean().backward()\n", + " updater.step()\n", + " else: # 使用定制的优化器和损失函数\n", + " l.sum().backward()\n", + " updater(X.shape[0])\n", + " metric.add(float(l.sum()), accuracy(y_hat, y), y.numel())\n", + " # 返回训练损失和训练精度\n", + " return metric[0] / metric[2], metric[1] / metric[2]" + ] + }, + { + "cell_type": "markdown", + "id": "dd06ef05-3914-4ecd-ba5e-59c4fea740c0", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "定义一个在动画中绘制数据的实用程序类" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "3ecf8ca6-1797-497d-abc3-867a591bf464", + "metadata": { + "origin_pos": 42, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "class Animator: \n", + " \"\"\"在动画中绘制数据\"\"\"\n", + " def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,\n", + " ylim=None, xscale='linear', yscale='linear',\n", + " fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1,\n", + " figsize=(3.5, 2.5)):\n", + " if legend is None:\n", + " legend = [] # 增量地绘制多条线\n", + " d2l.use_svg_display()\n", + " self.fig, self.axes = d2l.plt.subplots(nrows, ncols, figsize=figsize)\n", + " if nrows * ncols == 1:\n", + " self.axes = [self.axes, ]\n", + " # 使用lambda函数捕获参数\n", + " self.config_axes = lambda: d2l.set_axes(\n", + " self.axes[0], xlabel, ylabel, xlim, ylim, xscale, yscale, legend)\n", + " self.X, self.Y, self.fmts = None, None, fmts\n", + "\n", + " def add(self, x, y):\n", + " # 向图表中添加多个数据点\n", + " if not hasattr(y, \"__len__\"):\n", + " y = [y]\n", + " n = len(y)\n", + " if not hasattr(x, \"__len__\"):\n", + " x = [x] * n\n", + " if not self.X:\n", + " self.X = [[] for _ in range(n)]\n", + " if not self.Y:\n", + " self.Y = [[] for _ in range(n)]\n", + " for i, (a, b) in enumerate(zip(x, y)):\n", + " if a is not None and b is not None:\n", + " self.X[i].append(a)\n", + " self.Y[i].append(b)\n", + " self.axes[0].cla()\n", + " for x, y, fmt in zip(self.X, self.Y, self.fmts):\n", + " self.axes[0].plot(x, y, fmt)\n", + " self.config_axes()\n", + " display.display(self.fig)\n", + " display.clear_output(wait=True)" + ] + }, + { + "cell_type": "markdown", + "id": "91290f6c-081c-430f-8563-73166defe322", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练函数" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "500cf101-f3cb-44c5-abc6-bbfb770e7953", + "metadata": { + "origin_pos": 44, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def train_ch3(net, train_iter, test_iter, loss, num_epochs, updater): \n", + " \"\"\"训练模型(定义见第3章)\"\"\"\n", + " animator = Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0.3, 0.9],\n", + " legend=['train loss', 'train acc', 'test acc'])\n", + " for epoch in range(num_epochs):\n", + " train_metrics = train_epoch_ch3(net, train_iter, loss, updater)\n", + " test_acc = evaluate_accuracy(net, test_iter)\n", + " animator.add(epoch + 1, train_metrics + (test_acc,))\n", + " train_loss, train_acc = train_metrics\n", + " assert train_loss < 0.5, train_loss\n", + " assert train_acc <= 1 and train_acc > 0.7, train_acc\n", + " assert test_acc <= 1 and test_acc > 0.7, test_acc" + ] + }, + { + "cell_type": "markdown", + "id": "09e83970-88ed-4d77-a10a-92fad6a89449", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "小批量随机梯度下降来优化模型的损失函数" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "ed05114c-e815-44c4-bcce-10880f11286a", + "metadata": { + "origin_pos": 46, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "lr = 0.1\n", + "\n", + "def updater(batch_size):\n", + " return d2l.sgd([W, b], lr, batch_size)" + ] + }, + { + "cell_type": "markdown", + "id": "6e153647-c2d1-4497-a261-42dfc29ccfd9", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练模型10个迭代周期" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "cc4149fd-a083-4a50-a164-d71513ea5edb", + "metadata": { + "origin_pos": 49, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-26T14:17:07.123256\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "num_epochs = 10\n", + "train_ch3(net, train_iter, test_iter, cross_entropy, num_epochs, updater)" + ] + }, + { + "cell_type": "markdown", + "id": "2ec84ab6-8482-46a9-ab68-522f2faba21c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "对图像进行分类预测" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "f5909439-a31c-4ade-aab8-5bb3edb9fa23", + "metadata": { + "origin_pos": 51, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-26T14:17:16.183922\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def predict_ch3(net, test_iter, n=8): \n", + " \"\"\"预测标签(定义见第3章)\"\"\"\n", + " for X, y in test_iter:\n", + " break\n", + " trues = d2l.get_fashion_mnist_labels(y)\n", + " preds = d2l.get_fashion_mnist_labels(net(X).argmax(axis=1))\n", + " titles = [true +'\\n' + pred for true, pred in zip(trues, preds)]\n", + " d2l.show_images(\n", + " X[0:n].reshape((n, 28, 28)), 1, n, titles=titles[0:n])\n", + "\n", + "predict_ch3(net, test_iter)" + ] + }, + { + "cell_type": "markdown", + "id": "fb05ae93-ca43-4978-b77b-cca182c45c8f", + "metadata": { + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## softmax回归的简洁实现" + ] + }, + { + "cell_type": "markdown", + "id": "a7da7eb8-6bad-4d17-83a7-24ec09779ef0", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "通过深度学习框架的高级API能够使实现softmax回归变得更加容易" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "b598a5ff-1618-4bc3-a8aa-c627a3d3caa7", + "metadata": { + "origin_pos": 4, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "batch_size = 256\n", + "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)" + ] + }, + { + "cell_type": "markdown", + "id": "dd341874-7652-4afe-aa0a-41207311318f", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Softmax回归的输出层是一个全连接层" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "ee566c3d-2669-4149-bdcb-d9832ed4c4d5", + "metadata": { + "origin_pos": 7, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "# PyTorch不会隐式地调整输入的形状。因此,\n", + "# 我们在线性层前定义了展平层(flatten),来调整网络输入的形状\n", + "net = nn.Sequential(nn.Flatten(), nn.Linear(784, 10))\n", + "\n", + "def init_weights(m):\n", + " if type(m) == nn.Linear:\n", + " nn.init.normal_(m.weight, std=0.01)\n", + "\n", + "net.apply(init_weights);" + ] + }, + { + "cell_type": "markdown", + "id": "1330ec1d-3c66-46a0-8422-948553a33c41", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "在交叉熵损失函数中传递未规范化的预测,并同时计算softmax及其对数" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "fcac843e-7436-4e88-bdf3-480bdfa12f26", + "metadata": { + "origin_pos": 11, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "loss = nn.CrossEntropyLoss(reduction='none')" + ] + }, + { + "cell_type": "markdown", + "id": "1c7344bf-3aa5-4a19-8b25-b0296f9e238d", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "使用学习率为0.1的小批量随机梯度下降作为优化算法" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "3c5df7de-6a41-4069-bab3-3b518da1f30f", + "metadata": { + "origin_pos": 15, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "trainer = torch.optim.SGD(net.parameters(), lr=0.1)" + ] + }, + { + "cell_type": "markdown", + "id": "eef015d8-1fbc-4dd1-b1c0-5bbb46ec6d34", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "调用\n", + "之前\n", + "定义的训练函数来训练模型" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "2b28c827-05cd-4c4b-9dd7-db27d8067a27", + "metadata": { + "origin_pos": 18, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-26T14:19:54.955113\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "num_epochs = 10\n", + "d2l.train_ch3(net, train_iter, test_iter, loss, num_epochs, trainer)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/04_linear_classifier/LinearClassifier_1.py b/04_linear_classifier/LinearClassifier_1.py new file mode 100644 index 0000000..beabc50 --- /dev/null +++ b/04_linear_classifier/LinearClassifier_1.py @@ -0,0 +1,129 @@ +import sys +sys.path.insert(1, '../') +from utils.utils import * +from utils.dataset4learners import * + + +class LinearClassifier: + def learn(self, learning_rate, epochs): + raise NotImplementedError + + def predict(self, x): + raise NotImplementedError + + +class PerceptionLinearLearner(LinearClassifier): + """ + Perception linear classifier: hard threshold + """ + def __init__(self, dataset, learning_rate=0.01, epochs=100): + self.idx_i = dataset.inputs + self.idx_t = dataset.target + self.examples = dataset.examples + self.num_examples = len(self.examples) + # initialize random weights + self.w = random_weights(min_value=-0.5, max_value=0.5, num_weights=len(self.idx_i) + 1) + # learning loop + self.learn(learning_rate, epochs) + + def learn(self, learning_rate, epochs): + """ learning loop """ + def loss(example, w, idx_i, idx_t): + """ error: difference between estimation and true value """ + raise NotImplementedError + + def update(w, learning_rate, err, X_col, num_examples): + """ update weights """ + for i in range(len(w)): + w[i] = w[i] - learning_rate * (np.dot(err, X_col[i]) / num_examples) + + def homogeneous(num_examples): + """ build homogeneous coordinates """ + raise NotImplementedError + + X_col = homogeneous(self.num_examples) + for epoch in range(epochs): + err = [] + # pass over all examples + for example in self.examples: + err.append(loss(example, self.w, self.idx_i, self.idx_t)) + + # update weights + update(self.w, learning_rate, err, X_col, self.num_examples) + + def predict(self, x): + """ make prediction """ + return int(np.dot(self.w, [1] + x)) + + +class LogisticLinearLeaner(LinearClassifier): + def __init__(self, dataset, learning_rate=0.01, epochs=100): + self.idx_i = dataset.inputs + self.idx_t = dataset.target + self.examples = dataset.examples + self.num_examples = len(self.examples) + # initialize random weights + self.w = random_weights(min_value=-0.5, max_value=0.5, num_weights=len(self.idx_i) + 1) + # learning loop + self.learn(learning_rate, epochs) + + def learn(self, learning_rate, epochs): + """ learning loop """ + def loss(example, w, idx_i, idx_t, h): + """ error: difference between estimation and true value """ + raise NotImplementedError + + def update(w, learning_rate, err, h, X_col, num_examples): + """ update weights """ + for i in range(len(w)): + buffer = [x * y for x, y in zip(err, h)] + w[i] = w[i] - learning_rate * (np.dot(buffer, X_col[i]) / num_examples) + + def homogeneous(num_examples): + """ build homogeneous coordinates """ + raise NotImplementedError + + X_col = homogeneous(self.num_examples) + for epoch in range(epochs): + err = [] + h = [] + # pass over all examples + for example in self.examples: + err.append(loss(example, self.w, self.idx_i, self.idx_t, h)) + + # update weights + update(self.w, learning_rate, err, h, X_col, self.num_examples) + + def predict(self, x): + """ make prediction """ + return int(np.dot(self.w, [1] + x)) + + +if __name__ == "__main__": + iris = DataSet(name="iris") + iris.classes_to_numbers() + + tests = [([5, 3, 1, 0.1], 0), + ([5, 3.5, 1, 0], 0), + ([6, 3, 4, 1.1], 1), + ([6, 2, 3.5, 1], 1), + ([7.5, 4, 6, 2], 2), + ([7, 3, 6, 2.5], 2)] + + print(f'===================\nperceptron:') + perceptron = PerceptionLinearLearner(iris) + g = grade_learner(perceptron, tests) + print(f' learner grade: {g}') + # assert g > 1. / 2 + e = err_ratio(perceptron, iris) + print(f' error ration: {e}') + # assert e < 0.4 + + print(f'===================\nlogistic:') + logisticer = LogisticLinearLeaner(iris) + g = grade_learner(logisticer, tests) + print(f' learner grade: {g}') + # assert g > 1. / 2 + e = err_ratio(logisticer, iris) + print(f' error ration: {e}') + # assert e < 0.4 diff --git a/04_linear_classifier/LinearClassifier_2.py b/04_linear_classifier/LinearClassifier_2.py new file mode 100644 index 0000000..5908dd6 --- /dev/null +++ b/04_linear_classifier/LinearClassifier_2.py @@ -0,0 +1,126 @@ +import sys +sys.path.insert(1, '../') +from utils.utils import * +from utils.dataset4learners import * + + +class LinearClassifier: + def learn(self, learning_rate, epochs): + raise NotImplementedError + + def predict(self, x): + raise NotImplementedError + + +class PerceptionLinearLearner(LinearClassifier): + """ + Perception linear classifier: hard threshold + """ + def __init__(self, dataset, learning_rate=0.01, epochs=100): + self.idx_i = dataset.inputs + self.idx_t = dataset.target + self.examples = dataset.examples + self.num_examples = len(self.examples) + # initialize random weights + self.w = random_weights(min_value=-0.5, max_value=0.5, num_weights=len(self.idx_i) + 1) + # learning loop + self.learn(learning_rate, epochs) + + def learn(self, learning_rate, epochs): + """ learning loop """ + def loss(example, w, idx_i, idx_t): + """ error: difference between estimation and true value """ + raise NotImplementedError + + def update(w, learning_rate, err, X_col, num_examples): + """ update weights """ + raise NotImplementedError + + def homogeneous(num_examples): + """ build homogeneous coordinates """ + raise NotImplementedError + + X_col = homogeneous(self.num_examples) + for epoch in range(epochs): + err = [] + # pass over all examples + for example in self.examples: + err.append(loss(example, self.w, self.idx_i, self.idx_t)) + + # update weights + update(self.w, learning_rate, err, X_col, self.num_examples) + + def predict(self, x): + """ make prediction """ + return int(np.dot(self.w, [1] + x)) + + +class LogisticLinearLeaner(LinearClassifier): + def __init__(self, dataset, learning_rate=0.01, epochs=100): + self.idx_i = dataset.inputs + self.idx_t = dataset.target + self.examples = dataset.examples + self.num_examples = len(self.examples) + # initialize random weights + self.w = random_weights(min_value=-0.5, max_value=0.5, num_weights=len(self.idx_i) + 1) + # learning loop + self.learn(learning_rate, epochs) + + def learn(self, learning_rate, epochs): + """ learning loop """ + def loss(example, w, idx_i, idx_t, h): + """ error: difference between estimation and true value """ + raise NotImplementedError + + def update(w, learning_rate, err, h, X_col, num_examples): + """ update weights """ + raise NotImplementedError + + def homogeneous(num_examples): + """ build homogeneous coordinates """ + raise NotImplementedError + + X_col = homogeneous(self.num_examples) + for epoch in range(epochs): + err = [] + h = [] + # pass over all examples + for example in self.examples: + err.append(loss(example, self.w, self.idx_i, self.idx_t, h)) + + # update weights + update(self.w, learning_rate, err, h, X_col, self.num_examples) + + def predict(self, x): + """ make prediction """ + return int(np.dot(self.w, [1] + x)) + + +if __name__ == "__main__": + iris = DataSet(name="iris") + iris.classes_to_numbers() + + tests = [([5, 3, 1, 0.1], 0), + ([5, 3.5, 1, 0], 0), + ([6, 3, 4, 1.1], 1), + ([6, 2, 3.5, 1], 1), + ([7.5, 4, 6, 2], 2), + ([7, 3, 6, 2.5], 2)] + + print(f'===================\nperceptron:') + perceptron = PerceptionLinearLearner(iris) + g = grade_learner(perceptron, tests) + print(f' learner grade: {g}') + # assert g > 1. / 2 + e = err_ratio(perceptron, iris) + print(f' error ration: {e}') + # assert e < 0.4 + + print(f'===================\nlogistic:') + logisticer = LogisticLinearLeaner(iris) + g = grade_learner(logisticer, tests) + print(f' learner grade: {g}') + # assert g > 1. / 2 + e = err_ratio(logisticer, iris) + print(f' error ration: {e}') + # assert e < 0.4 diff --git a/04_linear_classifier/LinearClassifier_3.py b/04_linear_classifier/LinearClassifier_3.py new file mode 100644 index 0000000..04d929d --- /dev/null +++ b/04_linear_classifier/LinearClassifier_3.py @@ -0,0 +1,109 @@ +import sys +sys.path.insert(1, '../') +from utils.utils import * +from utils.dataset4learners import * + + +class LinearClassifier: + def learn(self, learning_rate, epochs): + raise NotImplementedError + + def predict(self, x): + raise NotImplementedError + + +class PerceptionLinearLearner(LinearClassifier): + """ + Perception linear classifier: hard threshold + """ + def __init__(self, dataset, learning_rate=0.01, epochs=100): + self.idx_i = dataset.inputs + self.idx_t = dataset.target + self.examples = dataset.examples + self.num_examples = len(self.examples) + # initialize random weights + self.w = random_weights(min_value=-0.5, max_value=0.5, num_weights=len(self.idx_i) + 1) + # learning loop + self.learn(learning_rate, epochs) + + def learn(self, learning_rate, epochs): + """ learning loop """ + def loss(example, w, idx_i, idx_t): + """ error: difference between estimation and true value """ + raise NotImplementedError + + def update(w, learning_rate, err, X_col, num_examples): + """ update weights """ + raise NotImplementedError + + def homogeneous(num_examples): + """ build homogeneous coordinates """ + raise NotImplementedError + + raise NotImplementedError + + def predict(self, x): + """ make prediction """ + return int(np.dot(self.w, [1] + x)) + + +class LogisticLinearLeaner(LinearClassifier): + def __init__(self, dataset, learning_rate=0.01, epochs=100): + self.idx_i = dataset.inputs + self.idx_t = dataset.target + self.examples = dataset.examples + self.num_examples = len(self.examples) + # initialize random weights + self.w = random_weights(min_value=-0.5, max_value=0.5, num_weights=len(self.idx_i) + 1) + # learning loop + self.learn(learning_rate, epochs) + + def learn(self, learning_rate, epochs): + """ learning loop """ + def loss(example, w, idx_i, idx_t, h): + """ error: difference between estimation and true value """ + raise NotImplementedError + + def update(w, learning_rate, err, h, X_col, num_examples): + """ update weights """ + raise NotImplementedError + + def homogeneous(num_examples): + """ build homogeneous coordinates """ + raise NotImplementedError + + raise NotImplementedError + + def predict(self, x): + """ make prediction """ + return int(np.dot(self.w, [1] + x)) + + +if __name__ == "__main__": + iris = DataSet(name="iris") + iris.classes_to_numbers() + + tests = [([5, 3, 1, 0.1], 0), + ([5, 3.5, 1, 0], 0), + ([6, 3, 4, 1.1], 1), + ([6, 2, 3.5, 1], 1), + ([7.5, 4, 6, 2], 2), + ([7, 3, 6, 2.5], 2)] + + print(f'===================\nperceptron:') + perceptron = PerceptionLinearLearner(iris) + g = grade_learner(perceptron, tests) + print(f' learner grade: {g}') + # assert g > 1. / 2 + e = err_ratio(perceptron, iris) + print(f' error ration: {e}') + # assert e < 0.4 + + print(f'===================\nlogistic:') + logisticer = LogisticLinearLeaner(iris) + g = grade_learner(logisticer, tests) + print(f' learner grade: {g}') + # assert g > 1. / 2 + e = err_ratio(logisticer, iris) + print(f' error ration: {e}') + # assert e < 0.4 diff --git a/04_linear_classifier/expected_output.txt b/04_linear_classifier/expected_output.txt new file mode 100644 index 0000000..17f9615 --- /dev/null +++ b/04_linear_classifier/expected_output.txt @@ -0,0 +1,8 @@ +=================== +perceptron: + learner grade: 0.5 + error ration: 0.3466666666666667 +=================== +logistic: + learner grade: 0.3333333333333333 + error ration: 0.6066666666666667 \ No newline at end of file diff --git a/04_linear_classifier/perceptron.png b/04_linear_classifier/perceptron.png new file mode 100644 index 0000000..68d2a25 Binary files /dev/null and b/04_linear_classifier/perceptron.png differ diff --git a/05_feedforward/05_feedforward.ipynb b/05_feedforward/05_feedforward.ipynb new file mode 100644 index 0000000..9442a9d --- /dev/null +++ b/05_feedforward/05_feedforward.ipynb @@ -0,0 +1,7659 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d5c95d6d-fd0f-4cc6-8b5d-3649e095c6fe", + "metadata": {}, + "source": [ + "# 前馈神经网络\n", + "\n", + "目录\n", + "\n", + "- 多层感知机\n", + "- 多层感知机从零开始实现\n", + "- 多层感知机的简洁实现" + ] + }, + { + "cell_type": "markdown", + "id": "2bed1efe-ef77-4d6a-a8e1-94a7f91c9730", + "metadata": { + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 多层感知机\n", + "\n", + "简要介绍一些常见的激活函数" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "80872a1f-275a-4037-b931-c45f7b825e8c", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import torch\n", + "from d2l import torch as d2l" + ] + }, + { + "cell_type": "markdown", + "id": "c8a3f328-4f3e-4014-9d0f-d40ac76786af", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "ReLU提供了一种非常简单的非线性变换\n", + "$$\\operatorname{ReLU}(x) = \\max(x, 0)$$" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "35352b67-f94b-41d1-996e-0ba8920e3d68", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:29:41.289771\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "x = torch.arange(-8.0, 8.0, 0.1, requires_grad=True)\n", + "y = torch.relu(x)\n", + "d2l.plot(x.detach(), y.detach(), 'x', 'relu(x)', figsize=(5, 2.5))" + ] + }, + { + "cell_type": "markdown", + "id": "0ea846b7-41a2-4ff2-97b5-256aff79d596", + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "source": [ + "`retain_graph=True`: prevent memory deallocation, so parts of the compute graph can be referenced later." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "354ce38b-c8e9-4666-8688-9f0033ea467b", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:29:41.587126\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y.backward(torch.ones_like(x), retain_graph=True)\n", + "d2l.plot(x.detach(), x.grad, 'x', 'grad of relu', figsize=(5, 2.5))" + ] + }, + { + "cell_type": "markdown", + "id": "b411349e-3256-4b49-8f37-30327368689c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "对于一个定义域在$\\mathbb{R}$中的输入,\n", + "*sigmoid函数*将输入变换为区间(0, 1)上的输出\n", + "$$\\operatorname{sigmoid}(x) = \\frac{1}{1 + \\exp(-x)}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1813b3e3-2bec-490d-bcf2-14b64c484e5c", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:29:41.817942\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y = torch.sigmoid(x)\n", + "d2l.plot(x.detach(), y.detach(), 'x', 'sigmoid(x)', figsize=(5, 2.5))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "b0186f7b-106c-40dc-a4dc-9441ee2a0ab1", + "metadata": { + "origin_pos": 18, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:29:42.064628\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "x.grad.data.zero_() # 清除以前的梯度\n", + "y.backward(torch.ones_like(x),retain_graph=True)\n", + "d2l.plot(x.detach(), x.grad, 'x', 'grad of sigmoid', figsize=(5, 2.5))" + ] + }, + { + "cell_type": "markdown", + "id": "85c3817f-e76f-47b2-8921-435f9d8ea19a", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Tanh(双曲正切)函数也能将其输入压缩转换到区间(-1, 1)上\n", + "$$\\operatorname{tanh}(x) = \\frac{1 - \\exp(-2x)}{1 + \\exp(-2x)}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "09ce59c5-f089-40d3-b72e-ec65c0c74e35", + "metadata": { + "origin_pos": 22, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:29:42.307511\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "y = torch.tanh(x)\n", + "d2l.plot(x.detach(), y.detach(), 'x', 'tanh(x)', figsize=(5, 2.5))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "4fe3ec86-9940-4d1f-946a-0efeb5ab4c37", + "metadata": { + "origin_pos": 26, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:29:42.555226\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "x.grad.data.zero_() # 清除以前的梯度\n", + "y.backward(torch.ones_like(x),retain_graph=True)\n", + "d2l.plot(x.detach(), x.grad, 'x', 'grad of tanh', figsize=(5, 2.5))" + ] + }, + { + "cell_type": "markdown", + "id": "641485cc-5306-49ec-8cbc-97601d8922af", + "metadata": { + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 多层感知机从零开始实现\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "850f9472-762c-4525-9376-ac170b27db83", + "metadata": { + "origin_pos": 4, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "batch_size = 256\n", + "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)" + ] + }, + { + "cell_type": "markdown", + "id": "667c9915-0f1c-4f28-9176-40a7cdafdf36", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "实现一个具有单隐藏层的多层感知机,\n", + "它包含256个隐藏单元" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "44bea74f-6f0f-4807-9473-b251c2c1ac28", + "metadata": { + "origin_pos": 7, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "num_inputs, num_outputs, num_hiddens = 784, 10, 256\n", + "\n", + "W1 = nn.Parameter(torch.randn(\n", + " num_inputs, num_hiddens, requires_grad=True) * 0.01)\n", + "b1 = nn.Parameter(torch.zeros(num_hiddens, requires_grad=True))\n", + "W2 = nn.Parameter(torch.randn(\n", + " num_hiddens, num_outputs, requires_grad=True) * 0.01)\n", + "b2 = nn.Parameter(torch.zeros(num_outputs, requires_grad=True))\n", + "\n", + "params = [W1, b1, W2, b2]" + ] + }, + { + "cell_type": "markdown", + "id": "da8f4700-315d-4249-83d0-bae1a4807d01", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "实现ReLU激活函数" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "68d398da-b046-4ad2-8c72-d5f2544bf68f", + "metadata": { + "origin_pos": 11, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def relu(X):\n", + " a = torch.zeros_like(X)\n", + " return torch.max(X, a)" + ] + }, + { + "cell_type": "markdown", + "id": "0dc3b5a5-dce1-4027-bf78-260fefb376df", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "实现我们的模型" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "cdfb32c3-c3a4-46cf-89c4-e4e1ab46cb20", + "metadata": { + "origin_pos": 19, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def net(X):\n", + " X = X.reshape((-1, num_inputs))\n", + " H = relu(X@W1 + b1) # 这里“@”代表矩阵乘法\n", + " return (H@W2 + b2)\n", + "\n", + "loss = nn.CrossEntropyLoss(reduction='none')" + ] + }, + { + "cell_type": "markdown", + "id": "3c233b99-3507-40c0-a185-bc8da9c16f68", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "多层感知机的训练过程与softmax回归的训练过程完全相同" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "c6254fc2-e783-4305-80d0-52b5bbf5108d", + "metadata": { + "origin_pos": 23, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:32:34.406355\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "num_epochs, lr = 10, 0.1\n", + "updater = torch.optim.SGD(params, lr=lr)\n", + "d2l.train_ch3(net, train_iter, test_iter, loss, num_epochs, updater)" + ] + }, + { + "cell_type": "markdown", + "id": "dc0ce797-d446-4a54-9a62-e1fc421854c1", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "在一些测试数据上应用这个模型" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "fcc61869-115c-408e-affd-ef36327ac7b8", + "metadata": { + "origin_pos": 26, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:32:40.968428\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "d2l.predict_ch3(net, test_iter)" + ] + }, + { + "cell_type": "markdown", + "id": "e35e6756-4c65-4a63-a512-41c831045596", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "## 多层感知机的简洁实现\n", + "\n", + "通过高级API更简洁地实现多层感知机" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "31883579-70e2-4930-ae26-72a7e7a2cce2", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l" + ] + }, + { + "cell_type": "markdown", + "id": "2547eec8-64d0-4a97-9e3e-75bd80db3c98", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "隐藏层\n", + "包含256个隐藏单元,并使用了ReLU激活函数" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "8316e7aa-873e-436b-bee5-9d5cf7c1e298", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "net = nn.Sequential(nn.Flatten(),\n", + " nn.Linear(784, 256),\n", + " nn.ReLU(),\n", + " nn.Linear(256, 10))\n", + "\n", + "def init_weights(m):\n", + " if type(m) == nn.Linear:\n", + " nn.init.normal_(m.weight, std=0.01)\n", + "\n", + "net.apply(init_weights);" + ] + }, + { + "cell_type": "markdown", + "id": "4a347357-8fac-4cf6-9baa-0a8d8d7c9790", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练过程" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "6d2b9cb2-e525-46e4-9283-0d059148d0e0", + "metadata": { + "origin_pos": 12, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:36:23.434996\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "batch_size, lr, num_epochs = 256, 0.1, 10\n", + "loss = nn.CrossEntropyLoss(reduction='none')\n", + "trainer = torch.optim.SGD(net.parameters(), lr=lr)\n", + "\n", + "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)\n", + "d2l.train_ch3(net, train_iter, test_iter, loss, num_epochs, trainer)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/05_feedforward/FeedForward-1.py b/05_feedforward/FeedForward-1.py new file mode 100644 index 0000000..ffc5a8d --- /dev/null +++ b/05_feedforward/FeedForward-1.py @@ -0,0 +1,150 @@ +import torch + +import torch.nn as nn +import torch.nn.functional as F + +import torch.optim as optim + +import torchvision +import torchvision.transforms as transforms + + +class FFNet(nn.Module): + """ + Feedforward neural network, virtual base class + """ + def __init__(self): + super(FFNet, self).__init__() + self.dnn_model = self.build_model() + + def build_model(self): + """ build specific model """ + raise NotImplementedError + + def forward(self, x): + """ feed data forward and return result """ + raise NotImplementedError + + def train_model(self, trainloader, testloader, loss_fn, optimizer, num_epochs): + """Train a model.""" + + def train_epoch(model, dataloader, loss_fn, optimizer): + """Train a single epoch""" + num_data = len(dataloader.dataset) + # Set the model to training mode + model.train() + for batch, (X, y) in enumerate(dataloader): + # Compute prediction error + pred = model(X) + loss = loss_fn(pred, y) + + # Backpropagation + optimizer.zero_grad() + loss.backward() + optimizer.step() + + if batch % 100 == 0: + loss, current = loss.item(), batch * len(X) + print(f"loss: {loss:>7f} [{current:>5d}/{num_data:>5d}]") + + def test_epoch(model, dataloader, loss_fn): + """Test a single epoch""" + num_data = len(dataloader.dataset) + num_batches = len(dataloader) + # Set the model to evaluate mode + model.eval() + test_loss, correct = 0, 0 + with torch.no_grad(): + for X, y in dataloader: + pred = model(X) + test_loss += loss_fn(pred, y).item() + correct += (pred.argmax(1) == y).type(torch.float).sum().item() + test_loss /= num_batches + correct /= num_data + print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n") + + for epoch in range(num_epochs): + print(f"Epoch {epoch+1}\n-------------------------------") + train_epoch(self, trainloader, loss_fn, optimizer) + test_epoch(self, testloader, loss_fn) + print("Done!") + + def make_predict(self, images): + """ predict labels for images """ + self.eval() + with torch.no_grad(): + outputs = self(images) + _, predicted = torch.max(outputs.data, 1) # (max, max_indices) + return predicted + + def evaluate_model(self, testloader, n): + """ evaluation: return accuracy """ + correct = 0 + for inputs, labels in testloader: + pred = self.make_predict(inputs) + correct += (pred == labels).sum() + return 100 * correct / n + + def predict_one(self, x): + self.eval() + with torch.no_grad(): + outputs = self(x) + predicted = outputs[0].argmax(0) + return predicted + + +class MLP(FFNet): + def __init__(self): + super(MLP, self).__init__() + + def build_model(self): + """ build specific model """ + raise NotImplementedError + + def forward(self, x): + """ feed data forward and return result """ + raise NotImplementedError + + +if __name__ == "__main__": + # load train and test set + trainset = torchvision.datasets.MNIST( + '../data', train=True, download=True, transform=transforms.ToTensor()) + testset = torchvision.datasets.MNIST( + '../data', train=False, download=True, transform=transforms.ToTensor()) + print(f'Train #: {len(trainset)}; Test #: {len(testset)}') + # data iterator + dataiter = iter(torch.utils.data.DataLoader(trainset, batch_size=8, shuffle=False)) + images, labels = next(dataiter) + print(f'Labels: {labels}; Batch shape: {images.size()}') + + # construct model + model = MLP() + print(model) + + # data loader: easier iteration + BATCH_SIZE = 256 + NUM_WORKERS = 4 + trainloader = torch.utils.data.DataLoader( + trainset, batch_size=BATCH_SIZE, shuffle=True, num_workers=NUM_WORKERS) + testloader = torch.utils.data.DataLoader( + testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS) + + # train the model + loss_fn = nn.CrossEntropyLoss() # cross entropy + optimizer = torch.optim.SGD(model.parameters(), lr=0.1) # SGD + NUM_EPOCH = 5 + model.train_model(trainloader, testloader, loss_fn, optimizer, NUM_EPOCH) + + # check prediction accuracy + print(f'Labels : {labels}') + print(f'Prediction: {model.make_predict(images)}') + print(f'Accuracy: {model.evaluate_model(testloader, len(testset)):.2f}') + + # application: predict hand-written digit + from PIL import Image + image = Image.open('number6c.png') + image = transforms.ToTensor()(image).unsqueeze(0) + print(f'loaded image shape: {image.size()}') + print(f'Predicted: "{model.predict_one(image)}", Actual: "{6}"') + \ No newline at end of file diff --git a/05_feedforward/FeedForward-2.py b/05_feedforward/FeedForward-2.py new file mode 100644 index 0000000..008091b --- /dev/null +++ b/05_feedforward/FeedForward-2.py @@ -0,0 +1,142 @@ +import torch + +import torch.nn as nn +import torch.nn.functional as F + +import torch.optim as optim + +import torchvision +import torchvision.transforms as transforms + + +class FFNet(nn.Module): + """ + Feedforward neural network, virtual base class + """ + def __init__(self): + super(FFNet, self).__init__() + self.dnn_model = self.build_model() + + def build_model(self): + """ build specific model """ + raise NotImplementedError + + def forward(self, x): + """ feed data forward and return result """ + raise NotImplementedError + + def train_model(self, trainloader, testloader, loss_fn, optimizer, num_epochs): + """Train a model.""" + + def train_epoch(model, dataloader, loss_fn, optimizer): + """Train a single epoch""" + num_data = len(dataloader.dataset) + # Set the model to training mode + model.train() + for batch, (X, y) in enumerate(dataloader): + # Compute prediction error + pred = model(X) + loss = loss_fn(pred, y) + + # Backpropagation + optimizer.zero_grad() + loss.backward() + optimizer.step() + + if batch % 100 == 0: + loss, current = loss.item(), batch * len(X) + print(f"loss: {loss:>7f} [{current:>5d}/{num_data:>5d}]") + + def test_epoch(model, dataloader, loss_fn): + """Test a single epoch""" + num_data = len(dataloader.dataset) + num_batches = len(dataloader) + # Set the model to evaluate mode + model.eval() + test_loss, correct = 0, 0 + with torch.no_grad(): + for X, y in dataloader: + pred = model(X) + test_loss += loss_fn(pred, y).item() + correct += (pred.argmax(1) == y).type(torch.float).sum().item() + test_loss /= num_batches + correct /= num_data + print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n") + + raise NotImplementedError + + def make_predict(self, images): + """ predict labels for images """ + self.eval() + with torch.no_grad(): + outputs = self(images) + _, predicted = torch.max(outputs.data, 1) # (max, max_indices) + return predicted + + def evaluate_model(self, testloader, n): + """ evaluation: return accuracy """ + raise NotImplementedError + + def predict_one(self, x): + self.eval() + with torch.no_grad(): + outputs = self(x) + predicted = outputs[0].argmax(0) + return predicted + + +class MLP(FFNet): + def __init__(self): + super(MLP, self).__init__() + + def build_model(self): + """ build specific model """ + raise NotImplementedError + + def forward(self, x): + """ feed data forward and return result """ + raise NotImplementedError + + +if __name__ == "__main__": + # load train and test set + trainset = torchvision.datasets.MNIST( + '../data', train=True, download=True, transform=transforms.ToTensor()) + testset = torchvision.datasets.MNIST( + '../data', train=False, download=True, transform=transforms.ToTensor()) + print(f'Train #: {len(trainset)}; Test #: {len(testset)}') + # data iterator + dataiter = iter(torch.utils.data.DataLoader(trainset, batch_size=8, shuffle=False)) + images, labels = next(dataiter) + print(f'Labels: {labels}; Batch shape: {images.size()}') + + # construct model + model = MLP() + print(model) + + # data loader: easier iteration + BATCH_SIZE = 256 + NUM_WORKERS = 4 + trainloader = torch.utils.data.DataLoader( + trainset, batch_size=BATCH_SIZE, shuffle=True, num_workers=NUM_WORKERS) + testloader = torch.utils.data.DataLoader( + testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS) + + # train the model + loss_fn = nn.CrossEntropyLoss() # cross entropy + optimizer = torch.optim.SGD(model.parameters(), lr=0.1) # SGD + NUM_EPOCH = 5 + model.train_model(trainloader, testloader, loss_fn, optimizer, NUM_EPOCH) + + # check prediction accuracy + print(f'Labels : {labels}') + print(f'Prediction: {model.make_predict(images)}') + print(f'Accuracy: {model.evaluate_model(testloader, len(testset)):.2f}') + + # application: predict hand-written digit + from PIL import Image + image = Image.open('number6c.png') + image = transforms.ToTensor()(image).unsqueeze(0) + print(f'loaded image shape: {image.size()}') + print(f'Predicted: "{model.predict_one(image)}", Actual: "{6}"') + \ No newline at end of file diff --git a/05_feedforward/FeedForward-3.py b/05_feedforward/FeedForward-3.py new file mode 100644 index 0000000..b60bacc --- /dev/null +++ b/05_feedforward/FeedForward-3.py @@ -0,0 +1,107 @@ +import torch + +import torch.nn as nn +import torch.nn.functional as F + +import torch.optim as optim + +import torchvision +import torchvision.transforms as transforms + + +class FFNet(nn.Module): + """ + Feedforward neural network, virtual base class + """ + def __init__(self): + super(FFNet, self).__init__() + self.dnn_model = self.build_model() + + def build_model(self): + """ build specific model """ + raise NotImplementedError + + def forward(self, x): + """ feed data forward and return result """ + raise NotImplementedError + + def train_model(self, trainloader, testloader, loss_fn, optimizer, num_epochs): + """Train a model.""" + + def train_epoch(model, dataloader, loss_fn, optimizer): + """Train a single epoch""" + raise NotImplementedError + + def test_epoch(model, dataloader, loss_fn): + """Test a single epoch""" + raise NotImplementedError + + raise NotImplementedError + + def make_predict(self, images): + """ predict labels for images """ + raise NotImplementedError + + def evaluate_model(self, testloader, n): + """ evaluation: return accuracy """ + raise NotImplementedError + + def predict_one(self, x): + raise NotImplementedError + + +class MLP(FFNet): + def __init__(self): + super(MLP, self).__init__() + + def build_model(self): + """ build specific model """ + raise NotImplementedError + + def forward(self, x): + """ feed data forward and return result """ + raise NotImplementedError + + +if __name__ == "__main__": + # load train and test set + trainset = torchvision.datasets.MNIST( + '../data', train=True, download=True, transform=transforms.ToTensor()) + testset = torchvision.datasets.MNIST( + '../data', train=False, download=True, transform=transforms.ToTensor()) + print(f'Train #: {len(trainset)}; Test #: {len(testset)}') + # data iterator + dataiter = iter(torch.utils.data.DataLoader(trainset, batch_size=8, shuffle=False)) + images, labels = next(dataiter) + print(f'Labels: {labels}; Batch shape: {images.size()}') + + # construct model + model = MLP() + print(model) + + # data loader: easier iteration + BATCH_SIZE = 256 + NUM_WORKERS = 4 + trainloader = torch.utils.data.DataLoader( + trainset, batch_size=BATCH_SIZE, shuffle=True, num_workers=NUM_WORKERS) + testloader = torch.utils.data.DataLoader( + testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS) + + # train the model + loss_fn = nn.CrossEntropyLoss() # cross entropy + optimizer = torch.optim.SGD(model.parameters(), lr=0.1) # SGD + NUM_EPOCH = 5 + model.train_model(trainloader, testloader, loss_fn, optimizer, NUM_EPOCH) + + # check prediction accuracy + print(f'Labels : {labels}') + print(f'Prediction: {model.make_predict(images)}') + print(f'Accuracy: {model.evaluate_model(testloader, len(testset)):.2f}') + + # application: predict hand-written digit + from PIL import Image + image = Image.open('number6c.png') + image = transforms.ToTensor()(image).unsqueeze(0) + print(f'loaded image shape: {image.size()}') + print(f'Predicted: "{model.predict_one(image)}", Actual: "{6}"') + \ No newline at end of file diff --git a/05_feedforward/expected_output.txt b/05_feedforward/expected_output.txt new file mode 100644 index 0000000..870f508 --- /dev/null +++ b/05_feedforward/expected_output.txt @@ -0,0 +1,56 @@ +Train #: 60000; Test #: 10000 +Labels: tensor([5, 0, 4, 1, 9, 2, 1, 3]); Batch shape: torch.Size([8, 1, 28, 28]) +MLP( + (flatten): Flatten(start_dim=1, end_dim=-1) + (dnn_model): Sequential( + (0): Linear(in_features=784, out_features=256, bias=True) + (1): ReLU() + (2): Linear(in_features=256, out_features=10, bias=True) + ) +) +Epoch 1 +------------------------------- +loss: 2.308738 [ 0/60000] +loss: 0.643267 [25600/60000] +loss: 0.384817 [51200/60000] +Test Error: + Accuracy: 90.0%, Avg loss: 0.368403 + +Epoch 2 +------------------------------- +loss: 0.323617 [ 0/60000] +loss: 0.300336 [25600/60000] +loss: 0.268706 [51200/60000] +Test Error: + Accuracy: 91.6%, Avg loss: 0.301624 + +Epoch 3 +------------------------------- +loss: 0.314411 [ 0/60000] +loss: 0.337723 [25600/60000] +loss: 0.234745 [51200/60000] +Test Error: + Accuracy: 92.4%, Avg loss: 0.267629 + +Epoch 4 +------------------------------- +loss: 0.258774 [ 0/60000] +loss: 0.222343 [25600/60000] +loss: 0.363658 [51200/60000] +Test Error: + Accuracy: 92.9%, Avg loss: 0.245184 + +Epoch 5 +------------------------------- +loss: 0.365407 [ 0/60000] +loss: 0.286917 [25600/60000] +loss: 0.213272 [51200/60000] +Test Error: + Accuracy: 93.9%, Avg loss: 0.222085 + +Done! +Labels : tensor([5, 0, 4, 1, 9, 2, 1, 3]) +Prediction: tensor([5, 0, 4, 1, 9, 2, 1, 3]) +Accuracy: 93.87 +loaded image shape: torch.Size([1, 1, 28, 28]) +Predicted: "3", Actual: "6" \ No newline at end of file diff --git a/05_feedforward/number5.png b/05_feedforward/number5.png new file mode 100644 index 0000000..24eb1c6 Binary files /dev/null and b/05_feedforward/number5.png differ diff --git a/05_feedforward/number6c.png b/05_feedforward/number6c.png new file mode 100644 index 0000000..ab336d5 Binary files /dev/null and b/05_feedforward/number6c.png differ diff --git a/06_deep_compute/05_deep_compute.ipynb b/06_deep_compute/05_deep_compute.ipynb new file mode 100644 index 0000000..608e5a3 --- /dev/null +++ b/06_deep_compute/05_deep_compute.ipynb @@ -0,0 +1,3105 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "25419d99-c8f7-4843-be39-7f21c7dd5266", + "metadata": { + "tags": [] + }, + "source": [ + "# 深度计算\n", + "\n", + "目录\n", + "\n", + "- 数值稳定性和模型初始化\n", + "- 层和块\n", + "- 参数管理\n", + "- 自定义层\n", + "- 读写文件\n", + "- GPU" + ] + }, + { + "cell_type": "markdown", + "id": "31aec33a-26b5-434f-8227-bcbea1b2fe10", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 数值稳定性和模型初始化\n", + "\n", + "梯度消失" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "f7f3e77b-9245-4c83-bad0-24eca173455e", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-03-08T18:45:41.490781\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "%matplotlib inline\n", + "import torch\n", + "from d2l import torch as d2l\n", + "\n", + "x = torch.arange(-8.0, 8.0, 0.1, requires_grad=True)\n", + "y = torch.sigmoid(x)\n", + "y.backward(torch.ones_like(x))\n", + "\n", + "d2l.plot(x.detach().numpy(), [y.detach().numpy(), x.grad.numpy()],\n", + " legend=['sigmoid', 'gradient'], figsize=(4.5, 2.5))" + ] + }, + { + "cell_type": "markdown", + "id": "f2cc49cd-aa08-4439-babb-5d7e2cefdb97", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "梯度爆炸" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b97a9ac6-93e5-4302-9637-beeff9caf0f8", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "一个矩阵 \n", + " tensor([[ 0.5039, -0.5113, 0.2666, 0.9192],\n", + " [ 0.8290, -0.3719, -0.4758, 0.2095],\n", + " [-2.8356, 1.5128, 3.2530, 1.2554],\n", + " [-0.4369, 0.0571, 1.3445, 0.5465]])\n", + "乘以100个矩阵后\n", + " tensor([[-2.6163e+27, -1.3641e+27, 3.7875e+26, 1.5981e+27],\n", + " [ 9.2931e+25, 4.8452e+25, -1.3453e+25, -5.6765e+25],\n", + " [-8.5595e+27, -4.4627e+27, 1.2391e+27, 5.2284e+27],\n", + " [-3.1940e+27, -1.6653e+27, 4.6239e+26, 1.9510e+27]])\n" + ] + } + ], + "source": [ + "M = torch.normal(0, 1, size=(4,4))\n", + "print('一个矩阵 \\n',M)\n", + "for i in range(100):\n", + " M = torch.mm(M,torch.normal(0, 1, size=(4, 4)))\n", + "\n", + "print('乘以100个矩阵后\\n', M)" + ] + }, + { + "cell_type": "markdown", + "id": "c5bbc300-2e11-46b5-97a0-8a94fbb355b0", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 层和块\n", + "\n", + "我们先回顾一下多层感知机" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "6bb8a16d-5ecc-454a-858d-4e9e1e3e2332", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[-0.0148, -0.0091, -0.1044, -0.0623, 0.1513, 0.0991, 0.1245, -0.1850,\n", + " 0.0858, 0.1818],\n", + " [-0.0970, 0.0267, 0.0026, -0.0933, -0.0292, 0.1253, 0.2153, -0.0900,\n", + " 0.0387, 0.0119]], grad_fn=)" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "from torch import nn\n", + "from torch.nn import functional as F\n", + "\n", + "net = nn.Sequential(nn.Linear(20, 256), nn.ReLU(), nn.Linear(256, 10))\n", + "\n", + "X = torch.rand(2, 20)\n", + "net(X)" + ] + }, + { + "cell_type": "markdown", + "id": "1a7e60b4-5af9-447d-8461-47455b5427b5", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "`nn.Sequential`定义了一种特殊的`Module`" + ] + }, + { + "cell_type": "markdown", + "id": "17417747-7867-4cc9-b6a6-6bea3c5fe307", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "自定义块" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c010a3fd-b10d-4862-8326-114deea44c9f", + "metadata": { + "origin_pos": 12, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "class MLP(nn.Module):\n", + " # 用模型参数声明层。这里,我们声明两个全连接的层\n", + " def __init__(self):\n", + " # 调用MLP的父类Module的构造函数来执行必要的初始化。\n", + " # 这样,在类实例化时也可以指定其他函数参数,例如模型参数params(稍后将介绍)\n", + " super().__init__()\n", + " self.hidden = nn.Linear(20, 256) # 隐藏层\n", + " self.out = nn.Linear(256, 10) # 输出层\n", + "\n", + " # 定义模型的前向传播,即如何根据输入X返回所需的模型输出\n", + " def forward(self, X):\n", + " # 注意,这里我们使用ReLU的函数版本,其在nn.functional模块中定义。\n", + " return self.out(F.relu(self.hidden(X)))" + ] + }, + { + "cell_type": "markdown", + "id": "514876b3-07c1-4238-b767-935fda3a3941", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "实例化多层感知机的层,然后在每次调用前向传播函数时调用这些层" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "97818129-99da-40e8-8079-fcae5484d270", + "metadata": { + "origin_pos": 16, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 0.2511, 0.1825, 0.0906, -0.1014, -0.0818, -0.2257, 0.0343, -0.0502,\n", + " -0.1205, 0.1808],\n", + " [ 0.0538, 0.2270, -0.0294, -0.0254, 0.0274, -0.1483, -0.1254, 0.0424,\n", + " -0.1205, 0.1796]], grad_fn=)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "net = MLP()\n", + "net(X)" + ] + }, + { + "cell_type": "markdown", + "id": "2647ff40-26ca-4e20-83de-9c92aca16272", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "顺序块" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2d84d2ab-f4b0-473a-9cfd-688f8b9be647", + "metadata": { + "origin_pos": 26, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "class MySequential(nn.Module):\n", + " def __init__(self, *args):\n", + " super().__init__()\n", + " for idx, module in enumerate(args):\n", + " # 这里,module是Module子类的一个实例。我们把它保存在'Module'类的成员\n", + " # 变量_modules中。module的类型是OrderedDict\n", + " self._modules[str(idx)] = module\n", + "\n", + " def forward(self, X):\n", + " # OrderedDict保证了按照成员添加的顺序遍历它们\n", + " for block in self._modules.values():\n", + " X = block(X)\n", + " return X" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7e7ee471-4a2b-4df2-a313-c83018c7ef92", + "metadata": { + "origin_pos": 26, + "slideshow": { + "slide_type": "-" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[-0.0345, 0.0057, 0.0227, -0.1984, -0.2237, -0.0464, -0.0189, -0.2631,\n", + " -0.2752, -0.1415],\n", + " [-0.1851, 0.0243, -0.0286, -0.1372, -0.1288, -0.1205, 0.1659, -0.3036,\n", + " -0.2170, -0.0545]], grad_fn=)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "net = MySequential(nn.Linear(20, 256), nn.ReLU(), nn.Linear(256, 10))\n", + "net(X)" + ] + }, + { + "cell_type": "markdown", + "id": "b13e8e0b-975d-4d24-a517-a1b7ed95712a", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "在前向传播函数中执行代码" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "8303f511-6ec1-4971-bc3a-ebc431b04b46", + "metadata": { + "origin_pos": 34, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(0.0362, grad_fn=)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class FixedHiddenMLP(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " # 不计算梯度的随机权重参数。因此其在训练期间保持不变\n", + " self.rand_weight = torch.rand((20, 20), requires_grad=False)\n", + " self.linear = nn.Linear(20, 20)\n", + "\n", + " def forward(self, X):\n", + " X = self.linear(X)\n", + " # 使用创建的常量参数以及relu和mm函数\n", + " X = F.relu(torch.mm(X, self.rand_weight) + 1)\n", + " # 复用全连接层。这相当于两个全连接层共享参数\n", + " X = self.linear(X)\n", + " # 控制流\n", + " while X.abs().sum() > 1:\n", + " X /= 2\n", + " return X.sum()\n", + "\n", + "net = FixedHiddenMLP()\n", + "net(X)" + ] + }, + { + "cell_type": "markdown", + "id": "d9275f5f-ae9c-4ac3-a17e-e497ccca62d2", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "混合搭配各种组合块的方法" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "2dab1c0a-9d6c-43ef-8fd2-e41bace79bf5", + "metadata": { + "origin_pos": 37, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(-0.1137, grad_fn=)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class NestMLP(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.net = nn.Sequential(nn.Linear(20, 64), nn.ReLU(),\n", + " nn.Linear(64, 32), nn.ReLU())\n", + " self.linear = nn.Linear(32, 16)\n", + "\n", + " def forward(self, X):\n", + " return self.linear(self.net(X))\n", + "\n", + "chimera = nn.Sequential(NestMLP(), nn.Linear(16, 20), FixedHiddenMLP())\n", + "chimera(X)" + ] + }, + { + "cell_type": "markdown", + "id": "5491e9df-11d8-4dfa-a0da-07a0dc06d661", + "metadata": { + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 参数管理\n", + "\n", + "我们首先看一下具有单隐藏层的多层感知机" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "20a7bea9-9dbf-4c3b-a11a-e78fef1c42c7", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[-0.1909],\n", + " [-0.2025]], grad_fn=)" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "from torch import nn\n", + "\n", + "net = nn.Sequential(nn.Linear(4, 8), nn.ReLU(), nn.Linear(8, 1))\n", + "X = torch.rand(size=(2, 4))\n", + "net(X)" + ] + }, + { + "cell_type": "markdown", + "id": "1d1b0906-6939-4bc0-a93e-c7ea0870f955", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### 参数访问" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a91089e7-a386-4915-b2ab-70503785ca50", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "OrderedDict([('weight', tensor([[-0.2358, -0.2256, -0.1930, -0.0475, -0.0732, -0.3483, 0.0520, 0.1466]])), ('bias', tensor([-0.0579]))])\n" + ] + } + ], + "source": [ + "print(net[2].state_dict())" + ] + }, + { + "cell_type": "markdown", + "id": "8d80d652-2846-4cb0-acf9-adec725b41bf", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### 目标参数\n", + "\n", + "注意,每个参数都表示为参数类的一个实例。 要对参数执行任何操作,首先我们需要访问底层的数值。" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "72fb1f60-b259-46c9-9033-d3ce3d13ef74", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Parameter containing:\n", + "tensor([-0.0579], requires_grad=True)\n", + "tensor([-0.0579])\n" + ] + } + ], + "source": [ + "print(type(net[2].bias))\n", + "print(net[2].bias)\n", + "print(net[2].bias.data)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "e6cc0743-663b-4452-8b65-53a6652e44f1", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "net[2].weight.grad == None" + ] + }, + { + "cell_type": "markdown", + "id": "35a83ecb-6df8-4cb6-a0ce-29f521b35d6e", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### 一次性访问所有参数" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "cdaf87e6-a880-4f9e-94c3-e7dc48e48878", + "metadata": { + "origin_pos": 17, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "('weight', torch.Size([8, 4])) ('bias', torch.Size([8]))\n", + "('0.weight', torch.Size([8, 4])) ('0.bias', torch.Size([8])) ('2.weight', torch.Size([1, 8])) ('2.bias', torch.Size([1]))\n" + ] + } + ], + "source": [ + "print(*[(name, param.shape) for name, param in net[0].named_parameters()])\n", + "print(*[(name, param.shape) for name, param in net.named_parameters()])" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a826965f-f15b-4fb0-8711-f523cde61fb5", + "metadata": { + "origin_pos": 21, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([-0.0579])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "net.state_dict()['2.bias'].data" + ] + }, + { + "cell_type": "markdown", + "id": "71f979b6-86b5-4fdf-8364-1281369f188c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### 从嵌套块收集参数" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "6cd58e6f-6e87-4b9d-9286-dba5383e035a", + "metadata": { + "origin_pos": 25, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.3983],\n", + " [0.3983]], grad_fn=)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def block1():\n", + " return nn.Sequential(nn.Linear(4, 8), nn.ReLU(),\n", + " nn.Linear(8, 4), nn.ReLU())\n", + "\n", + "def block2():\n", + " net = nn.Sequential()\n", + " for i in range(4):\n", + " # 在这里嵌套\n", + " net.add_module(f'block {i}', block1())\n", + " return net\n", + "\n", + "rgnet = nn.Sequential(block2(), nn.Linear(4, 1))\n", + "rgnet(X)" + ] + }, + { + "cell_type": "markdown", + "id": "f2d9b23f-428a-4428-b2fc-a2c58e7c27db", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "设计了网络后,我们看看它是如何工作的" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "c77d9236-9e3c-4eba-9cac-cd929ceb538f", + "metadata": { + "origin_pos": 29, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sequential(\n", + " (0): Sequential(\n", + " (block 0): Sequential(\n", + " (0): Linear(in_features=4, out_features=8, bias=True)\n", + " (1): ReLU()\n", + " (2): Linear(in_features=8, out_features=4, bias=True)\n", + " (3): ReLU()\n", + " )\n", + " (block 1): Sequential(\n", + " (0): Linear(in_features=4, out_features=8, bias=True)\n", + " (1): ReLU()\n", + " (2): Linear(in_features=8, out_features=4, bias=True)\n", + " (3): ReLU()\n", + " )\n", + " (block 2): Sequential(\n", + " (0): Linear(in_features=4, out_features=8, bias=True)\n", + " (1): ReLU()\n", + " (2): Linear(in_features=8, out_features=4, bias=True)\n", + " (3): ReLU()\n", + " )\n", + " (block 3): Sequential(\n", + " (0): Linear(in_features=4, out_features=8, bias=True)\n", + " (1): ReLU()\n", + " (2): Linear(in_features=8, out_features=4, bias=True)\n", + " (3): ReLU()\n", + " )\n", + " )\n", + " (1): Linear(in_features=4, out_features=1, bias=True)\n", + ")\n" + ] + } + ], + "source": [ + "print(rgnet)" + ] + }, + { + "cell_type": "markdown", + "id": "8af6c129-6b29-4bf0-afdf-93b71f68cf43", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "通过嵌套列表索引访问" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "05b273c1-1f6c-4d6d-b2df-2d204b6a60b6", + "metadata": { + "origin_pos": 33, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([-0.3155, 0.0512, 0.3313, 0.2001, -0.2423, -0.2325, -0.4816, -0.0248])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rgnet[0][1][0].bias.data" + ] + }, + { + "cell_type": "markdown", + "id": "a3592ddb-97a7-4979-a94f-076a70dc10a5", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### 内置初始化" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "f41e080d-de2f-42ef-9648-0fe114e949e7", + "metadata": { + "origin_pos": 41, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([ 0.0091, 0.0023, -0.0125, 0.0040]), tensor(0.))" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def init_normal(m):\n", + " if type(m) == nn.Linear:\n", + " nn.init.normal_(m.weight, mean=0, std=0.01)\n", + " nn.init.zeros_(m.bias)\n", + "net.apply(init_normal)\n", + "net[0].weight.data[0], net[0].bias.data[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "ebea5b23-9c84-40c2-a3d7-c1f2b4e77de3", + "metadata": { + "origin_pos": 45, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([1., 1., 1., 1.]), tensor(0.))" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def init_constant(m):\n", + " if type(m) == nn.Linear:\n", + " nn.init.constant_(m.weight, 1)\n", + " nn.init.zeros_(m.bias)\n", + "net.apply(init_constant)\n", + "net[0].weight.data[0], net[0].bias.data[0]" + ] + }, + { + "cell_type": "markdown", + "id": "e0cc312b-e55c-4945-9673-065812c7679e", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "对某些块应用不同的初始化方法" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "dae9a89c-623f-4fd5-b61d-95b7d63d4f69", + "metadata": { + "origin_pos": 49, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([ 0.5587, 0.2209, -0.0744, 0.3801])\n", + "tensor([[42., 42., 42., 42., 42., 42., 42., 42.]])\n" + ] + } + ], + "source": [ + "def xavier(m):\n", + " if type(m) == nn.Linear:\n", + " nn.init.xavier_uniform_(m.weight)\n", + "def init_42(m):\n", + " if type(m) == nn.Linear:\n", + " nn.init.constant_(m.weight, 42)\n", + "\n", + "net[0].apply(xavier)\n", + "net[2].apply(init_42)\n", + "print(net[0].weight.data[0])\n", + "print(net[2].weight.data)" + ] + }, + { + "cell_type": "markdown", + "id": "940a6d73-2b27-44ac-a7b5-c24a305e7371", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### 自定义初始化" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "2bacf98d-96a0-4d1f-abbc-dd2b9677ce89", + "metadata": { + "origin_pos": 56, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Init weight torch.Size([8, 4])\n", + "Init weight torch.Size([1, 8])\n" + ] + }, + { + "data": { + "text/plain": [ + "tensor([[-8.4986, 0.0000, -0.0000, -0.0000],\n", + " [ 6.7884, 0.0000, -9.8570, -6.8247]], grad_fn=)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def my_init(m):\n", + " if type(m) == nn.Linear:\n", + " print(\"Init\", *[(name, param.shape)\n", + " for name, param in m.named_parameters()][0])\n", + " nn.init.uniform_(m.weight, -10, 10)\n", + " m.weight.data *= m.weight.data.abs() >= 5 # |w| < 5 时清零\n", + "\n", + "net.apply(my_init)\n", + "net[0].weight[:2]" + ] + }, + { + "cell_type": "markdown", + "id": "b28ccf9d-391d-4f47-8865-e07d4fcfb32c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "直接设置参数" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "a5131518-5f49-4b84-8e52-b0c9f109615b", + "metadata": { + "origin_pos": 60, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([42., 1., 1., 1.])" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "net[0].weight.data[:] += 1\n", + "net[0].weight.data[0, 0] = 42\n", + "net[0].weight.data[0]" + ] + }, + { + "cell_type": "markdown", + "id": "e2c801fd-1770-4440-9870-b48892f23ef1", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### 参数绑定" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "fe7ce342-6ef6-40b5-aef3-2d4fcfc69cdf", + "metadata": { + "origin_pos": 65, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([True, True, True, True, True, True, True, True])\n", + "tensor([True, True, True, True, True, True, True, True])\n" + ] + } + ], + "source": [ + "# 我们需要给共享层一个名称,以便可以引用它的参数\n", + "shared = nn.Linear(8, 8)\n", + "net = nn.Sequential(nn.Linear(4, 8), nn.ReLU(),\n", + " shared, nn.ReLU(),\n", + " shared, nn.ReLU(),\n", + " nn.Linear(8, 1))\n", + "net(X)\n", + "# 检查参数是否相同\n", + "print(net[2].weight.data[0] == net[4].weight.data[0])\n", + "net[2].weight.data[0, 0] = 100\n", + "# 确保它们实际上是同一个对象,而不只是有相同的值\n", + "print(net[2].weight.data[0] == net[4].weight.data[0])" + ] + }, + { + "cell_type": "markdown", + "id": "33d935ce-0cf8-4208-bd13-b5178446bdbc", + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "source": [ + "思考:当参数绑定时,梯度会发生什么情况?" + ] + }, + { + "cell_type": "markdown", + "id": "82fa16e0-4f58-49dd-b9c4-4d06c87b58e6", + "metadata": { + "slideshow": { + "slide_type": "skip" + } + }, + "source": [ + "答案:由于模型参数包含梯度,因此在反向传播期间第二个隐藏层 (即第三个神经网络层)和第三个隐藏层(即第五个神经网络层)的梯度会加在一起。" + ] + }, + { + "cell_type": "markdown", + "id": "91739986-4217-4871-831c-84dc63320b06", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 自定义层\n", + "\n", + "构造一个没有任何参数的自定义层" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f0f9d92-6b58-416e-a7bd-5a4b9a42f056", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "from torch import nn\n", + "\n", + "\n", + "class CenteredLayer(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + "\n", + " def forward(self, X):\n", + " return X - X.mean()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "db87b3aa-5a0c-4643-b3f4-3d83fbaf9814", + "metadata": { + "origin_pos": 6, + "slideshow": { + "slide_type": "slide" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([-2., -1., 0., 1., 2.])" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "layer = CenteredLayer()\n", + "layer(torch.FloatTensor([1, 2, 3, 4, 5]))" + ] + }, + { + "cell_type": "markdown", + "id": "3ca64c2a-707b-4e90-a7ec-f2ea135206f2", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "将层作为组件合并到更复杂的模型中" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "3a23c6a9-fea6-412f-a33c-d7dc4c27f68e", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(2.6776e-09, grad_fn=)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "net = nn.Sequential(nn.Linear(8, 128), CenteredLayer())\n", + "\n", + "# 向该网络发送随机数据后,检查均值是否为0\n", + "Y = net(torch.rand(4, 8))\n", + "Y.mean()" + ] + }, + { + "cell_type": "markdown", + "id": "51ed2384-7439-4321-af2c-a012f3b9fef6", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "带参数的层" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e7657660-f475-410b-8ce2-56dbff3b2344", + "metadata": { + "origin_pos": 23, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Parameter containing:\n", + "tensor([[-0.5454, 1.2766, -0.3547],\n", + " [-0.4969, -0.2906, -0.9240],\n", + " [ 0.2956, -0.8858, 1.3960],\n", + " [-0.3093, 1.2917, 1.4760],\n", + " [ 0.3728, 1.4528, 0.7151]], requires_grad=True)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class MyLinear(nn.Module):\n", + " def __init__(self, in_units, units):\n", + " super().__init__()\n", + " self.weight = nn.Parameter(torch.randn(in_units, units))\n", + " self.bias = nn.Parameter(torch.randn(units,))\n", + " def forward(self, X):\n", + " linear = torch.matmul(X, self.weight.data) + self.bias.data\n", + " return F.relu(linear)\n", + "\n", + "linear = MyLinear(5, 3)\n", + "linear.weight" + ] + }, + { + "cell_type": "markdown", + "id": "ebb30df8-f029-4fb1-a92f-7ccaa331c1ce", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "使用自定义层直接执行前向传播计算" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "9e4b657e-d9a2-43eb-8abd-8c5529561b85", + "metadata": { + "origin_pos": 27, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0000, 1.4069, 0.0000],\n", + " [0.0000, 1.1079, 0.0000]])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "linear(torch.rand(2, 5))" + ] + }, + { + "cell_type": "markdown", + "id": "0346feaf-d2d8-4d7d-8bd7-d82461a184d3", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "使用自定义层构建模型" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "4680cfdf-6cf1-48d0-b835-9126fda45ba7", + "metadata": { + "origin_pos": 31, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[10.2999],\n", + " [ 8.0606]])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "net = nn.Sequential(MyLinear(64, 8), MyLinear(8, 1))\n", + "net(torch.rand(2, 64))" + ] + }, + { + "cell_type": "markdown", + "id": "94facba3-5fd3-4880-9dba-feb7c5d34621", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 读写文件\n", + "\n", + "加载和保存张量" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "1893269f-ee8e-4afd-ae7f-a67d4ce2d264", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([0, 1, 2, 3])" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "from torch import nn\n", + "from torch.nn import functional as F\n", + "\n", + "x = torch.arange(4)\n", + "torch.save(x, 'x-file')\n", + "\n", + "x2 = torch.load('x-file')\n", + "x2" + ] + }, + { + "cell_type": "markdown", + "id": "e4dc5887-89df-4560-97a0-da93e373dc27", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "存储一个张量列表,然后把它们读回内存" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "7cc2777c-fe64-4a36-9fc3-4056514c6485", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([0, 1, 2, 3]), tensor([0., 0., 0., 0.]))" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "y = torch.zeros(4)\n", + "torch.save([x, y],'x-files')\n", + "x2, y2 = torch.load('x-files')\n", + "(x2, y2)" + ] + }, + { + "cell_type": "markdown", + "id": "46d5e258-b229-49b1-a193-a40082724aa1", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "写入或读取从字符串映射到张量的字典" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "e1f88ab2-85f8-45ad-8cff-13a33deca4f4", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'x': tensor([0, 1, 2, 3]), 'y': tensor([0., 0., 0., 0.])}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mydict = {'x': x, 'y': y}\n", + "torch.save(mydict, 'mydict')\n", + "mydict2 = torch.load('mydict')\n", + "mydict2" + ] + }, + { + "cell_type": "markdown", + "id": "22107304-b2eb-4160-b212-31634da00451", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "加载和保存模型参数" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "25c76445-520d-4635-849e-bf28f8ecec56", + "metadata": { + "origin_pos": 18, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "class MLP(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.hidden = nn.Linear(20, 256)\n", + " self.output = nn.Linear(256, 10)\n", + "\n", + " def forward(self, x):\n", + " return self.output(F.relu(self.hidden(x)))\n", + "\n", + "net = MLP()\n", + "X = torch.randn(size=(2, 20))\n", + "Y = net(X)" + ] + }, + { + "cell_type": "markdown", + "id": "ecbd326b-9f20-4497-81d1-cfd51a6b2c1f", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "将模型的参数存储在一个叫做“mlp.params”的文件中" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3cdb1197-4a7b-48ba-9ad7-ce6f3b913354", + "metadata": { + "origin_pos": 22, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "torch.save(net.state_dict(), 'mlp.params')" + ] + }, + { + "cell_type": "markdown", + "id": "0c7b6e86-20c7-4d13-b4e7-8159f38dcbfc", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "实例化了原始多层感知机模型的一个备份。\n", + "直接读取文件中存储的参数" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "3dd363e9-1823-4ef5-ab97-201af72c359d", + "metadata": { + "origin_pos": 26, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "MLP(\n", + " (hidden): Linear(in_features=20, out_features=256, bias=True)\n", + " (output): Linear(in_features=256, out_features=10, bias=True)\n", + ")" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "clone = MLP()\n", + "clone.load_state_dict(torch.load('mlp.params'))\n", + "clone.eval()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f8b8af86-8667-413f-91c2-50dbc05680cf", + "metadata": { + "origin_pos": 30, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[True, True, True, True, True, True, True, True, True, True],\n", + " [True, True, True, True, True, True, True, True, True, True]])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y_clone = clone(X)\n", + "Y_clone == Y" + ] + }, + { + "cell_type": "markdown", + "id": "1fecfb83-9b58-4bcd-97ee-10385bd85371", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## GPU\n", + "\n", + "查看显卡信息" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "346f4a85-2a82-4ad4-a3f6-1bae6807dbe2", + "metadata": { + "origin_pos": 1, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sat Mar 12 20:02:56 2022 \n", + "+-----------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 510.47.03 Driver Version: 511.65 CUDA Version: 11.6 |\n", + "|-------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|===============================+======================+======================|\n", + "| 0 NVIDIA GeForce ... On | 00000000:01:00.0 On | N/A |\n", + "| N/A 38C P5 19W / N/A | 1395MiB / 6144MiB | 14% Default |\n", + "| | | N/A |\n", + "+-------------------------------+----------------------+----------------------+\n", + " \n", + "+-----------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=============================================================================|\n", + "| 0 N/A N/A 112 G /Xwayland N/A |\n", + "| 0 N/A N/A 9974 G /msedge N/A |\n", + "+-----------------------------------------------------------------------------+\n" + ] + } + ], + "source": [ + "!nvidia-smi" + ] + }, + { + "cell_type": "markdown", + "id": "f0c59f2e-bb6a-4992-b3b3-1d6a7c1797b3", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "计算设备" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "9f1e7222-c4ac-4b74-94f9-416a49003441", + "metadata": { + "origin_pos": 8, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(device(type='cpu'), device(type='cuda'), device(type='cuda', index=1))" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import torch\n", + "from torch import nn\n", + "\n", + "torch.device('cpu'), torch.device('cuda'), torch.device('cuda:1')" + ] + }, + { + "cell_type": "markdown", + "id": "8fdb7f48-b679-42d9-9a39-849efea5f05f", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "查询可用gpu的数量" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "121825ab-a77b-4497-8d1a-cba441765348", + "metadata": { + "origin_pos": 12, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.cuda.device_count()" + ] + }, + { + "cell_type": "markdown", + "id": "b55283c2-0276-41f5-8592-3ad08d1bda01", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "这两个函数允许我们在不存在所需所有GPU的情况下运行代码" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "820280b0-fb14-4e5e-93e2-df014ad57fc8", + "metadata": { + "origin_pos": 16, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(device(type='cuda', index=0),\n", + " device(type='cpu'),\n", + " [device(type='cuda', index=0), device(type='cuda', index=1)])" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def try_gpu(i=0): \n", + " \"\"\"如果存在,则返回gpu(i),否则返回cpu()\"\"\"\n", + " if torch.cuda.device_count() >= i + 1:\n", + " return torch.device(f'cuda:{i}')\n", + " return torch.device('cpu')\n", + "\n", + "def try_all_gpus(): \n", + " \"\"\"返回所有可用的GPU,如果没有GPU,则返回[cpu(),]\"\"\"\n", + " devices = [torch.device(f'cuda:{i}')\n", + " for i in range(torch.cuda.device_count())]\n", + " return devices if devices else [torch.device('cpu')]\n", + "\n", + "try_gpu(), try_gpu(10), try_all_gpus()" + ] + }, + { + "cell_type": "markdown", + "id": "420a08ee-e894-46b8-9534-3fb60230ba34", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "查询张量所在的设备" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "0ee876a1-a7f8-4217-97ee-4813c356a004", + "metadata": { + "origin_pos": 20, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "device(type='cpu')" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = torch.tensor([1, 2, 3])\n", + "x.device" + ] + }, + { + "cell_type": "markdown", + "id": "b8523060-40ff-47ff-9bd4-9631a5f8f008", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "存储在GPU上" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "c9af569d-d446-4976-a04c-599b8c3bd600", + "metadata": { + "origin_pos": 24, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[1., 1., 1.],\n", + " [1., 1., 1.]], device='cuda:0')" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.ones(2, 3, device=try_gpu())\n", + "X" + ] + }, + { + "cell_type": "markdown", + "id": "2a3b5790-6ecf-4e9a-bbae-d53a8b61cad6", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "第二个GPU上创建一个随机张量" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "9c96e08b-110f-4bc9-881d-a97faa5da05c", + "metadata": { + "origin_pos": 28, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.5473, 0.1942, 0.2213],\n", + " [0.5998, 0.5565, 0.0372]], device='cuda:1')" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y = torch.rand(2, 3, device=try_gpu(1))\n", + "Y" + ] + }, + { + "cell_type": "markdown", + "id": "b5479fda-7ffb-4817-9565-c015cb52aaf3", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "要计算`X + Y`,我们需要决定在哪里执行这个操作" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "98bb6090-d2c0-431b-8872-91310ab36b32", + "metadata": { + "origin_pos": 32, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[1., 1., 1.],\n", + " [1., 1., 1.]], device='cuda:0')\n", + "tensor([[1., 1., 1.],\n", + " [1., 1., 1.]], device='cuda:1')\n" + ] + } + ], + "source": [ + "Z = X.cuda(1)\n", + "print(X)\n", + "print(Z)" + ] + }, + { + "cell_type": "markdown", + "id": "d999064a-6aa9-441f-8fe9-9ee7c2eb4289", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "现在数据在同一个GPU上(`Z`和`Y`都在),我们可以将它们相加" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "646839ab-f817-488c-8239-1ef6599d22ee", + "metadata": { + "origin_pos": 35, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[1.5473, 1.1942, 1.2213],\n", + " [1.5998, 1.5565, 1.0372]], device='cuda:1')" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y + Z" + ] + }, + { + "cell_type": "markdown", + "id": "ccfca3e4-4e0b-479e-b824-bd2cec2c3dc7", + "metadata": {}, + "source": [ + "变量Z已经在第二个GPU上,如果还是调用Z.cuda(1)将返回Z,而不会复制并分配新内存。" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "096c1468-9770-46e7-a179-5d6158f06c13", + "metadata": { + "origin_pos": 40, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Z.cuda(1) is Z" + ] + }, + { + "cell_type": "markdown", + "id": "e8bbb16a-514c-4e1a-b689-3571073feb87", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "神经网络与GPU" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "eda84331-029f-4e68-aaee-63245054052a", + "metadata": { + "origin_pos": 47, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[1.2194],\n", + " [1.2194]], device='cuda:0', grad_fn=)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "net = nn.Sequential(nn.Linear(3, 1))\n", + "net = net.to(device=try_gpu())\n", + "\n", + "net(X)" + ] + }, + { + "cell_type": "markdown", + "id": "c58b8e18-8d7b-4f37-874f-1441e329c94d", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "确认模型参数存储在同一个GPU上" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "e404204c-5e64-4b38-8900-c7b2f5c2ceb9", + "metadata": { + "origin_pos": 50, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "device(type='cuda', index=0)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "net[0].weight.data.device" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/06_deep_compute/05_model_select.ipynb b/06_deep_compute/05_model_select.ipynb new file mode 100644 index 0000000..d1c44a0 --- /dev/null +++ b/06_deep_compute/05_model_select.ipynb @@ -0,0 +1,9186 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6b83a647-6801-4667-a822-0656b8e9556a", + "metadata": {}, + "source": [ + "# 模型选择\n", + "\n", + "目录\n", + "\n", + "- 欠拟合和过拟合\n", + "- 权重衰减\n", + "- 暂退法(Dropout)" + ] + }, + { + "cell_type": "markdown", + "id": "30394f0b-c857-48e6-b3f5-b7ceba0411c8", + "metadata": { + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 欠拟合和过拟合\n", + "\n", + "通过多项式拟合来探索这些概念" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "70d264d4-0dae-48f1-9288-375f9d84aa10", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import math\n", + "import numpy as np\n", + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l" + ] + }, + { + "cell_type": "markdown", + "id": "68ca9f7a-b179-4fd0-a183-65ce654526d7", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "使用以下三阶多项式来生成训练和测试数据的标签:\n", + "$$y = 5 + 1.2x - 3.4\\frac{x^2}{2!} + 5.6 \\frac{x^3}{3!} + \\epsilon \\text{ where }\n", + "\\epsilon \\sim \\mathcal{N}(0, 0.1^2)$$" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "513f2378-8df6-4eaa-b98a-59c2b890fbae", + "metadata": { + "origin_pos": 5, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "max_degree = 20 # 多项式的最大阶数\n", + "n_train, n_test = 100, 100 # 训练和测试数据集大小\n", + "true_w = np.zeros(max_degree) # 分配大量的空间\n", + "true_w[0:4] = np.array([5, 1.2, -3.4, 5.6])\n", + "\n", + "features = np.random.normal(size=(n_train + n_test, 1))\n", + "np.random.shuffle(features)\n", + "poly_features = np.power(features, np.arange(max_degree).reshape(1, -1))\n", + "for i in range(max_degree):\n", + " poly_features[:, i] /= math.gamma(i + 1) # gamma(n)=(n-1)!\n", + "# labels的维度:(n_train+n_test,)\n", + "labels = np.dot(poly_features, true_w)\n", + "labels += np.random.normal(scale=0.1, size=labels.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "eda1d879-8d72-40bb-9d01-8aaa5ca28f4e", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "查看一下前2个样本" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "a1988415-3bc9-4e2b-bb2c-8f10d26b97e1", + "metadata": { + "origin_pos": 8, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([[0.7135],\n", + " [0.4976]]),\n", + " tensor([[1.0000e+00, 7.1346e-01, 2.5451e-01, 6.0527e-02, 1.0796e-02, 1.5405e-03,\n", + " 1.8318e-04, 1.8670e-05, 1.6650e-06, 1.3199e-07, 9.4170e-09, 6.1079e-10,\n", + " 3.6314e-11, 1.9930e-12, 1.0156e-13, 4.8308e-15, 2.1541e-16, 9.0403e-18,\n", + " 3.5833e-19, 1.3455e-20],\n", + " [1.0000e+00, 4.9764e-01, 1.2382e-01, 2.0540e-02, 2.5554e-03, 2.5434e-04,\n", + " 2.1095e-05, 1.4997e-06, 9.3288e-08, 5.1582e-09, 2.5670e-10, 1.1613e-11,\n", + " 4.8160e-13, 1.8436e-14, 6.5531e-16, 2.1741e-17, 6.7620e-19, 1.9794e-20,\n", + " 5.4725e-22, 1.4334e-23]]),\n", + " tensor([5.3958, 5.5251]))" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# NumPy ndarray转换为tensor\n", + "true_w, features, poly_features, labels = [torch.tensor(x, dtype=\n", + " torch.float32) for x in [true_w, features, poly_features, labels]]\n", + "\n", + "features[:2], poly_features[:2, :], labels[:2]" + ] + }, + { + "cell_type": "markdown", + "id": "f39a6b66-4877-415f-88e7-fee01a15283f", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "实现一个函数来评估模型在给定数据集上的损失" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a3a49a16-7428-46f9-89e9-44ae554a182d", + "metadata": { + "origin_pos": 11, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def evaluate_loss(net, data_iter, loss): \n", + " \"\"\"评估给定数据集上模型的损失\"\"\"\n", + " metric = d2l.Accumulator(2) # 损失的总和,样本数量\n", + " for X, y in data_iter:\n", + " out = net(X)\n", + " y = y.reshape(out.shape)\n", + " l = loss(out, y)\n", + " metric.add(l.sum(), l.numel())\n", + " return metric[0] / metric[1]" + ] + }, + { + "cell_type": "markdown", + "id": "1d3ab43b-025e-47fa-a787-b8e051703282", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "定义训练函数" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "3b3a4d31-223a-43ba-9e05-751078977a7c", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def train(train_features, test_features, train_labels, test_labels,\n", + " num_epochs=400):\n", + " loss = nn.MSELoss(reduction='none')\n", + " input_shape = train_features.shape[-1]\n", + " # 不设置偏置,因为我们已经在多项式中实现了它\n", + " net = nn.Sequential(nn.Linear(input_shape, 1, bias=False))\n", + " batch_size = min(10, train_labels.shape[0])\n", + " train_iter = d2l.load_array((train_features, train_labels.reshape(-1,1)),\n", + " batch_size)\n", + " test_iter = d2l.load_array((test_features, test_labels.reshape(-1,1)),\n", + " batch_size, is_train=False)\n", + " trainer = torch.optim.SGD(net.parameters(), lr=0.01)\n", + " animator = d2l.Animator(xlabel='epoch', ylabel='loss', yscale='log',\n", + " xlim=[1, num_epochs], ylim=[1e-3, 1e2],\n", + " legend=['train', 'test'])\n", + " for epoch in range(num_epochs):\n", + " d2l.train_epoch_ch3(net, train_iter, loss, trainer)\n", + " if epoch == 0 or (epoch + 1) % 20 == 0:\n", + " animator.add(epoch + 1, (evaluate_loss(net, train_iter, loss),\n", + " evaluate_loss(net, test_iter, loss)))\n", + " print('weight:', net[0].weight.data.numpy())" + ] + }, + { + "cell_type": "markdown", + "id": "12e3f5b7-701b-48d7-b962-cd307dd3f93e", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "三阶多项式函数拟合(正常)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "4dbd5c85-6b5e-40d7-a21e-c486758926e1", + "metadata": { + "origin_pos": 17, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "weight: [[ 4.9961076 1.1977049 -3.383854 5.6232586]]\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:16:47.839828\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# 从多项式特征中选择前4个维度,即1,x,x^2/2!,x^3/3!\n", + "train(poly_features[:n_train, :4], poly_features[n_train:, :4],\n", + " labels[:n_train], labels[n_train:])" + ] + }, + { + "cell_type": "markdown", + "id": "56cd7a76-7d08-44bf-a0eb-f5249642c893", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "线性函数拟合(欠拟合)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "255bb596-d3fc-479f-8296-5da42c6a333a", + "metadata": { + "origin_pos": 19, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "weight: [[3.1164684 4.298125 ]]\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:17:04.064205\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# 从多项式特征中选择前2个维度,即1和x\n", + "train(poly_features[:n_train, :2], poly_features[n_train:, :2],\n", + " labels[:n_train], labels[n_train:])" + ] + }, + { + "cell_type": "markdown", + "id": "c001f342-1be2-4b1a-a76a-41198de00ea4", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "高阶多项式函数拟合(过拟合)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "37034ea3-9a8f-41bd-8af4-a36612b2c82a", + "metadata": { + "origin_pos": 21, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "weight: [[ 4.99766 1.2872416 -3.3798084 5.19445 -0.01223703 1.1709547\n", + " 0.18639933 -0.03800117 0.22357842 -0.19907166 -0.04570822 -0.11779588\n", + " 0.01551174 0.08947276 0.10764467 0.2204279 -0.15191121 -0.09354579\n", + " -0.12271424 -0.10525914]]\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:18:09.170658\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# 从多项式特征中选取所有维度\n", + "train(poly_features[:n_train, :], poly_features[n_train:, :],\n", + " labels[:n_train], labels[n_train:], num_epochs=1500)" + ] + }, + { + "cell_type": "markdown", + "id": "8389c4a8-fa32-4615-9bc0-e39c4d158736", + "metadata": { + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 权重衰减\n", + "\n", + "权重衰减是最广泛使用的正则化的技术之一" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "c76286b2-e78b-4a9e-8f94-f714086da60e", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l" + ] + }, + { + "cell_type": "markdown", + "id": "659e2837-48e9-4674-b70f-356a7673c99f", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "像以前一样生成一些数据\n", + "$$y = 0.05 + \\sum_{i = 1}^d 0.01 x_i + \\epsilon \\text{ where }\n", + "\\epsilon \\sim \\mathcal{N}(0, 0.01^2)$$" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ba4a796e-7965-4c8a-b0cb-11522498e8e6", + "metadata": { + "origin_pos": 5, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "n_train, n_test, num_inputs, batch_size = 20, 100, 200, 5\n", + "true_w, true_b = torch.ones((num_inputs, 1)) * 0.01, 0.05\n", + "train_data = d2l.synthetic_data(true_w, true_b, n_train)\n", + "train_iter = d2l.load_array(train_data, batch_size)\n", + "test_data = d2l.synthetic_data(true_w, true_b, n_test)\n", + "test_iter = d2l.load_array(test_data, batch_size, is_train=False)" + ] + }, + { + "cell_type": "markdown", + "id": "832601c4-cad0-4e90-9efa-8cb03e1da391", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "初始化模型参数。注意:$w$的标准差异常高,容易导致优化问题" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "f38cd2b7-2537-4c6c-9157-804cdde17ff9", + "metadata": { + "origin_pos": 8, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def init_params():\n", + " w = torch.normal(0, 1, size=(num_inputs, 1), requires_grad=True)\n", + " b = torch.zeros(1, requires_grad=True)\n", + " return [w, b]" + ] + }, + { + "cell_type": "markdown", + "id": "e5875ed8-6ae1-46ca-82fc-7b0eaf870aeb", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "定义$L_2$范数惩罚" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "9e70a101-c049-481f-888c-76c492432cc1", + "metadata": { + "origin_pos": 12, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def l2_penalty(w):\n", + " return torch.sum(w.pow(2)) / 2" + ] + }, + { + "cell_type": "markdown", + "id": "1dcb1126-4b76-416c-ac79-4813f759f890", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "定义训练代码实现" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "970f04ea-eaab-436c-8044-6cc01390e530", + "metadata": { + "origin_pos": 16, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def train(lambd):\n", + " w, b = init_params()\n", + " net, loss = lambda X: d2l.linreg(X, w, b), d2l.squared_loss\n", + " num_epochs, lr = 100, 0.003\n", + " animator = d2l.Animator(xlabel='epochs', ylabel='loss', yscale='log',\n", + " xlim=[5, num_epochs], legend=['train', 'test'])\n", + " for epoch in range(num_epochs):\n", + " for X, y in train_iter:\n", + " # 广播机制使l2_penalty(w)成为一个长度为batch_size的向量\n", + " l = loss(net(X), y) + lambd * l2_penalty(w) # 增加了L2范数惩罚项\n", + " l.sum().backward()\n", + " d2l.sgd([w, b], lr, batch_size)\n", + " if (epoch + 1) % 5 == 0:\n", + " animator.add(epoch + 1, (d2l.evaluate_loss(net, train_iter, loss),\n", + " d2l.evaluate_loss(net, test_iter, loss)))\n", + " print('w的L2范数是:', torch.norm(w).item())" + ] + }, + { + "cell_type": "markdown", + "id": "14e879f7-bee3-49bf-acc8-0b33510de08f", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "忽略正则化直接训练。非常典型的欠拟合:训练误差还在下降阶段" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "7670e9cc-28ea-4dc5-ad88-53189f3ce1b3", + "metadata": { + "origin_pos": 19, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "w的L2范数是: 13.431254386901855\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:18:19.916991\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "train(lambd=0)" + ] + }, + { + "cell_type": "markdown", + "id": "ced15153-7f8c-4b6e-8114-0ed74d803dba", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "使用权重衰减。测试误差仍在下降阶段。" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "20f01d1a-e18f-4af1-86c0-63d762d64cb1", + "metadata": { + "origin_pos": 21, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "w的L2范数是: 0.3891151249408722\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:18:27.934603\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "train(lambd=3)" + ] + }, + { + "cell_type": "markdown", + "id": "75f01a5f-fe45-44d1-a7b8-1f57ebd4543c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "简洁实现" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "bf65bdb2-46ee-4618-b40b-6408fc54637b", + "metadata": { + "origin_pos": 27, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def train_concise(wd):\n", + " net = nn.Sequential(nn.Linear(num_inputs, 1))\n", + " for param in net.parameters():\n", + " param.data.normal_() # 默认值恰好是(mean=0, std=1)\n", + " loss = nn.MSELoss(reduction='none')\n", + " num_epochs, lr = 100, 0.003\n", + " # 偏置参数没有衰减\n", + " trainer = torch.optim.SGD([\n", + " {\"params\":net[0].weight,'weight_decay': wd},\n", + " {\"params\":net[0].bias}], lr=lr)\n", + " animator = d2l.Animator(xlabel='epochs', ylabel='loss', yscale='log',\n", + " xlim=[5, num_epochs], legend=['train', 'test'])\n", + " for epoch in range(num_epochs):\n", + " for X, y in train_iter:\n", + " trainer.zero_grad()\n", + " l = loss(net(X), y)\n", + " l.mean().backward()\n", + " trainer.step()\n", + " if (epoch + 1) % 5 == 0:\n", + " animator.add(epoch + 1,\n", + " (d2l.evaluate_loss(net, train_iter, loss),\n", + " d2l.evaluate_loss(net, test_iter, loss)))\n", + " print('w的L2范数:', net[0].weight.norm().item())" + ] + }, + { + "cell_type": "markdown", + "id": "081cf988-6c1b-46fe-8af7-986b387581dc", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "这些图看起来与之前类似。但训练误差更快趋向水平:库函数有内部优化" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "809320c9-7ef9-414c-9df2-a10b36f6e8ce", + "metadata": { + "origin_pos": 30, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "w的L2范数: 14.588664054870605\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:18:42.182739\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "train_concise(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "d256f3af-1647-4398-84d8-357e8c31629f", + "metadata": { + "origin_pos": 31, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "w的L2范数: 0.373628705739975\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:18:54.170364\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "train_concise(3)" + ] + }, + { + "cell_type": "markdown", + "id": "5dea2d34-f4a7-4ae1-87d3-09b1fdd7dadb", + "metadata": { + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 暂退法(Dropout)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "4ba0141a-e3b9-43d8-9b38-78b2d788d707", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "\n", + "def dropout_layer(X, dropout):\n", + " \"\"\"以`dropout`的概率丢弃张量输入`X`中的元素\"\"\"\n", + " assert 0 <= dropout <= 1\n", + " if dropout == 1: # 在本情况中,所有元素都被丢弃\n", + " return torch.zeros_like(X)\n", + " if dropout == 0: # 在本情况中,所有元素都被保留\n", + " return X\n", + " # 为什么要掩码?并行计算两个分支\n", + " mask = (torch.rand(X.shape) > dropout).float()\n", + " return mask * X / (1.0 - dropout)" + ] + }, + { + "cell_type": "markdown", + "id": "88bd8f63-4716-4314-bcaf-9fe4062b3fb1", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "测试`dropout_layer`函数" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "1186ef28-60fb-4bf3-8c25-3db7e19d4aee", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[ 0., 1., 2., 3., 4., 5., 6., 7.],\n", + " [ 8., 9., 10., 11., 12., 13., 14., 15.]])\n", + "tensor([[ 0., 1., 2., 3., 4., 5., 6., 7.],\n", + " [ 8., 9., 10., 11., 12., 13., 14., 15.]])\n", + "tensor([[ 0., 2., 0., 6., 8., 10., 0., 14.],\n", + " [ 0., 0., 0., 0., 24., 0., 0., 0.]])\n", + "tensor([[0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0.]])\n" + ] + } + ], + "source": [ + "X= torch.arange(16, dtype = torch.float32).reshape((2, 8))\n", + "print(X)\n", + "print(dropout_layer(X, 0.))\n", + "print(dropout_layer(X, 0.5))\n", + "print(dropout_layer(X, 1.))" + ] + }, + { + "cell_type": "markdown", + "id": "4f18f009-335f-429a-b794-9e28e653fa39", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "定义具有两个隐藏层的多层感知机,每个隐藏层包含256个单元" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "26c4ba3a-48e1-4eac-86da-c4846e0ac04e", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "num_inputs, num_outputs, num_hiddens1, num_hiddens2 = 784, 10, 256, 256\n", + "\n", + "dropout1, dropout2 = 0.2, 0.5\n", + "\n", + "class Net(nn.Module):\n", + " def __init__(self, num_inputs, num_outputs, num_hiddens1, num_hiddens2,\n", + " is_training = True):\n", + " super(Net, self).__init__()\n", + " self.num_inputs = num_inputs\n", + " self.training = is_training\n", + " self.lin1 = nn.Linear(num_inputs, num_hiddens1)\n", + " self.lin2 = nn.Linear(num_hiddens1, num_hiddens2)\n", + " self.lin3 = nn.Linear(num_hiddens2, num_outputs)\n", + " self.relu = nn.ReLU()\n", + "\n", + " def forward(self, X):\n", + " H1 = self.relu(self.lin1(X.reshape((-1, self.num_inputs))))\n", + " if self.training == True: # 只有在训练模型时才使用dropout\n", + " H1 = dropout_layer(H1, dropout1) # 在第一个全连接层之后添加一个dropout层\n", + " H2 = self.relu(self.lin2(H1))\n", + " if self.training == True:\n", + " H2 = dropout_layer(H2, dropout2) # 在第一个全连接层之后添加一个dropout层\n", + " out = self.lin3(H2)\n", + " return out\n", + "\n", + "\n", + "net = Net(num_inputs, num_outputs, num_hiddens1, num_hiddens2)" + ] + }, + { + "cell_type": "markdown", + "id": "48ea088d-77a0-4373-af8c-2545e7a6cccf", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练和测试" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "c08ed34e-a50e-44dc-832c-6bc1341cec58", + "metadata": { + "origin_pos": 18, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:22:37.456317\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "num_epochs, lr, batch_size = 10, 0.5, 256\n", + "loss = nn.CrossEntropyLoss(reduction='none')\n", + "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)\n", + "trainer = torch.optim.SGD(net.parameters(), lr=lr)\n", + "d2l.train_ch3(net, train_iter, test_iter, loss, num_epochs, trainer)" + ] + }, + { + "cell_type": "markdown", + "id": "539c3a27-31ef-4dd6-9655-3d161aaafb79", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "简洁实现" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "3d46073f-8828-4c26-b1e5-98c0fcddae43", + "metadata": { + "origin_pos": 22, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "net = nn.Sequential(nn.Flatten(),\n", + " nn.Linear(784, 256),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout1), # 在第一个全连接层之后添加一个dropout层\n", + " nn.Linear(256, 256),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout2), # 在第二个全连接层之后添加一个dropout层\n", + " nn.Linear(256, 10))\n", + "\n", + "def init_weights(m):\n", + " if type(m) == nn.Linear:\n", + " nn.init.normal_(m.weight, std=0.01)\n", + "\n", + "net.apply(init_weights);" + ] + }, + { + "cell_type": "markdown", + "id": "49b9f82e-14c3-433e-8ee8-3637ffafb2b1", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "对模型进行训练和测试" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "66683b72-aa3c-4cb8-97ef-ced571cca688", + "metadata": { + "origin_pos": 26, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T09:26:30.099732\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "trainer = torch.optim.SGD(net.parameters(), lr=lr)\n", + "d2l.train_ch3(net, train_iter, test_iter, loss, num_epochs, trainer)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/07_convolution/07_convolution.ipynb b/07_convolution/07_convolution.ipynb new file mode 100644 index 0000000..f443235 --- /dev/null +++ b/07_convolution/07_convolution.ipynb @@ -0,0 +1,2387 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f2160c1d-37bb-482b-a270-daa13c612f7c", + "metadata": {}, + "source": [ + "# 卷积神经网络\n", + "\n", + "目录\n", + "\n", + "- 图像卷积\n", + "- 填充和步幅\n", + "- 多输入多输出通道\n", + "- 池化层\n", + "- 卷积神经网络(LeNet)" + ] + }, + { + "cell_type": "markdown", + "id": "6846ad05-7209-4c31-9378-b9b532a3667e", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 图像卷积\n", + "\n", + "互相关运算" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ac9478e0-6433-4cf7-917f-600f08edc110", + "metadata": { + "origin_pos": 3, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "def corr2d(X, K): \n", + " \"\"\"计算二维互相关运算\"\"\"\n", + " h, w = K.shape\n", + " Y = torch.zeros((X.shape[0] - h + 1, X.shape[1] - w + 1))\n", + " for i in range(Y.shape[0]):\n", + " for j in range(Y.shape[1]):\n", + " Y[i, j] = (X[i:i + h, j:j + w] * K).sum()\n", + " return Y" + ] + }, + { + "cell_type": "markdown", + "id": "314a155d-9475-4a87-950d-c14bbcad6d44", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "验证上述二维互相关运算的输出" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "3f8076da-f6b1-4b35-ba49-c31cb1314304", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[19., 25.],\n", + " [37., 43.]])" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.tensor([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]])\n", + "K = torch.tensor([[0.0, 1.0], [2.0, 3.0]])\n", + "corr2d(X, K)" + ] + }, + { + "cell_type": "markdown", + "id": "aab13ec8-679c-41b7-a686-98527d50a6f7", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "实现二维卷积层" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "8c2795ba-6b7e-4d65-a718-7834d515eebf", + "metadata": { + "origin_pos": 9, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "class Conv2D(nn.Module):\n", + " def __init__(self, kernel_size):\n", + " super().__init__()\n", + " self.weight = nn.Parameter(torch.rand(kernel_size))\n", + " self.bias = nn.Parameter(torch.zeros(1))\n", + "\n", + " def forward(self, x):\n", + " return corr2d(x, self.weight) + self.bias" + ] + }, + { + "cell_type": "markdown", + "id": "9a4b28c4-b74d-47f0-a549-2b2d7d350e55", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "卷积层的一个简单应用:\n", + "检测图像中不同颜色的边缘" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "2763c05e-3af6-48ab-8660-e7ffff81bbf5", + "metadata": { + "origin_pos": 12, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[1., 1., 0., 0., 0., 0., 1., 1.],\n", + " [1., 1., 0., 0., 0., 0., 1., 1.],\n", + " [1., 1., 0., 0., 0., 0., 1., 1.],\n", + " [1., 1., 0., 0., 0., 0., 1., 1.],\n", + " [1., 1., 0., 0., 0., 0., 1., 1.],\n", + " [1., 1., 0., 0., 0., 0., 1., 1.]])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.ones((6, 8))\n", + "X[:, 2:6] = 0\n", + "X" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "c3d67537-eb49-4136-b5bd-51cc13eaafb2", + "metadata": { + "origin_pos": 15, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "K = torch.tensor([[1.0, -1.0]])" + ] + }, + { + "cell_type": "markdown", + "id": "965ca956-b67c-451f-9dd4-8ab725165f1a", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "输出`Y`中的1代表从白色到黑色的边缘,-1代表从黑色到白色的边缘" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5cf508d4-c35a-428d-9bb1-96fc32ae5864", + "metadata": { + "origin_pos": 17, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 0., 1., 0., 0., 0., -1., 0.],\n", + " [ 0., 1., 0., 0., 0., -1., 0.],\n", + " [ 0., 1., 0., 0., 0., -1., 0.],\n", + " [ 0., 1., 0., 0., 0., -1., 0.],\n", + " [ 0., 1., 0., 0., 0., -1., 0.],\n", + " [ 0., 1., 0., 0., 0., -1., 0.]])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Y = corr2d(X, K)\n", + "Y" + ] + }, + { + "cell_type": "markdown", + "id": "7bc03775-24f6-4d2a-b2aa-e870717de626", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "卷积核`K`只可以检测垂直边缘" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "39d014be-40db-4d15-8caf-591a2710845c", + "metadata": { + "origin_pos": 19, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0.]])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "corr2d(X.t(), K)" + ] + }, + { + "cell_type": "markdown", + "id": "87ad3bb5-f1b5-4788-9db8-2c1c27cffbfa", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "学习由`X`生成`Y`的卷积核" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd75f3b8-6a8e-4956-8f9c-f39ac8ceb791", + "metadata": { + "origin_pos": 22, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "# 构造一个二维卷积层,它具有1个输出通道和形状为(1,2)的卷积核\n", + "conv2d = nn.Conv2d(1,1, kernel_size=(1, 2), bias=False)\n", + "\n", + "# 这个二维卷积层使用四维输入和输出格式(批量大小、通道、高度、宽度),\n", + "# 其中批量大小和通道数都为1\n", + "X = X.reshape((1, 1, 6, 8))\n", + "Y = Y.reshape((1, 1, 6, 7))\n", + "lr = 3e-2" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "0d28ad4a-bf04-4352-b0c4-c5d5ec11646a", + "metadata": { + "origin_pos": 22, + "slideshow": { + "slide_type": "slide" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch 2, loss 8.836\n", + "epoch 4, loss 1.692\n", + "epoch 6, loss 0.370\n", + "epoch 8, loss 0.097\n", + "epoch 10, loss 0.031\n" + ] + } + ], + "source": [ + "for i in range(10):\n", + " Y_hat = conv2d(X)\n", + " l = (Y_hat - Y) ** 2\n", + " conv2d.zero_grad()\n", + " l.sum().backward()\n", + " # 迭代更新卷积核\n", + " conv2d.weight.data[:] -= lr * conv2d.weight.grad\n", + " if (i + 1) % 2 == 0:\n", + " print(f'epoch {i+1}, loss {l.sum():.3f}')" + ] + }, + { + "cell_type": "markdown", + "id": "eaeb6855-12e3-40b0-acc8-71cadbf9f55e", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "所学的卷积核的权重张量" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "b473f6f7-6fca-4aa6-8c77-87c57b7f9b5b", + "metadata": { + "origin_pos": 26, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 1.0015, -0.9692]])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "conv2d.weight.data.reshape((1, 2))" + ] + }, + { + "cell_type": "markdown", + "id": "5879af59-0af7-4128-a750-84d5ce60f25d", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 填充和步幅\n", + "\n", + "在所有侧边填充1个像素" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "979a995a-96f9-4d41-8442-2d32af9074bf", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "\n", + "\n", + "def comp_conv2d(conv2d, X):\n", + " \"\"\"初始化卷积层权重,并对输入和输出提高和缩减相应的维数\"\"\"\n", + " X = X.reshape((1, 1) + X.shape) # 批量大小和通道数都是1\n", + " Y = conv2d(X)\n", + " return Y.reshape(Y.shape[2:]) # 省略前两个维度:批量大小和通道" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "f9f0ed64-7d08-4788-bf0b-932b9ec4803b", + "metadata": { + "origin_pos": 2, + "slideshow": { + "slide_type": "slide" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([8, 8])" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 注意,每边都填充了1行或1列,因此总共添加了2行或2列\n", + "conv2d = nn.Conv2d(1, 1, kernel_size=3, padding=1)\n", + "X = torch.rand(size=(8, 8))\n", + "comp_conv2d(conv2d, X).shape" + ] + }, + { + "cell_type": "markdown", + "id": "cf74250e-eb1b-47df-9141-f931fafc3af0", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "填充不同的高度和宽度" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "545bfb03-c86e-4f29-9f2f-940c7ec49aaf", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([8, 8])" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "conv2d = nn.Conv2d(1, 1, kernel_size=(5, 3), padding=(2, 1))\n", + "comp_conv2d(conv2d, X).shape" + ] + }, + { + "cell_type": "markdown", + "id": "6a8ce394-d251-438d-a6ad-0ae688a17a91", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "将高度和宽度的步幅设置为2" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "15d25626-14d6-49c2-a7d2-f5b32d4ae621", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([4, 4])" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "conv2d = nn.Conv2d(1, 1, kernel_size=3, padding=1, stride=2)\n", + "comp_conv2d(conv2d, X).shape" + ] + }, + { + "cell_type": "markdown", + "id": "64f13f70-7ba6-4c68-adc4-951c77fb7b6c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "一个稍微复杂的例子" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7aae79d5-a75d-4769-bac5-52c0dd6c2cd4", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([2, 2])" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "conv2d = nn.Conv2d(1, 1, kernel_size=(3, 5), padding=(0, 1), stride=(3, 4))\n", + "comp_conv2d(conv2d, X).shape" + ] + }, + { + "cell_type": "markdown", + "id": "b0ade0d5-8c2e-42d3-874d-bb9880c89dab", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 多输入多输出通道\n", + "\n", + "实现一下多输入通道互相关运算" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "9be9ab73-d95c-4a89-acfe-ac912e33e831", + "metadata": { + "origin_pos": 3, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from d2l import torch as d2l\n", + "\n", + "def corr2d_multi_in(X, K):\n", + " # 先遍历“X”和“K”的第0个维度(通道维度,忽略批量),再把它们加在一起\n", + " return sum(d2l.corr2d(x, k) for x, k in zip(X, K))" + ] + }, + { + "cell_type": "markdown", + "id": "5cb1a6f4-9155-4c6e-b452-6f0575ac6bc3", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "验证互相关运算的输出" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "ffc69b60-744c-4a12-8f3c-ffb93993e6b8", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 56., 72.],\n", + " [104., 120.]])" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.tensor([[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]],\n", + " [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]])\n", + "K = torch.tensor([[[0.0, 1.0], [2.0, 3.0]], [[1.0, 2.0], [3.0, 4.0]]])\n", + "\n", + "corr2d_multi_in(X, K)" + ] + }, + { + "cell_type": "markdown", + "id": "2b26766f-0d38-4dec-b444-49564c7b1706", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "计算多个通道的输出的互相关函数" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "f973b848-4341-43b5-bdeb-dde75fc953e9", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([3, 2, 2, 2])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def corr2d_multi_in_out(X, K):\n", + " # 迭代“K”的第0个维度,每次都对输入“X”执行互相关运算。\n", + " # 最后将所有结果都叠加在一起\n", + " return torch.stack([corr2d_multi_in(X, k) for k in K], 0)\n", + "\n", + "K = torch.stack((K, K + 1, K + 2), 0)\n", + "K.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "8a5ef08c-2688-4551-9fe4-4fe83476b568", + "metadata": { + "origin_pos": 12, + "slideshow": { + "slide_type": "slide" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[ 56., 72.],\n", + " [104., 120.]],\n", + "\n", + " [[ 76., 100.],\n", + " [148., 172.]],\n", + "\n", + " [[ 96., 128.],\n", + " [192., 224.]]])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "corr2d_multi_in_out(X, K)" + ] + }, + { + "cell_type": "markdown", + "id": "8b5646a7-4eaf-4c73-ae23-a6b0ae4b3667", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "1x1卷积" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd63dcb7-8f02-46e9-8696-8cd24313460d", + "metadata": { + "origin_pos": 18, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def corr2d_multi_in_out_1x1(X, K):\n", + " c_i, h, w = X.shape\n", + " c_o = K.shape[0]\n", + " X = X.reshape((c_i, h * w))\n", + " K = K.reshape((c_o, c_i))\n", + " # 全连接层中的矩阵乘法\n", + " Y = torch.matmul(K, X)\n", + " return Y.reshape((c_o, h, w))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "18e97ece-4c09-4273-9e7b-a0934046c97c", + "metadata": { + "origin_pos": 18, + "slideshow": { + "slide_type": "slide" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "X = torch.normal(0, 1, (3, 3, 3))\n", + "K = torch.normal(0, 1, (2, 3, 1, 1))\n", + "\n", + "Y1 = corr2d_multi_in_out_1x1(X, K)\n", + "Y2 = corr2d_multi_in_out(X, K)\n", + "assert float(torch.abs(Y1 - Y2).sum()) < 1e-6" + ] + }, + { + "cell_type": "markdown", + "id": "64c48e4b-8c94-4de9-bd8d-e03512695e7e", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 池化层\n", + "\n", + "池化也有译成汇聚。实现池化层的前向传播" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "7ce8507e-5ba3-427a-9066-d1031580472d", + "metadata": { + "origin_pos": 3, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "def pool2d(X, pool_size, mode='max'):\n", + " p_h, p_w = pool_size\n", + " Y = torch.zeros((X.shape[0] - p_h + 1, X.shape[1] - p_w + 1))\n", + " for i in range(Y.shape[0]):\n", + " for j in range(Y.shape[1]):\n", + " if mode == 'max':\n", + " Y[i, j] = X[i: i + p_h, j: j + p_w].max()\n", + " elif mode == 'avg':\n", + " Y[i, j] = X[i: i + p_h, j: j + p_w].mean()\n", + " return Y" + ] + }, + { + "cell_type": "markdown", + "id": "fd4f9890-9159-4d0f-92bf-629c91f522ac", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "验证二维最大汇聚层的输出" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4d6aca2f-7a2b-4e4e-a33f-51cfeabe3149", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[4., 5.],\n", + " [7., 8.]])" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.tensor([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]])\n", + "pool2d(X, (2, 2))" + ] + }, + { + "cell_type": "markdown", + "id": "da22e032-1759-4359-836e-36ecfd643583", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "验证平均汇聚层" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "2939723a-f3ea-4f70-b291-cf5dbfd8fe30", + "metadata": { + "origin_pos": 8, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[2., 3.],\n", + " [5., 6.]])" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pool2d(X, (2, 2), 'avg')" + ] + }, + { + "cell_type": "markdown", + "id": "c0cb77fd-3e7b-4d1d-a616-63c6100d2feb", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "填充和步幅" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "a574a0ed-4184-4ad5-8f16-b802bea2bb99", + "metadata": { + "origin_pos": 11, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[[ 0., 1., 2., 3.],\n", + " [ 4., 5., 6., 7.],\n", + " [ 8., 9., 10., 11.],\n", + " [12., 13., 14., 15.]]]])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.arange(16, dtype=torch.float32).reshape((1, 1, 4, 4))\n", + "X" + ] + }, + { + "cell_type": "markdown", + "id": "0209afc0-8eb4-4c58-a5c6-efdc1988e0fd", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "深度学习框架中的步幅与汇聚窗口的大小相同" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b66fd222-9233-4dc9-bbce-87925a86498b", + "metadata": { + "origin_pos": 15, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[[10.]]]])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pool2d = nn.MaxPool2d(3)\n", + "pool2d(X)" + ] + }, + { + "cell_type": "markdown", + "id": "0c169cd1-b0d6-4760-88e2-0ad530c51cfc", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "填充和步幅可以手动设定" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "4fdbb845-7f4b-4649-96c0-3c0300b6021a", + "metadata": { + "origin_pos": 19, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[[ 5., 7.],\n", + " [13., 15.]]]])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pool2d = nn.MaxPool2d(3, padding=1, stride=2)\n", + "pool2d(X)" + ] + }, + { + "cell_type": "markdown", + "id": "7b83b34f-526b-48b1-b5ff-963c84b57471", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "设定一个任意大小的矩形汇聚窗口,并分别设定填充和步幅的高度和宽度" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "fc4d58c9-df77-4765-8344-948d66e9d55f", + "metadata": { + "origin_pos": 25, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[[ 5., 7.],\n", + " [13., 15.]]]])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pool2d = nn.MaxPool2d((2, 3), stride=(2, 3), padding=(0, 1))\n", + "pool2d(X)" + ] + }, + { + "cell_type": "markdown", + "id": "6a771a4c-bda2-48e6-bd31-d8d4eec503ad", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "汇聚层在每个输入通道上单独运算" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8fc17adc-68a2-4395-8ca4-db4c853eae72", + "metadata": { + "origin_pos": 29, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[[ 0., 1., 2., 3.],\n", + " [ 4., 5., 6., 7.],\n", + " [ 8., 9., 10., 11.],\n", + " [12., 13., 14., 15.]],\n", + "\n", + " [[ 1., 2., 3., 4.],\n", + " [ 5., 6., 7., 8.],\n", + " [ 9., 10., 11., 12.],\n", + " [13., 14., 15., 16.]]]])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.cat((X, X + 1), 1)\n", + "X" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "4622f1b6-0352-446f-8a67-c336f42249d2", + "metadata": { + "origin_pos": 33, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[[[ 5., 7.],\n", + " [13., 15.]],\n", + "\n", + " [[ 6., 8.],\n", + " [14., 16.]]]])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pool2d = nn.MaxPool2d(3, padding=1, stride=2)\n", + "pool2d(X)" + ] + }, + { + "cell_type": "markdown", + "id": "a71866f9-e712-467a-9e0d-3832db22e4ad", + "metadata": { + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 卷积神经网络(LeNet)\n", + "\n", + "LeNet(LeNet-5)由两个部分组成:\n", + "卷积编码器和全连接层密集块" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "10bf89f8-9718-4d30-bceb-f64ccd57d7e8", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "net = nn.Sequential(\n", + " nn.Conv2d(1, 6, kernel_size=5, padding=2), nn.Sigmoid(),\n", + " nn.AvgPool2d(kernel_size=2, stride=2),\n", + " nn.Conv2d(6, 16, kernel_size=5), nn.Sigmoid(),\n", + " nn.AvgPool2d(kernel_size=2, stride=2),\n", + " nn.Flatten(),\n", + " nn.Linear(16 * 5 * 5, 120), nn.Sigmoid(),\n", + " nn.Linear(120, 84), nn.Sigmoid(),\n", + " nn.Linear(84, 10))" + ] + }, + { + "cell_type": "markdown", + "id": "b108595b-2b06-4670-b55e-4096d2c97e7d", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "检查模型" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "fca54cef-3f57-4901-b1e7-4039c9050a53", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conv2d output shape: \t torch.Size([1, 6, 28, 28])\n", + "Sigmoid output shape: \t torch.Size([1, 6, 28, 28])\n", + "AvgPool2d output shape: \t torch.Size([1, 6, 14, 14])\n", + "Conv2d output shape: \t torch.Size([1, 16, 10, 10])\n", + "Sigmoid output shape: \t torch.Size([1, 16, 10, 10])\n", + "AvgPool2d output shape: \t torch.Size([1, 16, 5, 5])\n", + "Flatten output shape: \t torch.Size([1, 400])\n", + "Linear output shape: \t torch.Size([1, 120])\n", + "Sigmoid output shape: \t torch.Size([1, 120])\n", + "Linear output shape: \t torch.Size([1, 84])\n", + "Sigmoid output shape: \t torch.Size([1, 84])\n", + "Linear output shape: \t torch.Size([1, 10])\n" + ] + } + ], + "source": [ + "X = torch.rand(size=(1, 1, 28, 28), dtype=torch.float32)\n", + "for layer in net:\n", + " X = layer(X)\n", + " print(layer.__class__.__name__,'output shape: \\t',X.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "97b8c6d4-e9ff-4a47-94f9-9f12348683c3", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "LeNet在Fashion-MNIST数据集上的表现" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f0f07725-b2d3-4461-92f3-e8465153bd96", + "metadata": { + "origin_pos": 9, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "batch_size = 256\n", + "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size=batch_size)" + ] + }, + { + "cell_type": "markdown", + "id": "4b457e3e-b2b0-4bab-9821-7ed7ce2e3f7f", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "对\n", + "`evaluate_accuracy`函数进行轻微的修改" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "43a68342-901c-4d8e-8066-c3f6b0f32a2a", + "metadata": { + "origin_pos": 13, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def evaluate_accuracy_gpu(net, data_iter, device=None): \n", + " \"\"\"使用GPU计算模型在数据集上的精度\"\"\"\n", + " if isinstance(net, nn.Module):\n", + " net.eval() # 设置为评估模式\n", + " if not device:\n", + " device = next(iter(net.parameters())).device\n", + " metric = d2l.Accumulator(2) # 正确预测的数量,总预测的数量\n", + " with torch.no_grad():\n", + " for X, y in data_iter:\n", + " if isinstance(X, list): # BERT微调所需(之后介绍)\n", + " X = [x.to(device) for x in X]\n", + " else:\n", + " X = X.to(device)\n", + " y = y.to(device)\n", + " metric.add(d2l.accuracy(net(X), y), y.numel())\n", + " return metric[0] / metric[1]" + ] + }, + { + "cell_type": "markdown", + "id": "0fb6644e-bfe3-47da-acdc-e223e632801d", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "为了使用GPU,我们还需要一点小改动" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c7b9b01a-3c00-4e6d-b991-06c081e442bf", + "metadata": { + "origin_pos": 16, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def train_ch6(net, train_iter, test_iter, num_epochs, lr, device):\n", + " \"\"\"用GPU训练模型(在第六章定义)\"\"\"\n", + " def init_weights(m):\n", + " if type(m) == nn.Linear or type(m) == nn.Conv2d:\n", + " nn.init.xavier_uniform_(m.weight)\n", + " net.apply(init_weights)\n", + " print('training on', device)\n", + " net.to(device)\n", + " optimizer = torch.optim.SGD(net.parameters(), lr=lr)\n", + " loss = nn.CrossEntropyLoss()\n", + " animator = d2l.Animator(xlabel='epoch', xlim=[1, num_epochs],\n", + " legend=['train loss', 'train acc', 'test acc'])\n", + " timer, num_batches = d2l.Timer(), len(train_iter)\n", + " for epoch in range(num_epochs):\n", + " metric = d2l.Accumulator(3) # 训练损失之和,训练准确率之和,样本数\n", + " net.train()\n", + " for i, (X, y) in enumerate(train_iter):\n", + " timer.start()\n", + " optimizer.zero_grad()\n", + " X, y = X.to(device), y.to(device)\n", + " y_hat = net(X)\n", + " l = loss(y_hat, y)\n", + " l.backward()\n", + " optimizer.step()\n", + " with torch.no_grad():\n", + " metric.add(l * X.shape[0], d2l.accuracy(y_hat, y), X.shape[0])\n", + " timer.stop()\n", + " train_l = metric[0] / metric[2]\n", + " train_acc = metric[1] / metric[2]\n", + " if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:\n", + " animator.add(epoch + (i + 1) / num_batches,\n", + " (train_l, train_acc, None))\n", + " test_acc = evaluate_accuracy_gpu(net, test_iter)\n", + " animator.add(epoch + 1, (None, None, test_acc))\n", + " print(f'loss {train_l:.3f}, train acc {train_acc:.3f}, '\n", + " f'test acc {test_acc:.3f}')\n", + " print(f'{metric[2] * num_epochs / timer.sum():.1f} examples/sec '\n", + " f'on {str(device)}')" + ] + }, + { + "cell_type": "markdown", + "id": "a6c9ffd8-fc3a-4480-bb2a-8ec164f7fe37", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练和评估LeNet-5模型" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "103901ca-60e4-4694-96fd-02f56736c134", + "metadata": { + "origin_pos": 19, + "tab": [ + "pytorch" + ], + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loss 0.472, train acc 0.823, test acc 0.786\n", + "3560.3 examples/sec on cpu\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2023-02-28T15:39:17.744575\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.5.1, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "lr, num_epochs = 0.9, 10\n", + "train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/07_convolution/07_modern_cnn.ipynb b/07_convolution/07_modern_cnn.ipynb new file mode 100644 index 0000000..bcb194a --- /dev/null +++ b/07_convolution/07_modern_cnn.ipynb @@ -0,0 +1,7852 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "255f7fbf-617f-40da-a338-d3f8fc5a0e92", + "metadata": {}, + "source": [ + "# 现代卷积神经网络\n", + "\n", + "目录\n", + "\n", + "- 深度卷积神经网络(AlexNet)\n", + "- 使用块的网络(VGG)\n", + "- 网络中的网络(NiN)\n", + "- 含并行连结的网络(GoogLeNet)\n", + "- 批量规范化\n", + "- 残差网络(ResNet)" + ] + }, + { + "cell_type": "markdown", + "id": "3955813c-8792-42ae-af95-d88448189353", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 深度卷积神经网络(AlexNet)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "250437f3-532a-4eb2-9ed5-fed19a106e15", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "net = nn.Sequential(\n", + " # 这里,我们使用一个11*11的更大窗口来捕捉对象。\n", + " # 同时,步幅为4,以减少输出的高度和宽度。\n", + " # 另外,输出通道的数目远大于LeNet\n", + " nn.Conv2d(1, 96, kernel_size=11, stride=4, padding=1), nn.ReLU(),\n", + " nn.MaxPool2d(kernel_size=3, stride=2),\n", + " # 减小卷积窗口,使用填充为2来使得输入与输出的高和宽一致,且增大输出通道数\n", + " nn.Conv2d(96, 256, kernel_size=5, padding=2), nn.ReLU(),\n", + " nn.MaxPool2d(kernel_size=3, stride=2),\n", + " # 使用三个连续的卷积层和较小的卷积窗口。\n", + " # 除了最后的卷积层,输出通道的数量进一步增加。\n", + " # 在前两个卷积层之后,汇聚层不用于减少输入的高度和宽度\n", + " nn.Conv2d(256, 384, kernel_size=3, padding=1), nn.ReLU(),\n", + " nn.Conv2d(384, 384, kernel_size=3, padding=1), nn.ReLU(),\n", + " nn.Conv2d(384, 256, kernel_size=3, padding=1), nn.ReLU(),\n", + " nn.MaxPool2d(kernel_size=3, stride=2),\n", + " nn.Flatten(),\n", + " # 这里,全连接层的输出数量是LeNet中的好几倍。使用dropout层来减轻过拟合\n", + " nn.Linear(6400, 4096), nn.ReLU(),\n", + " nn.Dropout(p=0.5),\n", + " nn.Linear(4096, 4096), nn.ReLU(),\n", + " nn.Dropout(p=0.5),\n", + " # 最后是输出层。由于这里使用Fashion-MNIST,所以用类别数为10,而非论文中的1000\n", + " nn.Linear(4096, 10))" + ] + }, + { + "cell_type": "markdown", + "id": "9a6f3671-6eb9-44c3-bfe4-fd01bcfa2805", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "我们构造一个\n", + "单通道数据,来观察每一层输出的形状" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "96608413-913b-4dfe-8f23-9ffce9ea0582", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conv2d output shape:\t torch.Size([1, 96, 54, 54])\n", + "ReLU output shape:\t torch.Size([1, 96, 54, 54])\n", + "MaxPool2d output shape:\t torch.Size([1, 96, 26, 26])\n", + "Conv2d output shape:\t torch.Size([1, 256, 26, 26])\n", + "ReLU output shape:\t torch.Size([1, 256, 26, 26])\n", + "MaxPool2d output shape:\t torch.Size([1, 256, 12, 12])\n", + "Conv2d output shape:\t torch.Size([1, 384, 12, 12])\n", + "ReLU output shape:\t torch.Size([1, 384, 12, 12])\n", + "Conv2d output shape:\t torch.Size([1, 384, 12, 12])\n", + "ReLU output shape:\t torch.Size([1, 384, 12, 12])\n", + "Conv2d output shape:\t torch.Size([1, 256, 12, 12])\n", + "ReLU output shape:\t torch.Size([1, 256, 12, 12])\n", + "MaxPool2d output shape:\t torch.Size([1, 256, 5, 5])\n", + "Flatten output shape:\t torch.Size([1, 6400])\n", + "Linear output shape:\t torch.Size([1, 4096])\n", + "ReLU output shape:\t torch.Size([1, 4096])\n", + "Dropout output shape:\t torch.Size([1, 4096])\n", + "Linear output shape:\t torch.Size([1, 4096])\n", + "ReLU output shape:\t torch.Size([1, 4096])\n", + "Dropout output shape:\t torch.Size([1, 4096])\n", + "Linear output shape:\t torch.Size([1, 10])\n" + ] + } + ], + "source": [ + "X = torch.randn(1, 1, 224, 224)\n", + "for layer in net:\n", + " X=layer(X)\n", + " print(layer.__class__.__name__,'output shape:\\t',X.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "356e7ffe-330d-48f9-ba76-3a54acddbeaa", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Fashion-MNIST图像的分辨率\n", + "低于ImageNet图像。\n", + "我们将它们增加到$224 \\times 224$" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "df2cf34a-9cd9-4384-a453-00043a33f530", + "metadata": { + "origin_pos": 9, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "batch_size = 128\n", + "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=224)" + ] + }, + { + "cell_type": "markdown", + "id": "0bc4d037-556d-4847-b154-19f265aa660c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练AlexNet。注意:这里是欠拟合的标准曲线" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "967372e1-5db6-4762-a0d4-fc990ff5f0f7", + "metadata": { + "origin_pos": 11, + "scrolled": true, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loss 0.333, train acc 0.878, test acc 0.878\n", + "1762.7 examples/sec on cuda:0\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-03-21T08:52:06.634916\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "lr, num_epochs = 0.01, 10\n", + "d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())" + ] + }, + { + "cell_type": "markdown", + "id": "fc1d1ce8-9b00-4b87-9ff1-177da5ee139d", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 使用块的网络(VGG)\n", + "\n", + "VGG块" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "cf0cb392-48d9-451d-a7ea-e1892865657d", + "metadata": { + "origin_pos": 4, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "\n", + "def vgg_block(num_convs, in_channels, out_channels):\n", + " layers = []\n", + " for _ in range(num_convs):\n", + " layers.append(nn.Conv2d(in_channels, out_channels,\n", + " kernel_size=3, padding=1))\n", + " layers.append(nn.ReLU())\n", + " in_channels = out_channels\n", + " layers.append(nn.MaxPool2d(kernel_size=2,stride=2))\n", + " return nn.Sequential(*layers)" + ] + }, + { + "cell_type": "markdown", + "id": "e40a3ce9-5838-491c-b02a-d4f626c5f024", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "VGG网络\n", + "\n", + "原始VGG网络有5个卷积块,其中前两个块各有一个卷积层,后三个块各包含两个卷积层。 第一个模块有64个输出通道,每个后续模块将输出通道数量翻倍,直到该数字达到512。由于该网络使用8个卷积层和3个全连接层,因此它通常被称为VGG-11。" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "aad71cf3-3c74-4162-bf78-b5b2e313cf65", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "conv_arch = ((1, 64), (1, 128), (2, 256), (2, 512), (2, 512))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "df9c3499-e926-4821-8136-e66baf433731", + "metadata": { + "origin_pos": 10, + "slideshow": { + "slide_type": "slide" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def vgg(conv_arch):\n", + " conv_blks = []\n", + " in_channels = 1\n", + " for (num_convs, out_channels) in conv_arch:\n", + " conv_blks.append(vgg_block(num_convs, in_channels, out_channels))\n", + " in_channels = out_channels\n", + "\n", + " return nn.Sequential(\n", + " *conv_blks, nn.Flatten(),\n", + " nn.Linear(out_channels * 7 * 7, 4096), nn.ReLU(), nn.Dropout(0.5),\n", + " nn.Linear(4096, 4096), nn.ReLU(), nn.Dropout(0.5),\n", + " nn.Linear(4096, 10))\n", + "\n", + "net = vgg(conv_arch)" + ] + }, + { + "cell_type": "markdown", + "id": "18ceef7a-71d8-4eab-b340-c0848b4675b6", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "观察每个层输出的形状" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "20065462-d243-406d-919e-5af13f73c088", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sequential output shape:\t torch.Size([1, 64, 112, 112])\n", + "Sequential output shape:\t torch.Size([1, 128, 56, 56])\n", + "Sequential output shape:\t torch.Size([1, 256, 28, 28])\n", + "Sequential output shape:\t torch.Size([1, 512, 14, 14])\n", + "Sequential output shape:\t torch.Size([1, 512, 7, 7])\n", + "Flatten output shape:\t torch.Size([1, 25088])\n", + "Linear output shape:\t torch.Size([1, 4096])\n", + "ReLU output shape:\t torch.Size([1, 4096])\n", + "Dropout output shape:\t torch.Size([1, 4096])\n", + "Linear output shape:\t torch.Size([1, 4096])\n", + "ReLU output shape:\t torch.Size([1, 4096])\n", + "Dropout output shape:\t torch.Size([1, 4096])\n", + "Linear output shape:\t torch.Size([1, 10])\n" + ] + } + ], + "source": [ + "X = torch.randn(size=(1, 1, 224, 224))\n", + "for blk in net:\n", + " X = blk(X)\n", + " print(blk.__class__.__name__,'output shape:\\t',X.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "49d25e85-676d-4b59-8148-58596e407411", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "由于VGG-11比AlexNet计算量更大,因此我们构建了一个通道数较少的网络" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "286b6e79-ca57-4ac4-b018-e64a210dd88e", + "metadata": { + "origin_pos": 17, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "ratio = 4\n", + "small_conv_arch = [(pair[0], pair[1] // ratio) for pair in conv_arch]\n", + "net = vgg(small_conv_arch)" + ] + }, + { + "cell_type": "markdown", + "id": "0e9ec14c-f318-4c0e-969c-4a17e9ec09a7", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "模型训练。仔细观察:这里出现了过拟合" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e986bb65-7c5f-4402-af8d-55ccc4760662", + "metadata": { + "origin_pos": 20, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loss 0.172, train acc 0.936, test acc 0.914\n", + "1119.5 examples/sec on cuda:0\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-03-21T09:16:51.006260\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "lr, num_epochs, batch_size = 0.05, 10, 128\n", + "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=224)\n", + "d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())" + ] + }, + { + "cell_type": "markdown", + "id": "051bf6dd-6153-4d4a-8a18-4878031998b0", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 网络中的网络(NiN)\n", + "\n", + "NiN块" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "1c18dda6-ece9-4c0a-9235-83e4b5bd6f34", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "\n", + "def nin_block(in_channels, out_channels, kernel_size, strides, padding):\n", + " return nn.Sequential(\n", + " nn.Conv2d(in_channels, out_channels, kernel_size, strides, padding),\n", + " nn.ReLU(),\n", + " nn.Conv2d(out_channels, out_channels, kernel_size=1), nn.ReLU(),\n", + " nn.Conv2d(out_channels, out_channels, kernel_size=1), nn.ReLU())" + ] + }, + { + "cell_type": "markdown", + "id": "9a43fb36-d21c-48d8-afc9-3940fc4151c8", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "NiN模型" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4d406faa-4e96-49f8-b226-a76be1f1cc40", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "net = nn.Sequential(\n", + " nin_block(1, 96, kernel_size=11, strides=4, padding=0),\n", + " nn.MaxPool2d(3, stride=2),\n", + " nin_block(96, 256, kernel_size=5, strides=1, padding=2),\n", + " nn.MaxPool2d(3, stride=2),\n", + " nin_block(256, 384, kernel_size=3, strides=1, padding=1),\n", + " nn.MaxPool2d(3, stride=2),\n", + " nn.Dropout(0.5),\n", + " nin_block(384, 10, kernel_size=3, strides=1, padding=1), # 标签类别数是10\n", + " nn.AdaptiveAvgPool2d((1, 1)),\n", + " nn.Flatten()) # 将四维的输出转成二维的输出,其形状为(批量大小,10)" + ] + }, + { + "cell_type": "markdown", + "id": "554152fc-1a8c-40dc-9892-996c7039d471", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "查看每个块的输出形状" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "73bc25b6-f379-4013-8ee2-b7e80c50a23f", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sequential output shape:\t torch.Size([1, 96, 54, 54])\n", + "MaxPool2d output shape:\t torch.Size([1, 96, 26, 26])\n", + "Sequential output shape:\t torch.Size([1, 256, 26, 26])\n", + "MaxPool2d output shape:\t torch.Size([1, 256, 12, 12])\n", + "Sequential output shape:\t torch.Size([1, 384, 12, 12])\n", + "MaxPool2d output shape:\t torch.Size([1, 384, 5, 5])\n", + "Dropout output shape:\t torch.Size([1, 384, 5, 5])\n", + "Sequential output shape:\t torch.Size([1, 10, 5, 5])\n", + "AdaptiveAvgPool2d output shape:\t torch.Size([1, 10, 1, 1])\n", + "Flatten output shape:\t torch.Size([1, 10])\n" + ] + } + ], + "source": [ + "X = torch.rand(size=(1, 1, 224, 224))\n", + "for layer in net:\n", + " X = layer(X)\n", + " print(layer.__class__.__name__,'output shape:\\t', X.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "feb20603-ba72-4fb4-9763-ba195b16da54", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练模型" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "5ef0873e-78e2-4f65-9ce0-4dba0c5f48c9", + "metadata": { + "origin_pos": 13, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loss 2.303, train acc 0.100, test acc 0.100\n", + "1445.5 examples/sec on cuda:0\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-03-22T09:47:52.697012\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "lr, num_epochs, batch_size = 0.1, 10, 128\n", + "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=224)\n", + "d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())" + ] + }, + { + "cell_type": "markdown", + "id": "3b78340b-c3f8-4617-876b-79f7e89d7d2e", + "metadata": {}, + "source": [ + "注意:这是典型的训练失败案例。\n", + "\n", + "- 思考:如何(调参以实现)提高准确性?" + ] + }, + { + "cell_type": "markdown", + "id": "0d7743da-c831-4ad6-bb10-bd55e05a7178", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 含并行连结的网络(GoogLeNet)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "8892c059-5c03-4cb2-9a2f-5d3cfecbf832", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from torch.nn import functional as F\n", + "from d2l import torch as d2l" + ] + }, + { + "cell_type": "markdown", + "id": "f161d0dd-95c0-4f0b-97e4-8e33bec1c5ac", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Inception块" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "4c8873f6-a2c8-43ed-bfb6-702d37fa897b", + "metadata": { + "origin_pos": 2, + "slideshow": { + "slide_type": "-" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "class Inception(nn.Module):\n", + " # c1--c4是每条路径的输出通道数\n", + " def __init__(self, in_channels, c1, c2, c3, c4, **kwargs):\n", + " super(Inception, self).__init__(**kwargs)\n", + " # 线路1,单1x1卷积层\n", + " self.p1_1 = nn.Conv2d(in_channels, c1, kernel_size=1)\n", + " # 线路2,1x1卷积层后接3x3卷积层\n", + " self.p2_1 = nn.Conv2d(in_channels, c2[0], kernel_size=1)\n", + " self.p2_2 = nn.Conv2d(c2[0], c2[1], kernel_size=3, padding=1)\n", + " # 线路3,1x1卷积层后接5x5卷积层\n", + " self.p3_1 = nn.Conv2d(in_channels, c3[0], kernel_size=1)\n", + " self.p3_2 = nn.Conv2d(c3[0], c3[1], kernel_size=5, padding=2)\n", + " # 线路4,3x3最大汇聚层后接1x1卷积层\n", + " self.p4_1 = nn.MaxPool2d(kernel_size=3, stride=1, padding=1)\n", + " self.p4_2 = nn.Conv2d(in_channels, c4, kernel_size=1)\n", + "\n", + " def forward(self, x):\n", + " p1 = F.relu(self.p1_1(x))\n", + " p2 = F.relu(self.p2_2(F.relu(self.p2_1(x))))\n", + " p3 = F.relu(self.p3_2(F.relu(self.p3_1(x))))\n", + " p4 = F.relu(self.p4_2(self.p4_1(x)))\n", + " # 在通道维度上连结输出\n", + " return torch.cat((p1, p2, p3, p4), dim=1)" + ] + }, + { + "cell_type": "markdown", + "id": "1f734be6-59ed-4dd2-b60f-5f7616942401", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "GoogLeNet模型" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "2ae4258c-339b-4bb2-b78a-7ad180697576", + "metadata": { + "origin_pos": 22, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "b1 = nn.Sequential(nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3),\n", + " nn.ReLU(),\n", + " nn.MaxPool2d(kernel_size=3, stride=2, padding=1))\n", + "\n", + "b2 = nn.Sequential(nn.Conv2d(64, 64, kernel_size=1),\n", + " nn.ReLU(),\n", + " nn.Conv2d(64, 192, kernel_size=3, padding=1),\n", + " nn.ReLU(),\n", + " nn.MaxPool2d(kernel_size=3, stride=2, padding=1))\n", + "\n", + "b3 = nn.Sequential(Inception(192, 64, (96, 128), (16, 32), 32),\n", + " Inception(256, 128, (128, 192), (32, 96), 64),\n", + " nn.MaxPool2d(kernel_size=3, stride=2, padding=1))\n", + "\n", + "b4 = nn.Sequential(Inception(480, 192, (96, 208), (16, 48), 64),\n", + " Inception(512, 160, (112, 224), (24, 64), 64),\n", + " Inception(512, 128, (128, 256), (24, 64), 64),\n", + " Inception(512, 112, (144, 288), (32, 64), 64),\n", + " Inception(528, 256, (160, 320), (32, 128), 128),\n", + " nn.MaxPool2d(kernel_size=3, stride=2, padding=1))\n", + "\n", + "b5 = nn.Sequential(Inception(832, 256, (160, 320), (32, 128), 128),\n", + " Inception(832, 384, (192, 384), (48, 128), 128),\n", + " nn.AdaptiveAvgPool2d((1,1)),\n", + " nn.Flatten())\n", + "\n", + "net = nn.Sequential(b1, b2, b3, b4, b5, nn.Linear(1024, 10))" + ] + }, + { + "cell_type": "markdown", + "id": "0d0f8c4d-c45f-4610-9d0d-eccc76839e4a", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "为了使Fashion-MNIST上的训练短小精悍,我们将输入的高和宽从224降到96" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "23870be5-1556-405a-bcef-2c0fe26cb645", + "metadata": { + "origin_pos": 26, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sequential output shape:\t torch.Size([1, 64, 24, 24])\n", + "Sequential output shape:\t torch.Size([1, 192, 12, 12])\n", + "Sequential output shape:\t torch.Size([1, 480, 6, 6])\n", + "Sequential output shape:\t torch.Size([1, 832, 3, 3])\n", + "Sequential output shape:\t torch.Size([1, 1024])\n", + "Linear output shape:\t torch.Size([1, 10])\n" + ] + } + ], + "source": [ + "X = torch.rand(size=(1, 1, 96, 96))\n", + "for layer in net:\n", + " X = layer(X)\n", + " print(layer.__class__.__name__,'output shape:\\t', X.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "9b814549-6649-4548-9297-d3847abaced1", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练模型" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "54f90511-0268-4d50-9316-a8ddbc953670", + "metadata": { + "origin_pos": 29, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loss 0.244, train acc 0.908, test acc 0.896\n", + "3490.2 examples/sec on cuda:0\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:55:44.273318\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "lr, num_epochs, batch_size = 0.1, 10, 128\n", + "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=96)\n", + "d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())" + ] + }, + { + "cell_type": "markdown", + "id": "7edc7652-ed97-4851-aac8-ed51bacae82f", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "## 批量规范化\n", + "\n", + "从零实现" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "c59dccc1-434e-4134-a988-b4f8c4ec63f0", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d6629b56-83fb-47bf-93cf-98becf2a088f", + "metadata": { + "origin_pos": 2, + "slideshow": { + "slide_type": "slide" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def batch_norm(X, gamma, beta, moving_mean, moving_var, eps, momentum):\n", + " if not torch.is_grad_enabled(): # 判断当前模式是训练模式还是预测模式\n", + " # 如果是在预测模式下,直接使用传入的移动平均所得的均值和方差\n", + " X_hat = (X - moving_mean) / torch.sqrt(moving_var + eps)\n", + " else:\n", + " assert len(X.shape) in (2, 4)\n", + " if len(X.shape) == 2: # 使用全连接层的情况,计算特征维上的均值和方差\n", + " mean = X.mean(dim=0)\n", + " var = ((X - mean) ** 2).mean(dim=0)\n", + " else: # 使用二维卷积层的情况,计算通道维上(axis=1)的均值和方差\n", + " # 这里我们需要保持X的形状以便后面可以做广播运算\n", + " mean = X.mean(dim=(0, 2, 3), keepdim=True)\n", + " var = ((X - mean) ** 2).mean(dim=(0, 2, 3), keepdim=True)\n", + " # 训练模式下,用当前的均值和方差做标准化\n", + " X_hat = (X - mean) / torch.sqrt(var + eps)\n", + " # 更新移动平均的均值和方差\n", + " moving_mean = momentum * moving_mean + (1.0 - momentum) * mean\n", + " moving_var = momentum * moving_var + (1.0 - momentum) * var\n", + " Y = gamma * X_hat + beta # 缩放和移位\n", + " return Y, moving_mean.data, moving_var.data" + ] + }, + { + "cell_type": "markdown", + "id": "f4872957-57c1-40c0-be54-6d00862c9d2f", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "创建一个正确的`BatchNorm`层" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f4c0ef6f-7546-4949-bfd9-b1e10f3abade", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "class BatchNorm(nn.Module):\n", + " def __init__(self, num_features, num_dims):\n", + " # num_features:全连接层的输出数量或卷积层的输出通道数。\n", + " # num_dims:2表示全连接层,4表示卷积层\n", + " super().__init__()\n", + " if num_dims == 2:\n", + " shape = (1, num_features)\n", + " else:\n", + " shape = (1, num_features, 1, 1)\n", + " # 参与求梯度和迭代的拉伸和偏移参数,分别初始化成1和0\n", + " self.gamma = nn.Parameter(torch.ones(shape))\n", + " self.beta = nn.Parameter(torch.zeros(shape))\n", + " # 非模型参数:均值、方差\n", + " self.moving_mean = torch.zeros(shape)\n", + " self.moving_var = torch.ones(shape)\n", + "\n", + " def forward(self, X):\n", + " if self.moving_mean.device != X.device:\n", + " # 如果X不在内存上,将moving_mean和moving_var复制到X所在显存上\n", + " self.moving_mean = self.moving_mean.to(X.device)\n", + " self.moving_var = self.moving_var.to(X.device)\n", + " # 保存更新过的moving_mean和moving_var\n", + " Y, self.moving_mean, self.moving_var = batch_norm(\n", + " X, self.gamma, self.beta, self.moving_mean,\n", + " self.moving_var, eps=1e-5, momentum=0.9)\n", + " return Y" + ] + }, + { + "cell_type": "markdown", + "id": "cf94bbae-4049-443e-82f9-b483b2b25a8e", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "应用`BatchNorm`\n", + "于LeNet模型" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "cc24a7dc-08ac-4a95-bfbc-b8aada3842e7", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "net = nn.Sequential(\n", + " nn.Conv2d(1, 6, kernel_size=5), BatchNorm(6, num_dims=4), nn.Sigmoid(),\n", + " nn.AvgPool2d(kernel_size=2, stride=2),\n", + " nn.Conv2d(6, 16, kernel_size=5), BatchNorm(16, num_dims=4), nn.Sigmoid(),\n", + " nn.AvgPool2d(kernel_size=2, stride=2), nn.Flatten(),\n", + " nn.Linear(16*4*4, 120), BatchNorm(120, num_dims=2), nn.Sigmoid(),\n", + " nn.Linear(120, 84), BatchNorm(84, num_dims=2), nn.Sigmoid(),\n", + " nn.Linear(84, 10))" + ] + }, + { + "cell_type": "markdown", + "id": "fbcd9945-0abd-4443-962f-56e928685200", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "在Fashion-MNIST数据集上训练网络。注意:学习率大很多" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "3ab1ecba-57e6-4cec-a902-e68dc49b64d1", + "metadata": { + "origin_pos": 13, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loss 0.263, train acc 0.902, test acc 0.857\n", + "31378.5 examples/sec on cuda:0\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-03-22T19:59:51.169965\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "lr, num_epochs, batch_size = 1.0, 10, 256\n", + "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)\n", + "d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())" + ] + }, + { + "cell_type": "markdown", + "id": "b8bc077b-1747-4de4-87c1-4da4e574df34", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "拉伸参数`gamma`和偏移参数`beta`" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ed6c4a56-a677-4c93-9d13-8a3b5f5c489d", + "metadata": { + "origin_pos": 17, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(tensor([2.9182, 3.9908, 2.5709, 2.1960, 3.0324, 2.0170], device='cuda:0',\n", + " grad_fn=),\n", + " tensor([-0.4980, 2.6303, -2.3259, -1.4185, -3.2187, -2.1353], device='cuda:0',\n", + " grad_fn=))" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "net[1].gamma.reshape((-1,)), net[1].beta.reshape((-1,))" + ] + }, + { + "cell_type": "markdown", + "id": "57715920-0def-4217-96b1-4425f07b02f6", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "简明实现" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ff5c985a-76c9-4a4a-bccc-feb3179e3c4b", + "metadata": { + "origin_pos": 21, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "net = nn.Sequential(\n", + " nn.Conv2d(1, 6, kernel_size=5), nn.BatchNorm2d(6), nn.Sigmoid(),\n", + " nn.AvgPool2d(kernel_size=2, stride=2),\n", + " nn.Conv2d(6, 16, kernel_size=5), nn.BatchNorm2d(16), nn.Sigmoid(),\n", + " nn.AvgPool2d(kernel_size=2, stride=2), nn.Flatten(),\n", + " nn.Linear(256, 120), nn.BatchNorm1d(120), nn.Sigmoid(),\n", + " nn.Linear(120, 84), nn.BatchNorm1d(84), nn.Sigmoid(),\n", + " nn.Linear(84, 10))" + ] + }, + { + "cell_type": "markdown", + "id": "c991fda9-53a2-4510-9ee5-2968b3f521a5", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "使用相同超参数来训练模型。注意:通常高级API运行速度快得多,因为它的代码已编译为C++或CUDA" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "2e6a118b-c4db-4db0-a15f-ab361efb2051", + "metadata": { + "origin_pos": 24, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loss 0.281, train acc 0.896, test acc 0.843\n", + "55970.4 examples/sec on cuda:0\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-03-22T20:00:16.979860\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())" + ] + }, + { + "cell_type": "markdown", + "id": "be2ce66c-0694-4c20-8f9c-e4c62b9e92b1", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "## 残差网络(ResNet)\n", + "\n", + "残差块" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b253049d-96a4-4871-a6a1-c9d2de3a9379", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from torch.nn import functional as F\n", + "from d2l import torch as d2l\n", + "\n", + "\n", + "class Residual(nn.Module): \n", + " def __init__(self, input_channels, num_channels,\n", + " use_1x1conv=False, strides=1):\n", + " super().__init__()\n", + " self.conv1 = nn.Conv2d(input_channels, num_channels,\n", + " kernel_size=3, padding=1, stride=strides)\n", + " self.conv2 = nn.Conv2d(num_channels, num_channels,\n", + " kernel_size=3, padding=1)\n", + " if use_1x1conv:\n", + " self.conv3 = nn.Conv2d(input_channels, num_channels,\n", + " kernel_size=1, stride=strides)\n", + " else:\n", + " self.conv3 = None\n", + " self.bn1 = nn.BatchNorm2d(num_channels)\n", + " self.bn2 = nn.BatchNorm2d(num_channels)\n", + "\n", + " def forward(self, X):\n", + " Y = F.relu(self.bn1(self.conv1(X)))\n", + " Y = self.bn2(self.conv2(Y))\n", + " if self.conv3:\n", + " X = self.conv3(X)\n", + " Y += X\n", + " return F.relu(Y)" + ] + }, + { + "cell_type": "markdown", + "id": "72824a52-1ed9-4f12-b826-e60d2ce42568", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "输入和输出形状一致" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "02cc50d9-5710-4e2f-bf0f-568f1046321e", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([4, 3, 6, 6])" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "blk = Residual(3,3)\n", + "X = torch.rand(4, 3, 6, 6)\n", + "Y = blk(X)\n", + "Y.shape" + ] + }, + { + "cell_type": "markdown", + "id": "c09c8e89-dfdb-4e6f-a75b-b949f8e04d6c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "增加输出通道数的同时,减半输出的高和宽" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "57b5f05e-853c-460e-afb7-3a3483c7e776", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([4, 6, 3, 3])" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "blk = Residual(3,6, use_1x1conv=True, strides=2)\n", + "blk(X).shape" + ] + }, + { + "cell_type": "markdown", + "id": "57bd6688-aa3a-4758-b0f6-67b786db8c7a", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "ResNet模型" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "95fcb71a-7682-4efe-9434-688e4f1606e0", + "metadata": { + "origin_pos": 26, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "b1 = nn.Sequential(nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3),\n", + " nn.BatchNorm2d(64), nn.ReLU(),\n", + " nn.MaxPool2d(kernel_size=3, stride=2, padding=1))\n", + "\n", + "def resnet_block(input_channels, num_channels, num_residuals,\n", + " first_block=False):\n", + " blk = []\n", + " for i in range(num_residuals):\n", + " if i == 0 and not first_block:\n", + " blk.append(Residual(input_channels, num_channels,\n", + " use_1x1conv=True, strides=2))\n", + " else:\n", + " blk.append(Residual(num_channels, num_channels))\n", + " return blk\n", + "\n", + "b2 = nn.Sequential(*resnet_block(64, 64, 2, first_block=True))\n", + "b3 = nn.Sequential(*resnet_block(64, 128, 2))\n", + "b4 = nn.Sequential(*resnet_block(128, 256, 2))\n", + "b5 = nn.Sequential(*resnet_block(256, 512, 2))\n", + "\n", + "net = nn.Sequential(b1, b2, b3, b4, b5,\n", + " nn.AdaptiveAvgPool2d((1,1)),\n", + " nn.Flatten(), nn.Linear(512, 10))" + ] + }, + { + "cell_type": "markdown", + "id": "ead75736-665d-48b8-9ba9-7634f5ed1395", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "观察一下ResNet中不同模块的输入形状是如何变化的" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "caaf4445-c02e-4d4e-af70-ead31c246332", + "metadata": { + "origin_pos": 30, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sequential output shape:\t torch.Size([1, 64, 56, 56])\n", + "Sequential output shape:\t torch.Size([1, 64, 56, 56])\n", + "Sequential output shape:\t torch.Size([1, 128, 28, 28])\n", + "Sequential output shape:\t torch.Size([1, 256, 14, 14])\n", + "Sequential output shape:\t torch.Size([1, 512, 7, 7])\n", + "AdaptiveAvgPool2d output shape:\t torch.Size([1, 512, 1, 1])\n", + "Flatten output shape:\t torch.Size([1, 512])\n", + "Linear output shape:\t torch.Size([1, 10])\n" + ] + } + ], + "source": [ + "X = torch.rand(size=(1, 1, 224, 224))\n", + "for layer in net:\n", + " X = layer(X)\n", + " print(layer.__class__.__name__,'output shape:\\t', X.shape)" + ] + }, + { + "cell_type": "markdown", + "id": "057bd90d-193f-4577-8d4b-e922d8e1ae18", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练模型" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "bd0e57ed-ed08-4d18-8cdf-4062fad7ffe1", + "metadata": { + "origin_pos": 33, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loss 0.009, train acc 0.998, test acc 0.922\n", + "4702.7 examples/sec on cuda:0\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:46:46.812949\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "lr, num_epochs, batch_size = 0.05, 10, 256\n", + "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=96)\n", + "d2l.train_ch6(net, train_iter, test_iter, num_epochs, lr, d2l.try_gpu())" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/07_convolution/Convolution-1.py b/07_convolution/Convolution-1.py new file mode 100644 index 0000000..dbd6e1e --- /dev/null +++ b/07_convolution/Convolution-1.py @@ -0,0 +1,150 @@ +import torch + +import torch.nn as nn +import torch.nn.functional as F + +import torch.optim as optim + +import torchvision +import torchvision.transforms as transforms + + +class FFNet(nn.Module): + """ + Feedforward neural network, virtual base class + """ + def __init__(self): + super(FFNet, self).__init__() + self.dnn_model = self.build_model() + + def build_model(self): + """ build specific model """ + raise NotImplementedError + + def forward(self, x): + """ feed data forward and return result """ + raise NotImplementedError + + def train_model(self, trainloader, testloader, loss_fn, optimizer, num_epochs): + """Train a model.""" + + def train_epoch(model, dataloader, loss_fn, optimizer): + """Train a single epoch""" + num_data = len(dataloader.dataset) + # Set the model to training mode + model.train() + for batch, (X, y) in enumerate(dataloader): + # Compute prediction error + pred = model(X) + loss = loss_fn(pred, y) + + # Backpropagation + optimizer.zero_grad() + loss.backward() + optimizer.step() + + if batch % 100 == 0: + loss, current = loss.item(), batch * len(X) + print(f"loss: {loss:>7f} [{current:>5d}/{num_data:>5d}]") + + def test_epoch(model, dataloader, loss_fn): + """Test a single epoch""" + num_data = len(dataloader.dataset) + num_batches = len(dataloader) + # Set the model to evaluate mode + model.eval() + test_loss, correct = 0, 0 + with torch.no_grad(): + for X, y in dataloader: + pred = model(X) + test_loss += loss_fn(pred, y).item() + correct += (pred.argmax(1) == y).type(torch.float).sum().item() + test_loss /= num_batches + correct /= num_data + print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n") + + for epoch in range(num_epochs): + print(f"Epoch {epoch+1}\n-------------------------------") + train_epoch(self, trainloader, loss_fn, optimizer) + test_epoch(self, testloader, loss_fn) + print("Done!") + + def make_predict(self, images): + """ predict labels for images """ + self.eval() + with torch.no_grad(): + outputs = self(images) + _, predicted = torch.max(outputs.data, 1) # (max, max_indices) + return predicted + + def evaluate_model(self, testloader, n): + """ evaluation: return accuracy """ + correct = 0 + for inputs, labels in testloader: + pred = self.make_predict(inputs) + correct += (pred == labels).sum() + return 100 * correct / n + + def predict_one(self, x): + self.eval() + with torch.no_grad(): + outputs = self(x) + predicted = outputs[0].argmax(0) + return predicted + + +class LeNet(FFNet): + def __init__(self): + super(LeNet, self).__init__() + + def build_model(self): + """ build specific model """ + raise NotImplementedError + + def forward(self, x): + """ feed data forward and return result """ + return self.dnn_model(x) + + +if __name__ == "__main__": + # load train and test set + trainset = torchvision.datasets.MNIST( + '../data', train=True, download=True, transform=transforms.ToTensor()) + testset = torchvision.datasets.MNIST( + '../data', train=False, download=True, transform=transforms.ToTensor()) + print(f'Train #: {len(trainset)}; Test #: {len(testset)}') + # data iterator + dataiter = iter(torch.utils.data.DataLoader(trainset, batch_size=8, shuffle=False)) + images, labels = next(dataiter) + print(f'Labels: {labels}; Batch shape: {images.size()}') + + # construct model + model = LeNet() + print(model) + + # data loader: easier iteration + BATCH_SIZE = 256 + NUM_WORKERS = 4 + trainloader = torch.utils.data.DataLoader( + trainset, batch_size=BATCH_SIZE, shuffle=True, num_workers=NUM_WORKERS) + testloader = torch.utils.data.DataLoader( + testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS) + + # train the model + LR = 0.1 + loss_fn = nn.CrossEntropyLoss() # cross entropy + optimizer = torch.optim.SGD(model.parameters(), lr=LR) # SGD + NUM_EPOCH = 5 + model.train_model(trainloader, testloader, loss_fn, optimizer, NUM_EPOCH) + + # check prediction accuracy + print(f'Labels : {labels}') + print(f'Prediction: {model.make_predict(images)}') + print(f'Accuracy: {model.evaluate_model(testloader, len(testset)):.2f}') + + # application: predict hand-written digit + from PIL import Image + image = Image.open('number6c.png') + image = transforms.ToTensor()(image).unsqueeze(0) + print(f'loaded image shape: {image.size()}') + print(f'Predicted: "{model.predict_one(image)}", Actual: "{6}"') diff --git a/07_convolution/expected_output.txt b/07_convolution/expected_output.txt new file mode 100644 index 0000000..6242834 --- /dev/null +++ b/07_convolution/expected_output.txt @@ -0,0 +1,64 @@ +Train #: 60000; Test #: 10000 +Labels: tensor([5, 0, 4, 1, 9, 2, 1, 3]); Batch shape: torch.Size([8, 1, 28, 28]) +LeNet( + (dnn_model): Sequential( + (0): Conv2d(1, 32, kernel_size=(5, 5), stride=(1, 1)) + (1): ReLU() + (2): Conv2d(32, 32, kernel_size=(5, 5), stride=(1, 1)) + (3): ReLU() + (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) + (5): Conv2d(32, 64, kernel_size=(5, 5), stride=(1, 1)) + (6): ReLU() + (7): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) + (8): Flatten(start_dim=1, end_dim=-1) + (9): Linear(in_features=576, out_features=256, bias=True) + (10): ReLU() + (11): Linear(in_features=256, out_features=10, bias=True) + ) +) +Epoch 1 +------------------------------- +loss: 2.299683 [ 0/60000] +loss: 0.348185 [25600/60000] +loss: 0.239826 [51200/60000] +Test Error: + Accuracy: 95.7%, Avg loss: 0.133152 + +Epoch 2 +------------------------------- +loss: 0.220472 [ 0/60000] +loss: 0.056995 [25600/60000] +loss: 0.078447 [51200/60000] +Test Error: + Accuracy: 90.3%, Avg loss: 0.284392 + +Epoch 3 +------------------------------- +loss: 0.255085 [ 0/60000] +loss: 0.071809 [25600/60000] +loss: 0.080565 [51200/60000] +Test Error: + Accuracy: 98.2%, Avg loss: 0.056539 + +Epoch 4 +------------------------------- +loss: 0.074963 [ 0/60000] +loss: 0.053749 [25600/60000] +loss: 0.042180 [51200/60000] +Test Error: + Accuracy: 98.3%, Avg loss: 0.048151 + +Epoch 5 +------------------------------- +loss: 0.018707 [ 0/60000] +loss: 0.047416 [25600/60000] +loss: 0.022978 [51200/60000] +Test Error: + Accuracy: 98.6%, Avg loss: 0.041758 + +Done! +Labels : tensor([5, 0, 4, 1, 9, 2, 1, 3]) +Prediction: tensor([5, 0, 4, 1, 9, 2, 1, 3]) +Accuracy: 98.60 +loaded image shape: torch.Size([1, 1, 28, 28]) +Predicted: "0", Actual: "6" \ No newline at end of file diff --git a/07_convolution/number5.png b/07_convolution/number5.png new file mode 100644 index 0000000..24eb1c6 Binary files /dev/null and b/07_convolution/number5.png differ diff --git a/07_convolution/number6c.png b/07_convolution/number6c.png new file mode 100644 index 0000000..ab336d5 Binary files /dev/null and b/07_convolution/number6c.png differ diff --git a/08_recurrent/08_modern_rnn.ipynb b/08_recurrent/08_modern_rnn.ipynb new file mode 100644 index 0000000..012a929 --- /dev/null +++ b/08_recurrent/08_modern_rnn.ipynb @@ -0,0 +1,5211 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "5c5922d3-95a1-4415-91a0-3399e39b6cd3", + "metadata": {}, + "source": [ + "# 现代循环神经网络\n", + "\n", + "目录\n", + "\n", + "- 长短期记忆网络(LSTM)\n", + "- 门控循环单元(GRU)\n", + "- 深度循环神经网络\n", + "- 双向循环神经网络" + ] + }, + { + "cell_type": "markdown", + "id": "1035997a-b182-4d01-b0a6-66ddd0d0c62b", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 长短期记忆网络(LSTM)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "526021ea-9bbe-42a8-9e64-22692860d2ff", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "batch_size, num_steps = 32, 35\n", + "train_iter, vocab = d2l.load_data_time_machine(batch_size, num_steps)" + ] + }, + { + "cell_type": "markdown", + "id": "842b0708-6b64-4ca0-adab-c809b2b5b1df", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "初始化模型参数" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "edee6ecd-f9a8-4b43-a509-aa8c48e874a0", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def get_lstm_params(vocab_size, num_hiddens, device):\n", + " num_inputs = num_outputs = vocab_size\n", + "\n", + " def normal(shape):\n", + " return torch.randn(size=shape, device=device)*0.01\n", + "\n", + " def three():\n", + " return (normal((num_inputs, num_hiddens)),\n", + " normal((num_hiddens, num_hiddens)),\n", + " torch.zeros(num_hiddens, device=device))\n", + "\n", + " W_xi, W_hi, b_i = three() # 输入门参数\n", + " W_xf, W_hf, b_f = three() # 遗忘门参数\n", + " W_xo, W_ho, b_o = three() # 输出门参数\n", + " W_xc, W_hc, b_c = three() # 候选记忆元参数\n", + " # 输出层参数\n", + " W_hq = normal((num_hiddens, num_outputs))\n", + " b_q = torch.zeros(num_outputs, device=device)\n", + " params = [W_xi, W_hi, b_i, W_xf, W_hf, b_f, W_xo, W_ho, b_o, W_xc, W_hc,\n", + " b_c, W_hq, b_q]\n", + " for param in params:\n", + " param.requires_grad_(True)\n", + " return params" + ] + }, + { + "cell_type": "markdown", + "id": "c5abc77a-229e-4aaa-9bbb-a5a6d8a061e1", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "初始化隐状态、记忆元" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "86888086-63cd-4411-98a5-a2e3f9a67bf7", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def init_lstm_state(batch_size, num_hiddens, device):\n", + " return (torch.zeros((batch_size, num_hiddens), device=device),\n", + " torch.zeros((batch_size, num_hiddens), device=device))" + ] + }, + { + "cell_type": "markdown", + "id": "453c58cf-54cc-490b-b337-ec5953e0c409", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "实际模型。注意:只有隐状态传递到输出层;记忆元不直接参与输出计算" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "fedefee3-0199-4532-9d3a-edafdfba32b1", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def lstm(inputs, state, params):\n", + " [W_xi, W_hi, b_i, W_xf, W_hf, b_f, W_xo, W_ho, b_o, W_xc, W_hc, b_c,\n", + " W_hq, b_q] = params\n", + " (H, C) = state\n", + " outputs = []\n", + " for X in inputs:\n", + " I = torch.sigmoid((X @ W_xi) + (H @ W_hi) + b_i)\n", + " F = torch.sigmoid((X @ W_xf) + (H @ W_hf) + b_f)\n", + " O = torch.sigmoid((X @ W_xo) + (H @ W_ho) + b_o)\n", + " C_tilda = torch.tanh((X @ W_xc) + (H @ W_hc) + b_c)\n", + " C = F * C + I * C_tilda\n", + " H = O * torch.tanh(C)\n", + " Y = (H @ W_hq) + b_q\n", + " outputs.append(Y)\n", + " return torch.cat(outputs, dim=0), (H, C)" + ] + }, + { + "cell_type": "markdown", + "id": "6a049a6f-80ed-4a3f-b2c0-8f1214446bc9", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d6485ad2-20f4-4d9c-a70d-09956426a472", + "metadata": { + "origin_pos": 17, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "perplexity 1.1, 19331.5 tokens/sec on cuda:0\n", + "time traveller for so it will be convenient to speak of himway a\n", + "travelleryou can show black is white by argument said filby\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:50:45.607624\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "vocab_size, num_hiddens, device = len(vocab), 256, d2l.try_gpu()\n", + "num_epochs, lr = 500, 1\n", + "model = d2l.RNNModelScratch(len(vocab), num_hiddens, device, get_lstm_params,\n", + " init_lstm_state, lstm)\n", + "d2l.train_ch8(model, train_iter, vocab, lr, num_epochs, device)" + ] + }, + { + "cell_type": "markdown", + "id": "7c73a98a-0081-4d5c-88ae-786f3874b62b", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "简洁实现" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "665010d4-5ce1-4610-af14-925606e1fe1b", + "metadata": { + "origin_pos": 21, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "perplexity 1.1, 222756.1 tokens/sec on cuda:0\n", + "time traveller for so it will be convenient to speak of himwas e\n", + "travellerit s against reason said filbywhat the exrect a do\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:51:13.624095\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "num_inputs = vocab_size\n", + "lstm_layer = nn.LSTM(num_inputs, num_hiddens)\n", + "model = d2l.RNNModel(lstm_layer, len(vocab))\n", + "model = model.to(device)\n", + "d2l.train_ch8(model, train_iter, vocab, lr, num_epochs, device)" + ] + }, + { + "cell_type": "markdown", + "id": "49eee153-dc83-4924-b635-c990537f6617", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 门控循环单元(GRU)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "20488c5d-4781-453f-a316-35b07bc58b84", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "batch_size, num_steps = 32, 35\n", + "train_iter, vocab = d2l.load_data_time_machine(batch_size, num_steps)" + ] + }, + { + "cell_type": "markdown", + "id": "99fcb45a-fb0c-4e78-b482-ee65b07d25f0", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "初始化模型参数" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ce37e242-fc70-43e6-b38e-9f9768df6637", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def get_params(vocab_size, num_hiddens, device):\n", + " num_inputs = num_outputs = vocab_size\n", + "\n", + " def normal(shape):\n", + " return torch.randn(size=shape, device=device)*0.01\n", + "\n", + " def three():\n", + " return (normal((num_inputs, num_hiddens)),\n", + " normal((num_hiddens, num_hiddens)),\n", + " torch.zeros(num_hiddens, device=device))\n", + "\n", + " W_xz, W_hz, b_z = three() # 更新门参数\n", + " W_xr, W_hr, b_r = three() # 重置门参数\n", + " W_xh, W_hh, b_h = three() # 候选隐状态参数\n", + " # 输出层参数\n", + " W_hq = normal((num_hiddens, num_outputs))\n", + " b_q = torch.zeros(num_outputs, device=device)\n", + " params = [W_xz, W_hz, b_z, W_xr, W_hr, b_r, W_xh, W_hh, b_h, W_hq, b_q]\n", + " for param in params:\n", + " param.requires_grad_(True)\n", + " return params" + ] + }, + { + "cell_type": "markdown", + "id": "674a2ce3-6c8b-401b-8390-34815195eeef", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "定义隐状态的初始化函数" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "c962f4e5-c015-42c8-b073-7c2774d5a1ce", + "metadata": { + "origin_pos": 10, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def init_gru_state(batch_size, num_hiddens, device):\n", + " return (torch.zeros((batch_size, num_hiddens), device=device), )" + ] + }, + { + "cell_type": "markdown", + "id": "d46b76ba-7567-4d19-b4a9-ef386254c0e8", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "定义门控循环单元模型" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "784bbe7b-b987-40f3-b228-cd640c420b06", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def gru(inputs, state, params):\n", + " W_xz, W_hz, b_z, W_xr, W_hr, b_r, W_xh, W_hh, b_h, W_hq, b_q = params\n", + " H, = state\n", + " outputs = []\n", + " for X in inputs:\n", + " Z = torch.sigmoid((X @ W_xz) + (H @ W_hz) + b_z)\n", + " R = torch.sigmoid((X @ W_xr) + (H @ W_hr) + b_r)\n", + " H_tilda = torch.tanh((X @ W_xh) + ((R * H) @ W_hh) + b_h)\n", + " H = Z * H + (1 - Z) * H_tilda\n", + " Y = H @ W_hq + b_q\n", + " outputs.append(Y)\n", + " return torch.cat(outputs, dim=0), (H,)" + ] + }, + { + "cell_type": "markdown", + "id": "9f142a23-6a30-4152-b4cd-320465fdde48", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "54d1caf6-df93-4b09-8ff7-a79d3fdc97f3", + "metadata": { + "origin_pos": 17, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "perplexity 1.1, 24308.6 tokens/sec on cuda:0\n", + "time travelleryou can show black is white by argument said filby\n", + "travelleryou can show black is white by argument said filby\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:37:00.239620\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "vocab_size, num_hiddens, device = len(vocab), 256, d2l.try_gpu()\n", + "num_epochs, lr = 500, 1\n", + "model = d2l.RNNModelScratch(len(vocab), num_hiddens, device, get_params,\n", + " init_gru_state, gru)\n", + "d2l.train_ch8(model, train_iter, vocab, lr, num_epochs, device)" + ] + }, + { + "cell_type": "markdown", + "id": "99e6c8d5-da91-49e6-927c-4b012fe6cf2c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "简洁实现" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "22312f7f-e43d-42a4-82ec-b66adf677210", + "metadata": { + "origin_pos": 21, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "perplexity 1.0, 302131.8 tokens/sec on cuda:0\n", + "time travelleryou can show black is white by argument said filby\n", + "travelleryou can show black is white by argument said filby\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:37:24.482861\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "num_inputs = vocab_size\n", + "gru_layer = nn.GRU(num_inputs, num_hiddens)\n", + "model = d2l.RNNModel(gru_layer, len(vocab))\n", + "model = model.to(device)\n", + "d2l.train_ch8(model, train_iter, vocab, lr, num_epochs, device)" + ] + }, + { + "cell_type": "markdown", + "id": "c0c85839-2eb3-4e89-bf10-63197ec67988", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "## 深度循环神经网络\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "bf1b6cd6-b5f7-4f09-bc2e-c03d9e614411", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading ../data/timemachine.txt from http://d2l-data.s3-accelerate.amazonaws.com/timemachine.txt...\n" + ] + } + ], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "batch_size, num_steps = 32, 35\n", + "train_iter, vocab = d2l.load_data_time_machine(batch_size, num_steps)" + ] + }, + { + "cell_type": "markdown", + "id": "827b8d71-9105-4103-8b86-b6cd6c2724cf", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "通过`num_layers`的值来设定隐藏层数" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "33fd898e-b4ed-4632-b7bf-43a8585a4715", + "metadata": { + "origin_pos": 5, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "vocab_size, num_hiddens, num_layers = len(vocab), 256, 2\n", + "num_inputs = vocab_size\n", + "device = d2l.try_gpu()\n", + "# 通过num_layers的值来设定隐藏层数\n", + "lstm_layer = nn.LSTM(num_inputs, num_hiddens, num_layers)\n", + "model = d2l.RNNModel(lstm_layer, len(vocab))\n", + "model = model.to(device)" + ] + }, + { + "cell_type": "markdown", + "id": "0bae32e2-c90a-4793-8e0a-8a5d78985745", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "ff5e04c1-9bde-4764-b568-c558e74772a3", + "metadata": { + "origin_pos": 7, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "perplexity 1.0, 199074.9 tokens/sec on cuda:0\n", + "time traveller for so it will be convenient to speak of himwas e\n", + "travelleryou can show black is white by argument said filby\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:18:12.617793\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "num_epochs, lr = 500, 2\n", + "d2l.train_ch8(model, train_iter, vocab, lr, num_epochs, device)" + ] + }, + { + "cell_type": "markdown", + "id": "fdc6d401-d1f3-4426-95f3-1385c1a07653", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "## 双向循环神经网络\n", + "\n", + "双向循环神经网络的错误应用" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "302ad7f5-3dc2-45f9-af35-67ca7b9bc527", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "# 加载数据\n", + "batch_size, num_steps, device = 32, 35, d2l.try_gpu()\n", + "train_iter, vocab = d2l.load_data_time_machine(batch_size, num_steps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21450277-6c6b-43f1-8bf6-6cb43af3aba4", + "metadata": {}, + "outputs": [], + "source": [ + "vocab_size, num_hiddens, num_layers = len(vocab), 256, 2\n", + "num_inputs = vocab_size\n", + "# 通过设置“bidirective=True”来定义双向LSTM模型\n", + "lstm_layer = nn.LSTM(num_inputs, num_hiddens, num_layers, bidirectional=True)\n", + "model = d2l.RNNModel(lstm_layer, len(vocab))\n", + "model = model.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "7e4f3c24-4166-4bfa-b04d-125163d748a6", + "metadata": { + "origin_pos": 2, + "slideshow": { + "slide_type": "slide" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "perplexity 1.1, 118001.0 tokens/sec on cuda:0\n", + "time travellerererererererererererererererererererererererererer\n", + "travellerererererererererererererererererererererererererer\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T02:15:21.385692\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# 训练模型\n", + "num_epochs, lr = 500, 1\n", + "d2l.train_ch8(model, train_iter, vocab, lr, num_epochs, device)\n", + "# 注意:困惑度非常低,但生成的文本质量很低" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/08_recurrent/08_recurrent.ipynb b/08_recurrent/08_recurrent.ipynb new file mode 100644 index 0000000..dc2737e --- /dev/null +++ b/08_recurrent/08_recurrent.ipynb @@ -0,0 +1,18621 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "34533a59-7396-4ae1-9110-f1963ef83817", + "metadata": {}, + "source": [ + "# 循环神经网络\n", + "\n", + "目录\n", + "\n", + "- 序列模型\n", + "- 文本预处理\n", + "- 语言模型和数据集\n", + "- 循环神经网络从零开始实现\n", + "- 循环神经网络的简洁实现" + ] + }, + { + "cell_type": "markdown", + "id": "28c034a0-2977-499e-9969-825ec31f15ae", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 序列模型\n", + "\n", + "使用正弦函数和一些可加性噪声来生成序列数据,时间步为$1, 2, \\ldots, 1000$" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "adbe0af5-a931-4eb7-a1c1-e06a817a36fd", + "metadata": { + "origin_pos": 4, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:23:53.459186\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "%matplotlib inline\n", + "import torch\n", + "from torch import nn\n", + "from d2l import torch as d2l\n", + "\n", + "T = 1000\n", + "time = torch.arange(1, T + 1, dtype=torch.float32)\n", + "x = torch.sin(0.01 * time) + torch.normal(0, 0.2, (T,))\n", + "d2l.plot(time, [x], 'time', 'x', xlim=[1, 1000], figsize=(6, 3))" + ] + }, + { + "cell_type": "markdown", + "id": "40016f4b-419b-4e8c-9f08-683954e4a879", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "将数据映射为数据对$y_t = x_t$和$\\mathbf{x}_t = [x_{t-\\tau}, \\ldots, x_{t-1}]$" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "4a297c5d-5576-4561-95eb-2912de6ff393", + "metadata": { + "origin_pos": 9, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "tau = 4 # Markov 链长度\n", + "features = torch.zeros((T - tau, tau)) # x_t\n", + "for i in range(tau):\n", + " features[:, i] = x[i: T - tau + i] # 顺延一个位置\n", + "labels = x[tau:].reshape((-1, 1)) # y_t\n", + "\n", + "batch_size, n_train = 16, 600\n", + "# 只有前n_train个样本用于训练\n", + "train_iter = d2l.load_array((features[:n_train], labels[:n_train]),\n", + " batch_size, is_train=True)" + ] + }, + { + "cell_type": "markdown", + "id": "d30984f6-b1e0-4184-bfb4-3850b34b72f7", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "使用一个相当简单的架构训练模型:\n", + "一个拥有两个全连接层的多层感知机" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "01306b9f-8e94-41ab-a946-e61d2125871a", + "metadata": { + "origin_pos": 12, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def init_weights(m):\n", + " if type(m) == nn.Linear:\n", + " nn.init.xavier_uniform_(m.weight)\n", + "\n", + "def get_net():\n", + " \"\"\"两个全连接层的多层感知机\"\"\"\n", + " net = nn.Sequential(nn.Linear(4, 10),\n", + " nn.ReLU(),\n", + " nn.Linear(10, 1))\n", + " net.apply(init_weights)\n", + " return net\n", + "\n", + "loss = nn.MSELoss(reduction='none')" + ] + }, + { + "cell_type": "markdown", + "id": "276e9d4d-a27c-4604-84b3-1f86210610ba", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "训练模型" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "efcaf6c4-98ab-41b9-9d88-1b8b8f3a2320", + "metadata": { + "origin_pos": 16, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch 1, loss: 0.075906\n", + "epoch 2, loss: 0.057385\n", + "epoch 3, loss: 0.055220\n", + "epoch 4, loss: 0.053874\n", + "epoch 5, loss: 0.054573\n" + ] + } + ], + "source": [ + "def train(net, train_iter, loss, epochs, lr):\n", + " trainer = torch.optim.Adam(net.parameters(), lr)\n", + " for epoch in range(epochs):\n", + " for X, y in train_iter:\n", + " trainer.zero_grad()\n", + " l = loss(net(X), y)\n", + " l.sum().backward()\n", + " trainer.step()\n", + " print(f'epoch {epoch + 1}, '\n", + " f'loss: {d2l.evaluate_loss(net, train_iter, loss):f}')\n", + "\n", + "net = get_net()\n", + "train(net, train_iter, loss, 5, 0.01)\n", + "# 损失值上升?训练瓶颈,或学习率偏大" + ] + }, + { + "cell_type": "markdown", + "id": "21a70294-3f5d-40a0-bacc-267f4b7c4930", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "单步预测:模型预测下一个时间步。注意:600+4(n_train + tau)之后都是预测" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "9a3e08d9-add6-4a5f-b7d8-343251790e3b", + "metadata": { + "origin_pos": 19, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:23:53.949340\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "onestep_preds = net(features) # features[:, i] = x[i: T - tau + i]\n", + "d2l.plot([time, time[tau:]],\n", + " [x.detach().numpy(), onestep_preds.detach().numpy()], 'time',\n", + " 'x', legend=['data', '1-step preds'], xlim=[1, 1000],\n", + " figsize=(6, 3))" + ] + }, + { + "cell_type": "markdown", + "id": "e2e14988-630f-45e5-af81-25b024a06784", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "多步预测:604之后预测很快衰减到常数,主要由于误差的累计" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "c86678ae-6cdc-4642-bb46-7562f498256d", + "metadata": { + "origin_pos": 23, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:23:54.257601\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "multistep_preds = torch.zeros(T)\n", + "multistep_preds[: n_train + tau] = x[: n_train + tau]\n", + "for i in range(n_train + tau, T):\n", + " multistep_preds[i] = net( # 当前预测基于之前的预测结果\n", + " multistep_preds[i - tau:i].reshape((1, -1)))\n", + "\n", + "d2l.plot([time, time[tau:], time[n_train + tau:]],\n", + " [x.detach().numpy(), onestep_preds.detach().numpy(),\n", + " multistep_preds[n_train + tau:].detach().numpy()], 'time',\n", + " 'x', legend=['data', '1-step preds', 'multistep preds'],\n", + " xlim=[1, 1000], figsize=(6, 3))" + ] + }, + { + "cell_type": "markdown", + "id": "cf285180-36e2-4579-bb96-b268e4d4907c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "更仔细地看一下$k$步预测" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77c62fa3-ef37-47e1-a3c2-f65cc0959150", + "metadata": { + "origin_pos": 28, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "max_steps = 64\n", + "\n", + "features = torch.zeros((T - tau - max_steps + 1, tau + max_steps))\n", + "# 列i(i=tau)是来自(i-tau+1)步的预测,其时间步从(i+1)到(i+T-tau-max_steps+1)\n", + "for i in range(tau, tau + max_steps):\n", + " features[:, i] = net(features[:, i - tau:i]).reshape(-1)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "3d25e582-80f2-4b88-824c-92a3175693b6", + "metadata": { + "origin_pos": 28, + "slideshow": { + "slide_type": "slide" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:23:54.542829\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "steps = (1, 4, 16, 64)\n", + "d2l.plot([time[tau + i - 1: T - max_steps + i] for i in steps],\n", + " [features[:, (tau + i - 1)].detach().numpy() for i in steps], 'time', 'x',\n", + " legend=[f'{i}-step preds' for i in steps], xlim=[5, 1000],\n", + " figsize=(6, 3))" + ] + }, + { + "cell_type": "markdown", + "id": "0b70f006-4f0b-44cf-960d-19c2eeca5b4e", + "metadata": { + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 文本预处理\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "ee92af9b-d39a-4036-a6ff-38601e7f02b4", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import collections\n", + "import re\n", + "from d2l import torch as d2l" + ] + }, + { + "cell_type": "markdown", + "id": "f84c0c18-28d3-4bdc-8467-f82f23803a39", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "将数据集读取到由多条文本行组成的列表中" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "e38932bf-39bd-4b43-8feb-78a3d5e90bdf", + "metadata": { + "origin_pos": 5, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# 文本总行数: 3221\n", + "the time machine by h g wells\n", + "twinkled and his usually pale face was flushed and animated the\n" + ] + } + ], + "source": [ + "d2l.DATA_HUB['time_machine'] = (d2l.DATA_URL + 'timemachine.txt',\n", + " '090b5e7e70c295757f55df93cb0a180b9691891a')\n", + "\n", + "def read_time_machine(): \n", + " \"\"\"将时间机器数据集加载到文本行的列表中\"\"\"\n", + " with open(d2l.download('time_machine'), 'r') as f:\n", + " lines = f.readlines()\n", + " return [re.sub('[^A-Za-z]+', ' ', line).strip().lower() for line in lines]\n", + "\n", + "lines = read_time_machine()\n", + "print(f'\n", + "print(lines[0])\n", + "print(lines[10])" + ] + }, + { + "cell_type": "markdown", + "id": "8f7891ef-5a67-4407-ab73-711901c9360a", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "文本序列拆分成词元列表:词元(token)是文本的基本单位" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "120b451f-f16a-4bd7-b105-7b88de2a5e2a", + "metadata": { + "origin_pos": 7, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['the', 'time', 'machine', 'by', 'h', 'g', 'wells']\n", + "[]\n", + "[]\n", + "[]\n", + "[]\n", + "['i']\n", + "[]\n", + "[]\n", + "['the', 'time', 'traveller', 'for', 'so', 'it', 'will', 'be', 'convenient', 'to', 'speak', 'of', 'him']\n", + "['was', 'expounding', 'a', 'recondite', 'matter', 'to', 'us', 'his', 'grey', 'eyes', 'shone', 'and']\n", + "['twinkled', 'and', 'his', 'usually', 'pale', 'face', 'was', 'flushed', 'and', 'animated', 'the']\n" + ] + } + ], + "source": [ + "def tokenize(lines, token='word'): \n", + " \"\"\"将文本行拆分为单词或字符词元\"\"\"\n", + " if token == 'word':\n", + " return [line.split() for line in lines]\n", + " elif token == 'char':\n", + " return [list(line) for line in lines]\n", + " else:\n", + " print('错误:未知词元类型:' + token)\n", + "\n", + "tokens = tokenize(lines)\n", + "for i in range(11):\n", + " print(tokens[i])" + ] + }, + { + "cell_type": "markdown", + "id": "5b31663a-577a-4565-9a32-88476c497da5", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "构建一个字典,通常也叫做*词表*(vocabulary),\n", + "用来将字符串类型的词元映射到从$0$开始的数字索引中" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99739e5c-8836-4707-94c3-acea573eb39d", + "metadata": { + "origin_pos": 9, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "class Vocab: \n", + " \"\"\"文本词表\"\"\"\n", + " def __init__(self, tokens=None, min_freq=0, reserved_tokens=None):\n", + " if tokens is None:\n", + " tokens = []\n", + " if reserved_tokens is None:\n", + " reserved_tokens = []\n", + " # 按出现频率排序\n", + " counter = count_corpus(tokens) # 下页\n", + " self._token_freqs = sorted(counter.items(), key=lambda x: x[1],\n", + " reverse=True)\n", + " # 未知词元的索引为0\n", + " self.idx_to_token = [''] + reserved_tokens\n", + " self.token_to_idx = {token: idx\n", + " for idx, token in enumerate(self.idx_to_token)}\n", + " for token, freq in self._token_freqs:\n", + " if freq < min_freq:\n", + " break\n", + " if token not in self.token_to_idx:\n", + " self.idx_to_token.append(token)\n", + " self.token_to_idx[token] = len(self.idx_to_token) - 1\n", + "\n", + " def __len__(self):\n", + " return len(self.idx_to_token)\n", + "\n", + " def __getitem__(self, tokens):\n", + " if not isinstance(tokens, (list, tuple)):\n", + " return self.token_to_idx.get(tokens, self.unk)\n", + " return [self.__getitem__(token) for token in tokens]\n", + "\n", + " def to_tokens(self, indices):\n", + " if not isinstance(indices, (list, tuple)):\n", + " return self.idx_to_token[indices]\n", + " return [self.idx_to_token[index] for index in indices]\n", + "\n", + " @property\n", + " def unk(self): # 未知词元的索引为0\n", + " return 0\n", + "\n", + " @property\n", + " def token_freqs(self):\n", + " return self._token_freqs" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "489af471-21c7-4d0d-a44b-eb57c0c94a80", + "metadata": { + "origin_pos": 9, + "slideshow": { + "slide_type": "slide" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def count_corpus(tokens): \n", + " \"\"\"统计词元的频率\"\"\"\n", + " # 这里的tokens是1D列表或2D列表\n", + " if len(tokens) == 0 or isinstance(tokens[0], list):\n", + " # 将词元列表展平成一个列表\n", + " tokens = [token for line in tokens for token in line]\n", + " return collections.Counter(tokens)" + ] + }, + { + "cell_type": "markdown", + "id": "fc7064d3-a223-412d-9760-60f325d19841", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "构建词表" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "23e10840-f076-4a54-a35f-519c383cfabc", + "metadata": { + "origin_pos": 11, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[('', 0), ('the', 1), ('i', 2), ('and', 3), ('of', 4), ('a', 5), ('to', 6), ('was', 7), ('in', 8), ('that', 9)]\n" + ] + } + ], + "source": [ + "vocab = Vocab(tokens)\n", + "print(list(vocab.token_to_idx.items())[:10])" + ] + }, + { + "cell_type": "markdown", + "id": "761c8b7c-5a1e-4abd-8017-13076c0e961c", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "将每一条文本行转换成一个数字索引列表" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "57438103-9e5f-4209-b150-4879cbe3448d", + "metadata": { + "origin_pos": 13, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "文本: ['the', 'time', 'machine', 'by', 'h', 'g', 'wells']\n", + "索引: [1, 19, 50, 40, 2183, 2184, 400]\n", + "文本: ['twinkled', 'and', 'his', 'usually', 'pale', 'face', 'was', 'flushed', 'and', 'animated', 'the']\n", + "索引: [2186, 3, 25, 1044, 362, 113, 7, 1421, 3, 1045, 1]\n" + ] + } + ], + "source": [ + "for i in [0, 10]:\n", + " print('文本:', tokens[i])\n", + " print('索引:', vocab[tokens[i]])" + ] + }, + { + "cell_type": "markdown", + "id": "46501e26-7258-42cc-ae51-f650ede4b7cc", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "将所有功能打包到`load_corpus_time_machine`函数中" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "1c8e17e7-4274-42bd-a92f-9d1d50d131c9", + "metadata": { + "origin_pos": 15, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(170580, 28)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def load_corpus_time_machine(max_tokens=-1): \n", + " \"\"\"返回时光机器数据集的词元索引列表和词表\"\"\"\n", + " lines = read_time_machine()\n", + " # 使用字符(而不是单词)实现文本词元化\n", + " tokens = tokenize(lines, 'char')\n", + " vocab = Vocab(tokens)\n", + " # 数据集中每个文本行不一定是一个句子或一个段落,所以将所有文本行展平到一个列表中\n", + " corpus = [vocab[token] for line in tokens for token in line]\n", + " if max_tokens > 0:\n", + " corpus = corpus[:max_tokens]\n", + " return corpus, vocab\n", + "\n", + "corpus, vocab = load_corpus_time_machine()\n", + "len(corpus), len(vocab)" + ] + }, + { + "cell_type": "markdown", + "id": "875ad793-5aa4-45a7-bae9-c05fbe802dd6", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 语言模型和数据集\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "eaac119f-5878-469a-8563-7ff348c46d98", + "metadata": { + "origin_pos": 4, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[('the', 2261),\n", + " ('i', 1267),\n", + " ('and', 1245),\n", + " ('of', 1155),\n", + " ('a', 816),\n", + " ('to', 695),\n", + " ('was', 552),\n", + " ('in', 541),\n", + " ('that', 443),\n", + " ('my', 440)]" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import random\n", + "import torch\n", + "from d2l import torch as d2l\n", + "\n", + "tokens = d2l.tokenize(d2l.read_time_machine())\n", + "# 每个文本行不一定是一个句子或一个段落,因此把所有文本行拼接到一起\n", + "corpus = [token for line in tokens for token in line]\n", + "vocab = d2l.Vocab(corpus)\n", + "vocab.token_freqs[:10]" + ] + }, + { + "cell_type": "markdown", + "id": "6012d46b-3d2d-47f7-9baf-5f708fdaa720", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "频率高且无实意的词被称为*停用词*。画出的词频图:词频快速衰减" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "067d085c-c043-40e2-a4cb-628f58d9f5d2", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:26:58.100683\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "freqs = [freq for token, freq in vocab.token_freqs]\n", + "d2l.plot(freqs, xlabel='token: x', ylabel='frequency: n(x)',\n", + " xscale='log', yscale='log')" + ] + }, + { + "cell_type": "markdown", + "id": "0345b238-914f-49e0-ba2c-603fb149e5cd", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "其他的词元组合,比如二元语法、三元语法等等,又会如何呢?" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "67ba9fd3-dd1d-46da-9110-ddb151311043", + "metadata": { + "origin_pos": 8, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[(('of', 'the'), 309),\n", + " (('in', 'the'), 169),\n", + " (('i', 'had'), 130),\n", + " (('i', 'was'), 112),\n", + " (('and', 'the'), 109),\n", + " (('the', 'time'), 102),\n", + " (('it', 'was'), 99),\n", + " (('to', 'the'), 85),\n", + " (('as', 'i'), 78),\n", + " (('of', 'a'), 73)]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bigram_tokens = [pair for pair in zip(corpus[:-1], corpus[1:])]\n", + "bigram_vocab = d2l.Vocab(bigram_tokens)\n", + "bigram_vocab.token_freqs[:10]" + ] + }, + { + "cell_type": "markdown", + "id": "95568f8e-458c-47f5-984a-2284cb3fcdc4", + "metadata": {}, + "source": [ + "注意:在十个最频繁的词对中,有九个是由两个停用词组成的" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "43e05f91-63eb-4bd9-9fbd-ac46231195a8", + "metadata": { + "origin_pos": 10, + "slideshow": { + "slide_type": "slide" + }, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[(('the', 'time', 'traveller'), 59),\n", + " (('the', 'time', 'machine'), 30),\n", + " (('the', 'medical', 'man'), 24),\n", + " (('it', 'seemed', 'to'), 16),\n", + " (('it', 'was', 'a'), 15),\n", + " (('here', 'and', 'there'), 15),\n", + " (('seemed', 'to', 'me'), 14),\n", + " (('i', 'did', 'not'), 14),\n", + " (('i', 'saw', 'the'), 13),\n", + " (('i', 'began', 'to'), 13)]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trigram_tokens = [triple for triple in zip(\n", + " corpus[:-2], corpus[1:-1], corpus[2:])]\n", + "trigram_vocab = d2l.Vocab(trigram_tokens)\n", + "trigram_vocab.token_freqs[:10]" + ] + }, + { + "cell_type": "markdown", + "id": "9b386e37-25e1-4fd0-8215-a2f0eb9fab76", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "对比结论:单词序列都遵循齐普夫定律;N元组的数量不大,说明语言中存在相当多的结构;很多N元组很少出现,使得拉普拉斯平滑非常不适合语言建模" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "bca0974a-7ec5-4cc8-8d21-9ed94da8191f", + "metadata": { + "origin_pos": 12, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:26:59.021245\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "bigram_freqs = [freq for token, freq in bigram_vocab.token_freqs]\n", + "trigram_freqs = [freq for token, freq in trigram_vocab.token_freqs]\n", + "d2l.plot([freqs, bigram_freqs, trigram_freqs], xlabel='token: x',\n", + " ylabel='frequency: n(x)', xscale='log', yscale='log',\n", + " legend=['unigram', 'bigram', 'trigram'])" + ] + }, + { + "cell_type": "markdown", + "id": "07fc517f-e835-427b-9b07-572cda271200", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "随机生成小批量数据的特征、标签以供读取。随机采样子序列" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "2b9ff3d9-5e25-4c7a-8c78-baa1841cd1ea", + "metadata": { + "origin_pos": 14, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def seq_data_iter_random(corpus, batch_size, num_steps): \n", + " \"\"\"使用随机抽样生成一个小批量子序列\"\"\"\n", + " # 从随机偏移量开始对序列进行分区,随机范围包括num_steps-1\n", + " corpus = corpus[random.randint(0, num_steps - 1):]\n", + " # 减去1,是因为我们需要考虑标签(即当前预测)\n", + " num_subseqs = (len(corpus) - 1) // num_steps\n", + " # 长度为num_steps的子序列的起始索引\n", + " initial_indices = list(range(0, num_subseqs * num_steps, num_steps))\n", + " # 随机抽样中,来自两个相邻的、随机的、小批量中的子序列不一定在原始序列上相邻\n", + " random.shuffle(initial_indices)\n", + "\n", + " def data(pos):\n", + " \"\"\"返回从pos位置开始的长度为num_steps的序列\"\"\"\n", + " return corpus[pos: pos + num_steps]\n", + "\n", + " num_batches = num_subseqs // batch_size\n", + " for i in range(0, batch_size * num_batches, batch_size):\n", + " # 在这里,initial_indices包含子序列的随机起始索引\n", + " initial_indices_per_batch = initial_indices[i: i + batch_size]\n", + " X = [data(j) for j in initial_indices_per_batch]\n", + " Y = [data(j + 1) for j in initial_indices_per_batch]\n", + " yield torch.tensor(X), torch.tensor(Y) # 迭代器" + ] + }, + { + "cell_type": "markdown", + "id": "b58974a9-6884-4c6a-921e-74d46c3c17cd", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "生成一个从$0$到$34$的序列" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "b872fee0-a08f-4d56-ae6d-c232706dd0d2", + "metadata": { + "origin_pos": 16, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X: tensor([[29, 30, 31, 32, 33],\n", + " [ 4, 5, 6, 7, 8]]) \n", + "Y: tensor([[30, 31, 32, 33, 34],\n", + " [ 5, 6, 7, 8, 9]])\n", + "X: tensor([[ 9, 10, 11, 12, 13],\n", + " [14, 15, 16, 17, 18]]) \n", + "Y: tensor([[10, 11, 12, 13, 14],\n", + " [15, 16, 17, 18, 19]])\n", + "X: tensor([[24, 25, 26, 27, 28],\n", + " [19, 20, 21, 22, 23]]) \n", + "Y: tensor([[25, 26, 27, 28, 29],\n", + " [20, 21, 22, 23, 24]])\n" + ] + } + ], + "source": [ + "my_seq = list(range(35)) # 每组10个,最多采样3组\n", + "for X, Y in seq_data_iter_random(my_seq, batch_size=2, num_steps=5):\n", + " print('X: ', X, '\\nY:', Y)" + ] + }, + { + "cell_type": "markdown", + "id": "30f05cda-af02-4177-b810-596e157dbb2a", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "保证两个相邻的小批量中的子序列在原始序列上也是相邻的" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "9af5e40f-5a79-402c-a810-4b5c0d738b60", + "metadata": { + "origin_pos": 18, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def seq_data_iter_sequential(corpus, batch_size, num_steps): \n", + " \"\"\"使用顺序分区生成一个小批量子序列\"\"\"\n", + " # 从随机偏移量开始划分序列\n", + " offset = random.randint(0, num_steps)\n", + " num_tokens = ((len(corpus) - offset - 1) // batch_size) * batch_size\n", + " Xs = torch.tensor(corpus[offset: offset + num_tokens])\n", + " Ys = torch.tensor(corpus[offset + 1: offset + 1 + num_tokens])\n", + " Xs, Ys = Xs.reshape(batch_size, -1), Ys.reshape(batch_size, -1)\n", + " num_batches = Xs.shape[1] // num_steps\n", + " for i in range(0, num_steps * num_batches, num_steps):\n", + " X = Xs[:, i: i + num_steps]\n", + " Y = Ys[:, i: i + num_steps]\n", + " yield X, Y # 迭代器" + ] + }, + { + "cell_type": "markdown", + "id": "99250fbf-29b1-4f69-86cb-d991e0628808", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "读取每个小批量的子序列的特征`X`和标签`Y`" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "09c1d8ba-5770-4529-a0aa-8141e69d440e", + "metadata": { + "origin_pos": 21, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X: tensor([[ 3, 4, 5, 6, 7],\n", + " [18, 19, 20, 21, 22]]) \n", + "Y: tensor([[ 4, 5, 6, 7, 8],\n", + " [19, 20, 21, 22, 23]])\n", + "X: tensor([[ 8, 9, 10, 11, 12],\n", + " [23, 24, 25, 26, 27]]) \n", + "Y: tensor([[ 9, 10, 11, 12, 13],\n", + " [24, 25, 26, 27, 28]])\n", + "X: tensor([[13, 14, 15, 16, 17],\n", + " [28, 29, 30, 31, 32]]) \n", + "Y: tensor([[14, 15, 16, 17, 18],\n", + " [29, 30, 31, 32, 33]])\n" + ] + } + ], + "source": [ + "for X, Y in seq_data_iter_sequential(my_seq, batch_size=2, num_steps=5):\n", + " print('X: ', X, '\\nY:', Y)" + ] + }, + { + "cell_type": "markdown", + "id": "6fd09b29-9f9a-431c-9eca-98a67bfa0f3d", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "将上面的两个采样函数包装到一个类中" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "2171a4b7-e026-4087-adfd-9b1883f0c2f9", + "metadata": { + "origin_pos": 23, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "class SeqDataLoader: \n", + " \"\"\"加载序列数据的迭代器\"\"\"\n", + " def __init__(self, batch_size, num_steps, use_random_iter, max_tokens):\n", + " if use_random_iter:\n", + " self.data_iter_fn = d2l.seq_data_iter_random\n", + " else:\n", + " self.data_iter_fn = d2l.seq_data_iter_sequential\n", + " self.corpus, self.vocab = d2l.load_corpus_time_machine(max_tokens)\n", + " self.batch_size, self.num_steps = batch_size, num_steps\n", + "\n", + " def __iter__(self):\n", + " return self.data_iter_fn(self.corpus, self.batch_size, self.num_steps)" + ] + }, + { + "cell_type": "markdown", + "id": "cf929e6c-8b4c-48c6-a66d-058d653d2cb9", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "最后,我们定义了一个函数`load_data_time_machine`,\n", + "它同时返回数据迭代器和词表" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "2348d264-aa8b-42d8-a9f9-e1370fdebb78", + "metadata": { + "origin_pos": 25, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def load_data_time_machine(batch_size, num_steps, \n", + " use_random_iter=False, max_tokens=10000):\n", + " \"\"\"返回时光机器数据集的迭代器和词表\"\"\"\n", + " data_iter = SeqDataLoader(\n", + " batch_size, num_steps, use_random_iter, max_tokens)\n", + " return data_iter, data_iter.vocab" + ] + }, + { + "cell_type": "markdown", + "id": "d1cb218c-7cf3-44c5-9bda-eae9a8a9ecc5", + "metadata": { + "slideshow": { + "slide_type": "-" + }, + "tags": [] + }, + "source": [ + "## 循环神经网络从零开始实现\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "1a5c2509-2aac-4bc5-b1d6-c364e0319cf3", + "metadata": { + "origin_pos": 4, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import math\n", + "import torch\n", + "from torch import nn\n", + "from torch.nn import functional as F\n", + "from d2l import torch as d2l\n", + "\n", + "batch_size, num_steps = 32, 35\n", + "train_iter, vocab = d2l.load_data_time_machine(batch_size, num_steps)" + ] + }, + { + "cell_type": "markdown", + "id": "9443aef4-43ce-416f-8d4c-d5f917b82adb", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "独热编码:将词元编码为相互独立的单位向量" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "889aa7a8-59d4-4026-8a58-e2587e26be4b", + "metadata": { + "origin_pos": 8, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0],\n", + " [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0]])" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "F.one_hot(torch.tensor([0, 2]), len(vocab)) # 28个词元" + ] + }, + { + "cell_type": "markdown", + "id": "ea841674-0f3c-4d83-9e45-a2b48b7c90be", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "小批量数据形状是二维张量:\n", + "(批量大小,时间步数)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "030e1183-8e22-4157-a96d-fc06b53cd1f8", + "metadata": { + "origin_pos": 12, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([5, 2, 28])" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.arange(10).reshape((2, 5))\n", + "F.one_hot(X.T, 28).shape # (时间步数,批量大小,词表大小)" + ] + }, + { + "cell_type": "markdown", + "id": "ac0e9423-2a31-4ee4-bd29-f532d847fd9c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "初始化循环神经网络模型的模型参数" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "04638c3d-81a1-4dd0-b9a5-4336024f1518", + "metadata": { + "origin_pos": 16, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def get_params(vocab_size, num_hiddens, device):\n", + " num_inputs = num_outputs = vocab_size\n", + "\n", + " def normal(shape):\n", + " return torch.randn(size=shape, device=device) * 0.01\n", + "\n", + " # 隐藏层参数\n", + " W_xh = normal((num_inputs, num_hiddens))\n", + " W_hh = normal((num_hiddens, num_hiddens))\n", + " b_h = torch.zeros(num_hiddens, device=device)\n", + " # 输出层参数\n", + " W_hq = normal((num_hiddens, num_outputs))\n", + " b_q = torch.zeros(num_outputs, device=device)\n", + " params = [W_xh, W_hh, b_h, W_hq, b_q]\n", + " for param in params:\n", + " param.requires_grad_(True)\n", + " return params" + ] + }, + { + "cell_type": "markdown", + "id": "752a4d90-10e1-4eb9-adea-c57fedcb3215", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "一个`init_rnn_state`函数在初始化时返回隐状态" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "48ec7510-6392-4945-9474-ac78d6126caf", + "metadata": { + "origin_pos": 20, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def init_rnn_state(batch_size, num_hiddens, device):\n", + " # (批量大小,隐藏单元数)\n", + " return (torch.zeros((batch_size, num_hiddens), device=device), )" + ] + }, + { + "cell_type": "markdown", + "id": "75c17104-143e-44bb-9421-c5532863a935", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "下面的`rnn`函数定义了如何在一个时间步内计算隐状态和输出" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "6804ce54-cf84-43aa-a9df-a428c4e42288", + "metadata": { + "origin_pos": 24, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def rnn(inputs, state, params):\n", + " # inputs的形状:(时间步数量,批量大小,词表大小)\n", + " W_xh, W_hh, b_h, W_hq, b_q = params\n", + " H, = state\n", + " outputs = []\n", + " # X的形状:(批量大小,词表大小)\n", + " for X in inputs:\n", + " # 当元素在实数上均匀分布时,函数tanh的平均值为0\n", + " H = torch.tanh(torch.mm(X, W_xh) + torch.mm(H, W_hh) + b_h)\n", + " Y = torch.mm(H, W_hq) + b_q\n", + " outputs.append(Y)\n", + " return torch.cat(outputs, dim=0), (H,)" + ] + }, + { + "cell_type": "markdown", + "id": "5a77be26-fef3-49d3-b762-9517698d8ff7", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "创建一个类来包装这些函数" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a190ae84-bd64-42e3-87ef-df459859710d", + "metadata": { + "origin_pos": 28, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "class RNNModelScratch: \n", + " \"\"\"从零开始实现的循环神经网络模型\"\"\"\n", + " def __init__(self, vocab_size, num_hiddens, device,\n", + " get_params, init_state, forward_fn):\n", + " self.vocab_size, self.num_hiddens = vocab_size, num_hiddens\n", + " self.params = get_params(vocab_size, num_hiddens, device)\n", + " self.init_state, self.forward_fn = init_state, forward_fn\n", + "\n", + " def __call__(self, X, state):\n", + " X = F.one_hot(X.T, self.vocab_size).type(torch.float32)\n", + " return self.forward_fn(X, state, self.params)\n", + "\n", + " def begin_state(self, batch_size, device):\n", + " return self.init_state(batch_size, self.num_hiddens, device)" + ] + }, + { + "cell_type": "markdown", + "id": "73068fe7-dd1f-4046-9975-16069df2c693", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "检查输出是否具有正确的形状" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "36215e45-3840-49f7-ab6e-2176fbdbdfc2", + "metadata": { + "origin_pos": 32, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(torch.Size([10, 28]), 1, torch.Size([2, 512]))" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "num_hiddens = 512\n", + "net = RNNModelScratch(len(vocab), num_hiddens, d2l.try_gpu(), get_params,\n", + " init_rnn_state, rnn)\n", + "state = net.begin_state(X.shape[0], d2l.try_gpu())\n", + "Y, new_state = net(X.to(d2l.try_gpu()), state)\n", + "# 输出:(时间步数x批量大小,词表大小)\n", + "# 隐状态:(批量大小,隐藏单元数)\n", + "Y.shape, len(new_state), new_state[0].shape" + ] + }, + { + "cell_type": "markdown", + "id": "901c45ab-e314-4e4a-b23d-50831246dfe1", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "预测函数:生成`prefix`之后的新字符。注意:训练样本仅是`prefix`,远不能达到优化目的" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "0eaccd55-08c6-4ffa-88a9-f1424aedb9a3", + "metadata": { + "origin_pos": 39, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'time traveller lfy lfy lf'" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def predict_ch8(prefix, num_preds, net, vocab, device): \n", + " \"\"\"在prefix后面生成新字符\"\"\"\n", + " state = net.begin_state(batch_size=1, device=device)\n", + " outputs = [vocab[prefix[0]]]\n", + " get_input = lambda: torch.tensor([outputs[-1]], device=device).reshape((1, 1))\n", + " for y in prefix[1:]: # 预热期:不预测,只更新隐变量参数\n", + " _, state = net(get_input(), state)\n", + " outputs.append(vocab[y])\n", + " for _ in range(num_preds): # 预测num_preds步\n", + " y, state = net(get_input(), state)\n", + " outputs.append(int(y.argmax(dim=1).reshape(1)))\n", + " return ''.join([vocab.idx_to_token[i] for i in outputs])\n", + "\n", + "# 还未训练网络:预测结果相对随机\n", + "predict_ch8('time traveller ', 10, net, vocab, d2l.try_gpu())" + ] + }, + { + "cell_type": "markdown", + "id": "f30cd458-ea9e-49b8-8a70-07f09fe86232", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "梯度裁剪\n", + "$$\\mathbf{g} \\leftarrow \\min\\left(1, \\frac{\\theta}{\\|\\mathbf{g}\\|}\\right) \\mathbf{g}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "203b3b5d-6cf8-4790-b191-81707cc48462", + "metadata": { + "origin_pos": 43, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def grad_clipping(net, theta): \n", + " \"\"\"裁剪梯度\"\"\"\n", + " if isinstance(net, nn.Module):\n", + " params = [p for p in net.parameters() if p.requires_grad]\n", + " else:\n", + " params = net.params\n", + " norm = torch.sqrt(sum(torch.sum((p.grad ** 2)) for p in params))\n", + " if norm > theta:\n", + " for param in params:\n", + " param.grad[:] *= theta / norm" + ] + }, + { + "cell_type": "markdown", + "id": "b316418d-61db-4e56-befa-6b222d737868", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "定义一个函数在一个迭代周期内训练模型" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "9bf19e02-bfa0-49ab-930b-0b91eb71c00a", + "metadata": { + "origin_pos": 47, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def train_epoch_ch8(net, train_iter, loss, updater, device, use_random_iter):\n", + " \"\"\"训练网络一个迭代周期(定义见第8章)\"\"\"\n", + " state, timer = None, d2l.Timer()\n", + " metric = d2l.Accumulator(2) # 训练损失之和,词元数量\n", + " for X, Y in train_iter:\n", + " if state is None or use_random_iter:\n", + " # 在第一次迭代或使用随机抽样时初始化state\n", + " state = net.begin_state(batch_size=X.shape[0], device=device)\n", + " else:\n", + " if isinstance(net, nn.Module) and not isinstance(state, tuple):\n", + " state.detach_() # state对于nn.GRU是单个张量\n", + " else: # state对于nn.LSTM或对于从零开始实现的模型是张量元组\n", + " for s in state:\n", + " s.detach_()\n", + " y = Y.T.reshape(-1)\n", + " X, y = X.to(device), y.to(device)\n", + " y_hat, state = net(X, state)\n", + " l = loss(y_hat, y.long()).mean()\n", + " if isinstance(updater, torch.optim.Optimizer):\n", + " updater.zero_grad()\n", + " l.backward()\n", + " grad_clipping(net, 1)\n", + " updater.step()\n", + " else:\n", + " l.backward()\n", + " grad_clipping(net, 1)\n", + " updater(batch_size=1) # 已经调用了mean函数\n", + " metric.add(l * y.numel(), y.numel())\n", + " return math.exp(metric[0] / metric[1]), metric[1] / timer.stop()" + ] + }, + { + "cell_type": "markdown", + "id": "675d99cd-4f74-4245-b244-950177e64c68", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "循环神经网络模型的训练函数既支持从零开始实现,\n", + "也可以使用高级API来实现" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "98ee19b1-3584-4617-b78c-f9c24e641f62", + "metadata": { + "origin_pos": 51, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "def train_ch8(net, train_iter, vocab, lr, num_epochs, device,\n", + " use_random_iter=False):\n", + " \"\"\"训练模型(定义见第8章)\"\"\"\n", + " loss = nn.CrossEntropyLoss()\n", + " animator = d2l.Animator(xlabel='epoch', ylabel='perplexity',\n", + " legend=['train'], xlim=[10, num_epochs])\n", + " if isinstance(net, nn.Module):\n", + " updater = torch.optim.SGD(net.parameters(), lr)\n", + " else:\n", + " updater = lambda batch_size: d2l.sgd(net.params, lr, batch_size)\n", + " predict = lambda prefix: predict_ch8(prefix, 50, net, vocab, device)\n", + " for epoch in range(num_epochs):\n", + " ppl, speed = train_epoch_ch8(\n", + " net, train_iter, loss, updater, device, use_random_iter)\n", + " if (epoch + 1) % 10 == 0:\n", + " print(predict('time traveller'))\n", + " animator.add(epoch + 1, [ppl])\n", + " print(f'困惑度 {ppl:.1f}, {speed:.1f} 词元/秒 {str(device)}')\n", + " print(predict('time traveller'))\n", + " print(predict('traveller'))" + ] + }, + { + "cell_type": "markdown", + "id": "eedbcfc6-3622-44b5-9702-ac0143ea247c", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "现在,我们训练循环神经网络模型" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "116f5b2c-24cc-4042-8b72-fb5a0045a27e", + "metadata": { + "origin_pos": 54, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "困惑度 1.0, 67532.5 词元/秒 cuda:0\n", + "time traveller with a slight accession ofcheerfulness really thi\n", + "travelleryou can show black is white by argument said filby\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:26:04.396102\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "num_epochs, lr = 500, 1\n", + "train_ch8(net, train_iter, vocab, lr, num_epochs, d2l.try_gpu())" + ] + }, + { + "cell_type": "markdown", + "id": "3225dcff-1752-4dd3-9531-941bb39d1bd2", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "最后,让我们检查一下使用随机抽样方法的结果" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "b282297d-04cc-4370-a269-bb1a847d1f35", + "metadata": { + "origin_pos": 57, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "困惑度 1.4, 67608.9 词元/秒 cuda:0\n", + "time travellerit s against reason said filbywan a gurently in th\n", + "traveller after the pauserequired for the proper assimilati\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T01:27:24.880145\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "net = RNNModelScratch(len(vocab), num_hiddens, d2l.try_gpu(), get_params,\n", + " init_rnn_state, rnn)\n", + "train_ch8(net, train_iter, vocab, lr, num_epochs, d2l.try_gpu(),\n", + " use_random_iter=True)" + ] + }, + { + "cell_type": "markdown", + "id": "808d4df7-8c2e-4f5f-95fe-0bb55ce8571b", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "## 循环神经网络的简洁实现\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "55f4c2e3-1602-4d14-bd63-b3834142148b", + "metadata": { + "origin_pos": 2, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from torch.nn import functional as F\n", + "from d2l import torch as d2l\n", + "\n", + "batch_size, num_steps = 32, 35\n", + "train_iter, vocab = d2l.load_data_time_machine(batch_size, num_steps)" + ] + }, + { + "cell_type": "markdown", + "id": "844952af-89c0-4aaf-a4d1-3d1053c8d972", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "定义模型" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "6a69aca1-8254-4a28-bc79-949b5e023aaa", + "metadata": { + "origin_pos": 6, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "num_hiddens = 256\n", + "# 注意,rnn_layer只包含隐藏的循环层\n", + "rnn_layer = nn.RNN(len(vocab), num_hiddens)" + ] + }, + { + "cell_type": "markdown", + "id": "803cc15e-eca9-42f5-9abf-972d50b8785d", + "metadata": { + "slideshow": { + "slide_type": "-" + } + }, + "source": [ + "使用张量来初始化隐状态" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "916cee3a-4078-45fc-a5ab-0a4e32f4eb5e", + "metadata": { + "origin_pos": 11, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 32, 256])" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "state = torch.zeros((1, batch_size, num_hiddens))\n", + "state.shape # (隐藏层数,批量大小,隐藏单元数)" + ] + }, + { + "cell_type": "markdown", + "id": "f5287334-a64d-4d8f-918d-23342c037b24", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "通过一个隐状态和一个输入,我们就可以用更新后的隐状态计算输出" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "88e87cce-f70f-4539-9002-a12611749491", + "metadata": { + "origin_pos": 16, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(torch.Size([35, 32, 256]), torch.Size([1, 32, 256]))" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = torch.rand(size=(num_steps, batch_size, len(vocab)))\n", + "# “输出”(Y)不涉及输出层的计算:它是指每个时间步的隐状态,\n", + "# 这些隐状态可以用作后续输出层的输入\n", + "Y, state_new = rnn_layer(X, state)\n", + "Y.shape, state_new.shape" + ] + }, + { + "cell_type": "markdown", + "id": "d6faf827-63c5-47d5-9cb4-b0d8ab25e368", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "我们为一个完整的循环神经网络模型定义了一个`RNNModel`类" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7f70156c-bf21-4bc4-906e-ff5852669cd5", + "metadata": { + "origin_pos": 20, + "tab": [ + "pytorch" + ] + }, + "outputs": [], + "source": [ + "class RNNModel(nn.Module):\n", + " \"\"\"循环神经网络模型\"\"\"\n", + " def __init__(self, rnn_layer, vocab_size, **kwargs):\n", + " super(RNNModel, self).__init__(**kwargs)\n", + " self.rnn = rnn_layer\n", + " self.vocab_size = vocab_size\n", + " self.num_hiddens = self.rnn.hidden_size\n", + " # 如果RNN是双向的(之后将介绍),num_directions应该是2,否则应该是1\n", + " if not self.rnn.bidirectional:\n", + " self.num_directions = 1\n", + " self.linear = nn.Linear(self.num_hiddens, self.vocab_size)\n", + " else:\n", + " self.num_directions = 2\n", + " self.linear = nn.Linear(self.num_hiddens * 2, self.vocab_size)\n", + "\n", + " def forward(self, inputs, state):\n", + " X = F.one_hot(inputs.T.long(), self.vocab_size)\n", + " X = X.to(torch.float32)\n", + " Y, state = self.rnn(X, state)\n", + " # 全连接层首先将Y的形状改为(时间步数*批量大小,隐藏单元数)\n", + " # 它的输出形状是(时间步数*批量大小,词表大小)。\n", + " output = self.linear(Y.reshape((-1, Y.shape[-1])))\n", + " return output, state\n", + "\n", + " def begin_state(self, device, batch_size=1):\n", + " if not isinstance(self.rnn, nn.LSTM):\n", + " # nn.GRU以张量H作为隐状态\n", + " return torch.zeros((self.num_directions * self.rnn.num_layers,\n", + " batch_size, self.num_hiddens),\n", + " device=device)\n", + " else: # nn.LSTM以元组(H,C)作为隐状态\n", + " return (torch.zeros((\n", + " self.num_directions * self.rnn.num_layers,\n", + " batch_size, self.num_hiddens), device=device),\n", + " torch.zeros((\n", + " self.num_directions * self.rnn.num_layers,\n", + " batch_size, self.num_hiddens), device=device))" + ] + }, + { + "cell_type": "markdown", + "id": "4333f760-a48a-4038-b6ce-0a6cc408b028", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "基于随机权重初始化的模型进行预测:输出大量重复字符(类似之前人造数据`sin`函数的多步预测)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "16a236ef-aa2b-4d69-a65c-549e8dc79e96", + "metadata": { + "origin_pos": 24, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'time travellerpcpppppppp'" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "device = d2l.try_gpu()\n", + "net = RNNModel(rnn_layer, vocab_size=len(vocab))\n", + "net = net.to(device)\n", + "d2l.predict_ch8('time traveller', 10, net, vocab, device)" + ] + }, + { + "cell_type": "markdown", + "id": "833326d0-c61e-48af-b5fa-718f44e65e95", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "使用高级API训练模型" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "9b2d0ed1-0cdc-4b6b-989f-1c17e60ec2cb", + "metadata": { + "origin_pos": 28, + "tab": [ + "pytorch" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "perplexity 1.3, 293417.9 tokens/sec on cuda:0\n", + "time travellerif se gan ubsely fourth timethere is however a ten\n", + "traveller and wheth dimensisnal fis liger an andeterall hey\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " 2022-01-17T02:03:21.638811\n", + " image/svg+xml\n", + " \n", + " \n", + " Matplotlib v3.3.3, https://matplotlib.org/\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n" + ], + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "num_epochs, lr = 500, 1\n", + "d2l.train_ch8(net, train_iter, vocab, lr, num_epochs, device)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/08_recurrent/Recurrent-1.py b/08_recurrent/Recurrent-1.py new file mode 100644 index 0000000..969b0b0 --- /dev/null +++ b/08_recurrent/Recurrent-1.py @@ -0,0 +1,218 @@ +import torch +import torch.nn as nn +from torch.autograd import Variable + +class RNN(nn.Module): + """ Recurrent neural network - simple + x -> hidden -> out -> log-softmax + """ + def __init__(self, input_size, hidden_size, output_size): + super(RNN, self).__init__() + raise NotImplementedError + + def forward(self, input, hidden): + raise NotImplementedError + + def initHidden(self): + raise NotImplementedError + + +########################################################## +# Text data utilities - character-level +import glob +import unicodedata +import string + + +all_letters = string.ascii_letters + " .,;'-" # all legal letters +n_letters = len(all_letters) + +def unicodeToAscii(s): + """ Unicode string to plain ASCII, + ref. http://stackoverflow.com/a/518232/2809427 + """ + return ''.join( + c for c in unicodedata.normalize('NFD', s) + if unicodedata.category(c) != 'Mn' + and c in all_letters + ) + +def readLines(filename): + """ Read a file and split into lines """ + lines = open(filename).read().strip().split('\n') + return [unicodeToAscii(line) for line in lines] + +def build_category_lines(data_file='../data/names/*.txt'): + """ Build the category_lines dictionary, a list of lines per category """ + + def findFiles(path): return glob.glob(path) + + category_lines = {} + all_categories = [] + for filename in findFiles(data_file): + category = filename.split('/')[-1].split('.')[0] + all_categories.append(category) + lines = readLines(filename) + category_lines[category] = lines + + return category_lines, all_categories + +def letterToIndex(letter): + """ Find letter index from all_letters, e.g. "a" = 0 """ + return all_letters.find(letter) + +def lineToTensor(line): + """ Turn a line into a , + ie, an array of one-hot letter vectors + """ + tensor = torch.zeros(len(line), 1, n_letters) + for li, letter in enumerate(line): + tensor[li][0][letterToIndex(letter)] = 1 + return tensor +########################################################## + + +import random +import time +import math + + +class TextLearner: + @staticmethod + def train_epoch(model, category_tensor, line_tensor): + hidden = model.initHidden() + optimizer.zero_grad() + + for i in range(line_tensor.size()[0]): + output, hidden = model(line_tensor[i], hidden) + + loss = criterion(output, category_tensor) + loss.backward() + + optimizer.step() + + return output, loss.item() + + @staticmethod + def train(model, n_epochs, print_every, plot_every, learning_rate): + def categoryFromOutput(output): + top_n, top_i = output.data.topk(1) # Tensor out of Variable with .data + category_i = top_i[0][0] + return all_categories[category_i], category_i + + def randomTrainingPair(): + """ generate a random training pair """ + def randomChoice(l): return l[random.randint(0, len(l) - 1)] + + category = randomChoice(all_categories) + line = randomChoice(category_lines[category]) + category_tensor = Variable(torch.LongTensor([all_categories.index(category)])) + line_tensor = Variable(lineToTensor(line)) + return category, line, category_tensor, line_tensor + + def timeSince(since): + now = time.time() + s = now - since + m = math.floor(s / 60) + s -= m * 60 + return f'{m}m {int(s): >2d}s' + start = time.time() + + # Keep track of losses for plotting + current_loss = 0 + all_losses = [] + for epoch in range(1, n_epochs + 1): + category, line, category_tensor, line_tensor = randomTrainingPair() + output, loss = TextLearner.train_epoch(model, category_tensor, line_tensor) + current_loss += loss + + # Print epoch number, loss, name and guess + if epoch % print_every == 0: + guess, guess_i = categoryFromOutput(output) + correct = '✓' if guess == category else '✗ (%s)' % category + print(f'{epoch: >6d} {epoch / n_epochs * 100:5.1f}% ({timeSince(start)}) {loss:.4f} {current_loss:.2f} {line} / {guess} {correct}') + + # Add current loss avg to list of losses + if epoch % plot_every == 0: + all_losses.append(current_loss / plot_every) + current_loss = 0 + + torch.save(model, 'char_rnn_names.pt') + + @staticmethod + def predict_t(line_tensor): + """ return an output given a line """ + hidden = model.initHidden() + + for i in range(line_tensor.size()[0]): + output, hidden = model(line_tensor[i], hidden) + + return output + + @staticmethod + def predict(model, line, n_predictions=3): + output = TextLearner.predict_t(Variable(lineToTensor(line))) + + # Get top N categories + topv, topi = output.data.topk(n_predictions, 1, True) + predictions = [] + + print(f'prediction for {line}:') + for i in range(n_predictions): + value = topv[0][i] + category_index = topi[0][i] + print(f' ({value:.2f}) {all_categories[category_index]}') + predictions.append([value, all_categories[category_index]]) + + return predictions + + @staticmethod + def eval_all(model, n_predictions=5): + correct_n = [] + total_n = [] + correct_str = '' + for ci, category in enumerate(category_lines): + total_n.append(len(category)) + cni = 0 + for line in category: + output = TextLearner.predict_t(Variable(lineToTensor(line))) + _, topi = output.data.topk(n_predictions) + cc = 0 + for ii in range(n_predictions): + if (topi[0][ii] == ci): + cc = 1 + break + cni += cc + correct_n.append(cni) + correct_str += f'{all_categories[ci]}: {cni / total_n[-1] * 100:5.1f}; ' + + print(f'correct rate (per category): {correct_str}') + print(f'total correct rate: {sum(correct_n) / sum(total_n) * 100:5.1f}') + + +if __name__ == '__main__': + print(f'{n_letters} legal letters: {all_letters}') + category_lines, all_categories = build_category_lines() + n_categories = len(all_categories) + print(f'{n_categories} categories: {all_categories}') + print(f"first 5 Chinese names: {category_lines['Chinese'][:5]}") + + n_hidden = 128 + n_epochs = 100000 + print_every = 5000 + plot_every = 1000 + learning_rate = 0.005 + + model = RNN(n_letters, n_hidden, n_categories) + print(model) + + optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) + criterion = nn.NLLLoss() # negative log likelihood loss + TextLearner.train(model, n_epochs, print_every, plot_every, learning_rate) + + model = torch.load('char_rnn_names.pt') + TextLearner.predict(model, 'Wu') + TextLearner.predict(model, 'Harry') + TextLearner.predict(model, 'Louis') + + TextLearner.eval_all(model) diff --git a/08_recurrent/Recurrent-2.py b/08_recurrent/Recurrent-2.py new file mode 100644 index 0000000..588dcbc --- /dev/null +++ b/08_recurrent/Recurrent-2.py @@ -0,0 +1,215 @@ +import torch +import torch.nn as nn +from torch.autograd import Variable + +class RNN(nn.Module): + """ Recurrent neural network - simple + x -> hidden -> out -> log-softmax + """ + def __init__(self, input_size, hidden_size, output_size): + super(RNN, self).__init__() + raise NotImplementedError + + def forward(self, input, hidden): + raise NotImplementedError + + def initHidden(self): + raise NotImplementedError + + +########################################################## +# Text data utilities - character-level +import glob +import unicodedata +import string + + +all_letters = string.ascii_letters + " .,;'-" # all legal letters +n_letters = len(all_letters) + +def unicodeToAscii(s): + """ Unicode string to plain ASCII, + ref. http://stackoverflow.com/a/518232/2809427 + """ + return ''.join( + c for c in unicodedata.normalize('NFD', s) + if unicodedata.category(c) != 'Mn' + and c in all_letters + ) + +def readLines(filename): + """ Read a file and split into lines """ + lines = open(filename).read().strip().split('\n') + return [unicodeToAscii(line) for line in lines] + +def build_category_lines(data_file='../data/names/*.txt'): + """ Build the category_lines dictionary, a list of lines per category """ + + def findFiles(path): return glob.glob(path) + + category_lines = {} + all_categories = [] + for filename in findFiles(data_file): + category = filename.split('/')[-1].split('.')[0] + all_categories.append(category) + lines = readLines(filename) + category_lines[category] = lines + + return category_lines, all_categories + +def letterToIndex(letter): + """ Find letter index from all_letters, e.g. "a" = 0 """ + raise NotImplementedError + +def lineToTensor(line): + """ Turn a line into a , + ie, an array of one-hot letter vectors + """ + raise NotImplementedError +########################################################## + + +import random +import time +import math + + +class TextLearner: + @staticmethod + def train_epoch(model, category_tensor, line_tensor): + hidden = model.initHidden() + optimizer.zero_grad() + + for i in range(line_tensor.size()[0]): + output, hidden = model(line_tensor[i], hidden) + + loss = criterion(output, category_tensor) + loss.backward() + + optimizer.step() + + return output, loss.item() + + @staticmethod + def train(model, n_epochs, print_every, plot_every, learning_rate): + def categoryFromOutput(output): + top_n, top_i = output.data.topk(1) # Tensor out of Variable with .data + category_i = top_i[0][0] + return all_categories[category_i], category_i + + def randomTrainingPair(): + """ generate a random training pair """ + def randomChoice(l): return l[random.randint(0, len(l) - 1)] + + category = randomChoice(all_categories) + line = randomChoice(category_lines[category]) + category_tensor = Variable(torch.LongTensor([all_categories.index(category)])) + line_tensor = Variable(lineToTensor(line)) + return category, line, category_tensor, line_tensor + + def timeSince(since): + now = time.time() + s = now - since + m = math.floor(s / 60) + s -= m * 60 + return f'{m}m {int(s): >2d}s' + start = time.time() + + # Keep track of losses for plotting + current_loss = 0 + all_losses = [] + for epoch in range(1, n_epochs + 1): + category, line, category_tensor, line_tensor = randomTrainingPair() + output, loss = TextLearner.train_epoch(model, category_tensor, line_tensor) + current_loss += loss + + # Print epoch number, loss, name and guess + if epoch % print_every == 0: + guess, guess_i = categoryFromOutput(output) + correct = '✓' if guess == category else '✗ (%s)' % category + print(f'{epoch: >6d} {epoch / n_epochs * 100:5.1f}% ({timeSince(start)}) {loss:.4f} {current_loss:.2f} {line} / {guess} {correct}') + + # Add current loss avg to list of losses + if epoch % plot_every == 0: + all_losses.append(current_loss / plot_every) + current_loss = 0 + + torch.save(model, 'char_rnn_names.pt') + + @staticmethod + def predict_t(line_tensor): + """ return an output given a line """ + hidden = model.initHidden() + + for i in range(line_tensor.size()[0]): + output, hidden = model(line_tensor[i], hidden) + + return output + + @staticmethod + def predict(model, line, n_predictions=3): + output = TextLearner.predict_t(Variable(lineToTensor(line))) + + # Get top N categories + topv, topi = output.data.topk(n_predictions, 1, True) + predictions = [] + + print(f'prediction for {line}:') + for i in range(n_predictions): + value = topv[0][i] + category_index = topi[0][i] + print(f' ({value:.2f}) {all_categories[category_index]}') + predictions.append([value, all_categories[category_index]]) + + return predictions + + @staticmethod + def eval_all(model, n_predictions=5): + correct_n = [] + total_n = [] + correct_str = '' + for ci, category in enumerate(category_lines): + total_n.append(len(category)) + cni = 0 + for line in category: + output = TextLearner.predict_t(Variable(lineToTensor(line))) + _, topi = output.data.topk(n_predictions) + cc = 0 + for ii in range(n_predictions): + if (topi[0][ii] == ci): + cc = 1 + break + cni += cc + correct_n.append(cni) + correct_str += f'{all_categories[ci]}: {cni / total_n[-1] * 100:5.1f}; ' + + print(f'correct rate (per category): {correct_str}') + print(f'total correct rate: {sum(correct_n) / sum(total_n) * 100:5.1f}') + + +if __name__ == '__main__': + print(f'{n_letters} legal letters: {all_letters}') + category_lines, all_categories = build_category_lines() + n_categories = len(all_categories) + print(f'{n_categories} categories: {all_categories}') + print(f"first 5 Chinese names: {category_lines['Chinese'][:5]}") + + n_hidden = 128 + n_epochs = 100000 + print_every = 5000 + plot_every = 1000 + learning_rate = 0.005 + + model = RNN(n_letters, n_hidden, n_categories) + print(model) + + optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) + criterion = nn.NLLLoss() # negative log likelihood loss + TextLearner.train(model, n_epochs, print_every, plot_every, learning_rate) + + model = torch.load('char_rnn_names.pt') + TextLearner.predict(model, 'Wu') + TextLearner.predict(model, 'Harry') + TextLearner.predict(model, 'Louis') + + TextLearner.eval_all(model) diff --git a/08_recurrent/Recurrent-3.py b/08_recurrent/Recurrent-3.py new file mode 100644 index 0000000..9066ff7 --- /dev/null +++ b/08_recurrent/Recurrent-3.py @@ -0,0 +1,199 @@ +import torch +import torch.nn as nn +from torch.autograd import Variable + +class RNN(nn.Module): + """ Recurrent neural network - simple + x -> hidden -> out -> log-softmax + """ + def __init__(self, input_size, hidden_size, output_size): + super(RNN, self).__init__() + raise NotImplementedError + + def forward(self, input, hidden): + raise NotImplementedError + + def initHidden(self): + raise NotImplementedError + + +########################################################## +# Text data utilities - character-level +import glob +import unicodedata +import string + + +all_letters = string.ascii_letters + " .,;'-" # all legal letters +n_letters = len(all_letters) + +def unicodeToAscii(s): + """ Unicode string to plain ASCII, + ref. http://stackoverflow.com/a/518232/2809427 + """ + return ''.join( + c for c in unicodedata.normalize('NFD', s) + if unicodedata.category(c) != 'Mn' + and c in all_letters + ) + +def readLines(filename): + """ Read a file and split into lines """ + lines = open(filename).read().strip().split('\n') + return [unicodeToAscii(line) for line in lines] + +def build_category_lines(data_file='../data/names/*.txt'): + """ Build the category_lines dictionary, a list of lines per category """ + + def findFiles(path): return glob.glob(path) + + category_lines = {} + all_categories = [] + for filename in findFiles(data_file): + category = filename.split('/')[-1].split('.')[0] + all_categories.append(category) + lines = readLines(filename) + category_lines[category] = lines + + return category_lines, all_categories + +def letterToIndex(letter): + """ Find letter index from all_letters, e.g. "a" = 0 """ + raise NotImplementedError + +def lineToTensor(line): + """ Turn a line into a , + ie, an array of one-hot letter vectors + """ + raise NotImplementedError +########################################################## + + +import random +import time +import math + + +class TextLearner: + @staticmethod + def train_epoch(model, category_tensor, line_tensor): + raise NotImplementedError + + @staticmethod + def train(model, n_epochs, print_every, plot_every, learning_rate): + def categoryFromOutput(output): + top_n, top_i = output.data.topk(1) # Tensor out of Variable with .data + category_i = top_i[0][0] + return all_categories[category_i], category_i + + def randomTrainingPair(): + """ generate a random training pair """ + def randomChoice(l): return l[random.randint(0, len(l) - 1)] + + category = randomChoice(all_categories) + line = randomChoice(category_lines[category]) + category_tensor = Variable(torch.LongTensor([all_categories.index(category)])) + line_tensor = Variable(lineToTensor(line)) + return category, line, category_tensor, line_tensor + + def timeSince(since): + now = time.time() + s = now - since + m = math.floor(s / 60) + s -= m * 60 + return f'{m}m {int(s): >2d}s' + start = time.time() + + # Keep track of losses for plotting + current_loss = 0 + all_losses = [] + for epoch in range(1, n_epochs + 1): + category, line, category_tensor, line_tensor = randomTrainingPair() + output, loss = TextLearner.train_epoch(model, category_tensor, line_tensor) + current_loss += loss + + # Print epoch number, loss, name and guess + if epoch % print_every == 0: + guess, guess_i = categoryFromOutput(output) + correct = '✓' if guess == category else '✗ (%s)' % category + print(f'{epoch: >6d} {epoch / n_epochs * 100:5.1f}% ({timeSince(start)}) {loss:.4f} {current_loss:.2f} {line} / {guess} {correct}') + + # Add current loss avg to list of losses + if epoch % plot_every == 0: + all_losses.append(current_loss / plot_every) + current_loss = 0 + + torch.save(model, 'char_rnn_names.pt') + + @staticmethod + def predict_t(line_tensor): + """ return an output given a line """ + raise NotImplementedError + + @staticmethod + def predict(model, line, n_predictions=3): + output = TextLearner.predict_t(Variable(lineToTensor(line))) + + # Get top N categories + topv, topi = output.data.topk(n_predictions, 1, True) + predictions = [] + + print(f'prediction for {line}:') + for i in range(n_predictions): + value = topv[0][i] + category_index = topi[0][i] + print(f' ({value:.2f}) {all_categories[category_index]}') + predictions.append([value, all_categories[category_index]]) + + return predictions + + @staticmethod + def eval_all(model, n_predictions=5): + correct_n = [] + total_n = [] + correct_str = '' + for ci, category in enumerate(category_lines): + total_n.append(len(category)) + cni = 0 + for line in category: + output = TextLearner.predict_t(Variable(lineToTensor(line))) + _, topi = output.data.topk(n_predictions) + cc = 0 + for ii in range(n_predictions): + if (topi[0][ii] == ci): + cc = 1 + break + cni += cc + correct_n.append(cni) + correct_str += f'{all_categories[ci]}: {cni / total_n[-1] * 100:5.1f}; ' + + print(f'correct rate (per category): {correct_str}') + print(f'total correct rate: {sum(correct_n) / sum(total_n) * 100:5.1f}') + + +if __name__ == '__main__': + print(f'{n_letters} legal letters: {all_letters}') + category_lines, all_categories = build_category_lines() + n_categories = len(all_categories) + print(f'{n_categories} categories: {all_categories}') + print(f"first 5 Chinese names: {category_lines['Chinese'][:5]}") + + n_hidden = 128 + n_epochs = 100000 + print_every = 5000 + plot_every = 1000 + learning_rate = 0.005 + + model = RNN(n_letters, n_hidden, n_categories) + print(model) + + optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) + criterion = nn.NLLLoss() # negative log likelihood loss + TextLearner.train(model, n_epochs, print_every, plot_every, learning_rate) + + model = torch.load('char_rnn_names.pt') + TextLearner.predict(model, 'Wu') + TextLearner.predict(model, 'Harry') + TextLearner.predict(model, 'Louis') + + TextLearner.eval_all(model) diff --git a/08_recurrent/char_rnn_names.pt b/08_recurrent/char_rnn_names.pt new file mode 100644 index 0000000..0da6142 Binary files /dev/null and b/08_recurrent/char_rnn_names.pt differ diff --git a/08_recurrent/expected_output.txt b/08_recurrent/expected_output.txt new file mode 100644 index 0000000..693f9bb --- /dev/null +++ b/08_recurrent/expected_output.txt @@ -0,0 +1,42 @@ +58 legal letters: abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ .,;'- +18 categories: ['Czech', 'German', 'Arabic', 'Japanese', 'Chinese', 'Vietnamese', 'Russian', 'French', 'Irish', 'English', 'Spanish', 'Greek', 'Italian', 'Portuguese', 'Scottish', 'Dutch', 'Korean', 'Polish'] +first 5 Chinese names: ['Ang', 'Au-Yong', 'Bai', 'Ban', 'Bao'] +RNN( + (i2h): Linear(in_features=186, out_features=128, bias=True) + (i2o): Linear(in_features=186, out_features=18, bias=True) + (softmax): LogSoftmax(dim=1) +) + 5000 5.0% (0m 9s) 2.5912 2541.35 Zimmerman / Dutch ✗ (German) + 10000 10.0% (0m 18s) 1.5788 2128.60 Gong / Vietnamese ✗ (Chinese) + 15000 15.0% (0m 26s) 2.7233 1998.65 Solo / Chinese ✗ (Spanish) + 20000 20.0% (0m 36s) 1.8938 1815.65 Lohrenz / Spanish ✗ (German) + 25000 25.0% (0m 44s) 2.3676 1735.06 Hoch / Vietnamese ✗ (German) + 30000 30.0% (0m 52s) 1.8657 1634.36 Macfarland / French ✗ (Irish) + 35000 35.0% (1m 4s) 1.5567 1658.71 Devin / French ✗ (Irish) + 40000 40.0% (1m 14s) 3.7105 1568.38 Farmer / French ✗ (English) + 45000 45.0% (1m 22s) 1.9464 1524.05 Arthur / Arabic ✗ (French) + 50000 50.0% (1m 32s) 2.1889 1463.57 Nemec / Portuguese ✗ (Czech) + 55000 55.0% (1m 42s) 0.6579 1500.78 Naser / Arabic ✓ + 60000 60.0% (1m 50s) 0.7514 1427.26 Gallego / Spanish ✓ + 65000 65.0% (1m 58s) 0.8220 1383.95 Kawate / Japanese ✓ + 70000 70.0% (2m 6s) 0.7611 1319.17 Graner / German ✓ + 75000 75.0% (2m 14s) 1.2482 1331.15 Guerrero / Spanish ✓ + 80000 80.0% (2m 23s) 0.0109 1306.75 Antoniadis / Greek ✓ + 85000 85.0% (2m 31s) 0.0735 1404.86 O'Boyle / Irish ✓ + 90000 90.0% (2m 39s) 0.8547 1258.60 Said / Arabic ✓ + 95000 95.0% (2m 47s) 1.2185 1329.16 Dickson / English ✗ (Scottish) +100000 100.0% (2m 55s) 1.6481 1295.43 Abana / Italian ✗ (Spanish) +prediction for Wu: + (-0.70) Korean + (-1.36) Vietnamese + (-2.53) Chinese +prediction for Harry: + (-1.27) Arabic + (-1.63) English + (-1.81) French +prediction for Louis: + (-1.16) Arabic + (-1.45) Greek + (-1.58) Portuguese +correct rate (per category): Czech: 40.0; German: 66.7; Arabic: 33.3; Japanese: 25.0; Chinese: 14.3; Vietnamese: 30.0; Russian: 0.0; French: 50.0; Irish: 20.0; English: 85.7; Spanish: 14.3; Greek: 0.0; Italian: 14.3; Portuguese: 20.0; Scottish: 25.0; Dutch: 0.0; Korean: 66.7; Polish: 0.0; +total correct rate: 28.1 \ No newline at end of file diff --git a/data/dataset4learners.ipynb b/data/dataset4learners.ipynb new file mode 100644 index 0000000..bcff3b2 --- /dev/null +++ b/data/dataset4learners.ipynb @@ -0,0 +1,1149 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d979be5c-1525-46e7-bd47-55ab3ffafcf8", + "metadata": {}, + "source": [ + "## 数据集\n", + "\n", + "对于课程教程,我们将使用一系列的数据集,以更好地展示算法的优势和劣势。这些数据集包括以下内容:\n", + "\n", + "* Fisher's Iris: 每个项目代表一朵花,有四个尺寸:萼片和花瓣的长度和宽度。每个项目/花都被归入三个物种之一。Setosa、Versicolor和Virginica。\n", + "\n", + "* Zoo: 该数据集持有不同的动物和它们的分类,如 \"哺乳动物\"、\"鱼类 \"等。我们要分类的新动物有以下测量值。1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 4, 1, 0, 1(不要关心这些测量值是什么意思)。" + ] + }, + { + "cell_type": "markdown", + "id": "de2ffd8d-d568-444a-b1a8-c31fdeeaa2b2", + "metadata": {}, + "source": [ + "### 介绍\n", + "\n", + "我们将使用的许多数据集是.csv文件(尽管也支持其他格式)。你可以在网上找到很多数据集,一个很好的数据集库是[UCI机器学习库](https://archive.ics.uci.edu/ml/datasets.html)。\n", + "\n", + "在这样的文件中,每一行都对应着一个项目/测量。一行中的每个单独的值代表一个*特征,通常还有一个值表示项目的*类别。\n", + "\n", + "你可以在这里找到该数据集的代码:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "729784a4-12df-48c0-8bab-4ffd149c5128", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "

\n", + "\n", + "
class DataSet:\n",
+       "    """\n",
+       "    A data set for a machine learning problem. It has the following fields:\n",
+       "\n",
+       "    d.examples   A list of examples. Each one is a list of attribute values.\n",
+       "    d.attrs      A list of integers to index into an example, so example[attr]\n",
+       "                 gives a value. Normally the same as range(len(d.examples[0])).\n",
+       "    d.attr_names Optional list of mnemonic names for corresponding attrs.\n",
+       "    d.target     The attribute that a learning algorithm will try to predict.\n",
+       "                 By default the final attribute.\n",
+       "    d.inputs     The list of attrs without the target.\n",
+       "    d.values     A list of lists: each sublist is the set of possible\n",
+       "                 values for the corresponding attribute. If initially None,\n",
+       "                 it is computed from the known examples by self.set_problem.\n",
+       "                 If not None, an erroneous value raises ValueError.\n",
+       "    d.distance   A function from a pair of examples to a non-negative number.\n",
+       "                 Should be symmetric, etc. Defaults to mean_boolean_error\n",
+       "                 since that can handle any field types.\n",
+       "    d.name       Name of the data set (for output display only).\n",
+       "    d.source     URL or other source where the data came from.\n",
+       "    d.exclude    A list of attribute indexes to exclude from d.inputs. Elements\n",
+       "                 of this list can either be integers (attrs) or attr_names.\n",
+       "\n",
+       "    Normally, you call the constructor and you're done; then you just\n",
+       "    access fields like d.examples and d.target and d.inputs.\n",
+       "    """\n",
+       "\n",
+       "    def __init__(self, examples=None, attrs=None, attr_names=None, target=-1, inputs=None,\n",
+       "                 values=None, distance=mean_boolean_error, name='', source='', exclude=()):\n",
+       "        """\n",
+       "        Accepts any of DataSet's fields. Examples can also be a\n",
+       "        string or file from which to parse examples using parse_csv.\n",
+       "        Optional parameter: exclude, as documented in .set_problem().\n",
+       "        >>> DataSet(examples='1, 2, 3')\n",
+       "        <DataSet(): 1 examples, 3 attributes>\n",
+       "        """\n",
+       "        self.name = name\n",
+       "        self.source = source\n",
+       "        self.values = values\n",
+       "        self.distance = distance\n",
+       "        self.got_values_flag = bool(values)\n",
+       "\n",
+       "        # initialize .examples from string or list or data directory\n",
+       "        if isinstance(examples, str):\n",
+       "            self.examples = parse_csv(examples)\n",
+       "        elif examples is None:\n",
+       "            self.examples = parse_csv(open_data(name + '.csv').read())\n",
+       "        else:\n",
+       "            self.examples = examples\n",
+       "\n",
+       "        # attrs are the indices of examples, unless otherwise stated.\n",
+       "        if self.examples is not None and attrs is None:\n",
+       "            attrs = list(range(len(self.examples[0])))\n",
+       "\n",
+       "        self.attrs = attrs\n",
+       "\n",
+       "        # initialize .attr_names from string, list, or by default\n",
+       "        if isinstance(attr_names, str):\n",
+       "            self.attr_names = attr_names.split()\n",
+       "        else:\n",
+       "            self.attr_names = attr_names or attrs\n",
+       "        self.set_problem(target, inputs=inputs, exclude=exclude)\n",
+       "\n",
+       "    def set_problem(self, target, inputs=None, exclude=()):\n",
+       "        """\n",
+       "        Set (or change) the target and/or inputs.\n",
+       "        This way, one DataSet can be used multiple ways. inputs, if specified,\n",
+       "        is a list of attributes, or specify exclude as a list of attributes\n",
+       "        to not use in inputs. Attributes can be -n .. n, or an attr_name.\n",
+       "        Also computes the list of possible values, if that wasn't done yet.\n",
+       "        """\n",
+       "        self.target = self.attr_num(target)\n",
+       "        exclude = list(map(self.attr_num, exclude))\n",
+       "        if inputs:\n",
+       "            self.inputs = remove_all(self.target, inputs)\n",
+       "        else:\n",
+       "            self.inputs = [a for a in self.attrs if a != self.target and a not in exclude]\n",
+       "        if not self.values:\n",
+       "            self.update_values()\n",
+       "        self.check_me()\n",
+       "\n",
+       "    def check_me(self):\n",
+       "        """Check that my fields make sense."""\n",
+       "        assert len(self.attr_names) == len(self.attrs)\n",
+       "        assert self.target in self.attrs\n",
+       "        assert self.target not in self.inputs\n",
+       "        assert set(self.inputs).issubset(set(self.attrs))\n",
+       "        if self.got_values_flag:\n",
+       "            # only check if values are provided while initializing DataSet\n",
+       "            list(map(self.check_example, self.examples))\n",
+       "\n",
+       "    def add_example(self, example):\n",
+       "        """Add an example to the list of examples, checking it first."""\n",
+       "        self.check_example(example)\n",
+       "        self.examples.append(example)\n",
+       "\n",
+       "    def check_example(self, example):\n",
+       "        """Raise ValueError if example has any invalid values."""\n",
+       "        if self.values:\n",
+       "            for a in self.attrs:\n",
+       "                if example[a] not in self.values[a]:\n",
+       "                    raise ValueError('Bad value {} for attribute {} in {}'\n",
+       "                                     .format(example[a], self.attr_names[a], example))\n",
+       "\n",
+       "    def attr_num(self, attr):\n",
+       "        """Returns the number used for attr, which can be a name, or -n .. n-1."""\n",
+       "        if isinstance(attr, str):\n",
+       "            return self.attr_names.index(attr)\n",
+       "        elif attr < 0:\n",
+       "            return len(self.attrs) + attr\n",
+       "        else:\n",
+       "            return attr\n",
+       "\n",
+       "    def update_values(self):\n",
+       "        self.values = list(map(unique, zip(*self.examples)))\n",
+       "\n",
+       "    def sanitize(self, example):\n",
+       "        """Return a copy of example, with non-input attributes replaced by None."""\n",
+       "        return [attr_i if i in self.inputs else None for i, attr_i in enumerate(example)][:-1]\n",
+       "\n",
+       "    def classes_to_numbers(self, classes=None):\n",
+       "        """Converts class names to numbers."""\n",
+       "        if not classes:\n",
+       "            # if classes were not given, extract them from values\n",
+       "            classes = sorted(self.values[self.target])\n",
+       "        for item in self.examples:\n",
+       "            item[self.target] = classes.index(item[self.target])\n",
+       "\n",
+       "    def remove_examples(self, value=''):\n",
+       "        """Remove examples that contain given value."""\n",
+       "        self.examples = [x for x in self.examples if value not in x]\n",
+       "        self.update_values()\n",
+       "\n",
+       "    def split_values_by_classes(self):\n",
+       "        """Split values into buckets according to their class."""\n",
+       "        buckets = defaultdict(lambda: [])\n",
+       "        target_names = self.values[self.target]\n",
+       "\n",
+       "        for v in self.examples:\n",
+       "            item = [a for a in v if a not in target_names]  # remove target from item\n",
+       "            buckets[v[self.target]].append(item)  # add item to bucket of its class\n",
+       "\n",
+       "        return buckets\n",
+       "\n",
+       "    def find_means_and_deviations(self):\n",
+       "        """\n",
+       "        Finds the means and standard deviations of self.dataset.\n",
+       "        means     : a dictionary for each class/target. Holds a list of the means\n",
+       "                    of the features for the class.\n",
+       "        deviations: a dictionary for each class/target. Holds a list of the sample\n",
+       "                    standard deviations of the features for the class.\n",
+       "        """\n",
+       "        target_names = self.values[self.target]\n",
+       "        feature_numbers = len(self.inputs)\n",
+       "\n",
+       "        item_buckets = self.split_values_by_classes()\n",
+       "\n",
+       "        means = defaultdict(lambda: [0] * feature_numbers)\n",
+       "        deviations = defaultdict(lambda: [0] * feature_numbers)\n",
+       "\n",
+       "        for t in target_names:\n",
+       "            # find all the item feature values for item in class t\n",
+       "            features = [[] for _ in range(feature_numbers)]\n",
+       "            for item in item_buckets[t]:\n",
+       "                for i in range(feature_numbers):\n",
+       "                    features[i].append(item[i])\n",
+       "\n",
+       "            # calculate means and deviations fo the class\n",
+       "            for i in range(feature_numbers):\n",
+       "                means[t][i] = mean(features[i])\n",
+       "                deviations[t][i] = stdev(features[i])\n",
+       "\n",
+       "        return means, deviations\n",
+       "\n",
+       "    def __repr__(self):\n",
+       "        return '<DataSet({}): {:d} examples, {:d} attributes>'.format(self.name, len(self.examples), len(self.attrs))\n",
+       "
\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import sys\n", + "sys.path.insert(1, '../')\n", + "from utils.utils import *\n", + "from utils.dataset4learners import *\n", + "\n", + "\n", + "psource(DataSet)" + ] + }, + { + "cell_type": "markdown", + "id": "9eaf930d-2803-492d-843a-983f8b3a34fc", + "metadata": {}, + "source": [ + "### 类属性\n", + "\n", + "- **examples**。保存数据集的项目。每个项目都是一个值的列表。\n", + "\n", + "- **attrs**: 特征的索引(默认范围是[0,f],其中*f*是特征的数量)。例如,`item[i]`返回*item*的索引*i*处的特征。\n", + "\n", + "- **attrnames**。一个包含属性名称的可选列表。例如,`item[s]`,其中*s*是一个特征名称,返回*item*中名称为*s*的特征。\n", + "\n", + "- **target**。学习算法将尝试预测的属性。默认是最后一个属性。\n", + "\n", + "- **inputs**: 这是不包含目标的属性列表。\n", + "\n", + "- **values**: 一个列表,包含了相应属性/特征的可能值的集合。如果最初是 \"无\",它将从例子中计算出来(通过函数 \"setproblem\")。\n", + "\n", + "- **distance**: 学习器中使用的距离函数,用于计算两个项目之间的距离。默认为`mean_boolean_error`。\n", + "\n", + "- **name**: 数据集的名称。\n", + "\n", + "- **source**: 数据集的来源(URL或其他)。在代码中不使用。\n", + "\n", + "- **exclude**: 要从`inputs`中排除的索引列表。该列表可以包括属性索引(attrs)或名称(attrnames)。" + ] + }, + { + "cell_type": "markdown", + "id": "da4cace9-dc22-4877-8aa6-d45741c7aca6", + "metadata": {}, + "source": [ + "### 类帮助函数\n", + "\n", + "这些函数有助于根据你的需要修改`DataSet`对象。\n", + "\n", + "- **sanitize**: 将一个例子作为输入,并在返回时将非输入(目标)属性替换为 \"无\"。对测试很有用。注意:给出的例子本身并没有被改动,而是返回一个经过处理的副本。\n", + "\n", + "- **classes_to_numbers**: 将数据集的类名映射为数字。如果没有给出类名,它们将从数据集的值中计算出来。对于返回数值而不是字符串的分类器很有用。\n", + "\n", + "- **remove_examples**: 删除包含一个给定值的例子。对于删除缺失值的例子,或者对于删除类(二元分类器需要)都很有用。" + ] + }, + { + "cell_type": "markdown", + "id": "ba3e68a3-6f61-43e6-9997-bbe667e777af", + "metadata": {}, + "source": [ + "### 导入数据集\n", + "\n", + "数据集可以用以下一行导入。" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ce6f7469-f531-420e-b13f-5f4e46f988bb", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "iris = DataSet(name=\"iris\")" + ] + }, + { + "cell_type": "markdown", + "id": "012561a2-8c35-418a-b415-5c3baa0b6ceb", + "metadata": {}, + "source": [ + "为了检查我们导入的数据集是否正确,我们可以做以下工作:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "cc583d3e-8e31-44ac-be87-d5a4d43c28ea", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[5.1, 3.5, 1.4, 0.2, 'setosa']\n", + "[0, 1, 2, 3]\n" + ] + } + ], + "source": [ + "print(iris.examples[0])\n", + "print(iris.inputs)" + ] + }, + { + "cell_type": "markdown", + "id": "c9ce4cb4-a9d8-4ca0-9915-5ea7d8c7d77e", + "metadata": {}, + "source": [ + "它正确地打印了csv文件的第一行和属性索引的列表。" + ] + }, + { + "cell_type": "markdown", + "id": "ef33f319-e60a-404d-a562-50427bbe2a13", + "metadata": {}, + "source": [ + "当导入一个数据集时,我们可以通过将参数`exclude`设置为属性索引或名称来指定排除某个属性(例如,在索引1处)。" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "f0e082a9-ca54-4e48-b17e-3f8f48bf3cc6", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0, 2, 3]\n" + ] + } + ], + "source": [ + "iris2 = DataSet(name=\"iris\",exclude=[1])\n", + "print(iris2.inputs)" + ] + }, + { + "cell_type": "markdown", + "id": "4864635a-fc8c-4bf6-839d-556b88c9e2fb", + "metadata": {}, + "source": [ + "### 属性\n", + "\n", + "这里我们展示一下属性。\n", + "\n", + "首先,我们将打印数据集中的前三个项目/例子。" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "69ed52a4-24b9-4b7d-9f14-4983dd7cc703", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[5.1, 3.5, 1.4, 0.2, 'setosa'], [4.9, 3.0, 1.4, 0.2, 'setosa'], [4.7, 3.2, 1.3, 0.2, 'setosa']]\n" + ] + } + ], + "source": [ + "print(iris.examples[:3])" + ] + }, + { + "cell_type": "markdown", + "id": "e1a6a8ea-6c02-4dd2-9c51-8151e4b4969e", + "metadata": {}, + "source": [ + "然后我们将打印 `attrs`, `attrnames`, `target`, `input`。注意`attrs`是如何持有[0,4]的值的,但是由于第四个属性是目标,`inputs`持有[0,3]的值。" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "2967a184-65e0-4cdc-b6aa-d0e9913219e7", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "attrs: [0, 1, 2, 3, 4]\n", + "attrnames (by default same as attrs): [0, 1, 2, 3, 4]\n", + "target: 4\n", + "inputs: [0, 1, 2, 3]\n" + ] + } + ], + "source": [ + "print(\"attrs:\", iris.attrs)\n", + "print(\"attrnames (by default same as attrs):\", iris.attr_names)\n", + "print(\"target:\", iris.target)\n", + "print(\"inputs:\", iris.inputs)" + ] + }, + { + "cell_type": "markdown", + "id": "25ccbc5f-939d-4872-a1f1-efaa16c35af0", + "metadata": {}, + "source": [ + "现在我们将打印第一个特征/属性的所有可能值。" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "f129643c-a501-4c97-bd3b-54b3d8087c4a", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[4.7, 5.5, 5.0, 4.9, 5.1, 4.6, 5.4, 4.4, 4.8, 4.3, 5.8, 7.0, 7.1, 4.5, 5.9, 5.6, 6.9, 6.5, 6.4, 6.6, 6.0, 6.1, 7.6, 7.4, 7.9, 5.7, 5.3, 5.2, 6.3, 6.7, 6.2, 6.8, 7.3, 7.2, 7.7]\n" + ] + } + ], + "source": [ + "print(iris.values[0])" + ] + }, + { + "cell_type": "markdown", + "id": "0f627640-dbfe-4a17-8908-f4cbc52778f7", + "metadata": {}, + "source": [ + "最后我们将打印数据集的名称和来源。请记住,我们没有为数据集设置来源,所以在这种情况下它是空的。" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a9fc51f8-688d-4f7b-98b4-eb2278b09abe", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "name: iris\n", + "source: \n" + ] + } + ], + "source": [ + "print(\"name:\", iris.name)\n", + "print(\"source:\", iris.source)" + ] + }, + { + "cell_type": "markdown", + "id": "1c0e3836-ef6a-41a6-81c6-ecce3ce54d0e", + "metadata": {}, + "source": [ + "上述的一个有用的组合是`dataset.values[dataset.target]`,它返回目标的可能值。对于分类问题,这将返回所有可能的类。让我们来试试:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "6a8e7638-72ad-4446-8b10-1ddc85132bee", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['virginica', 'versicolor', 'setosa']\n" + ] + } + ], + "source": [ + "print(iris.values[iris.target])" + ] + }, + { + "cell_type": "markdown", + "id": "c435e57f-1164-418e-9728-17031dd92293", + "metadata": {}, + "source": [ + "### 辅助函数\n", + "\n", + "我们现在将看看在这个类中发现的辅助函数。\n", + "\n", + "首先我们看一下`sanitize`函数,它将给定例子的非输入值设置为`None`。\n", + "\n", + "在这种情况下,我们想隐藏第一个例子的类,所以我们将对其进行消毒处理。\n", + "\n", + "注意,这个函数实际上并没有改变给定的例子;它返回一个经过处理的*副本*。" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "6451c9da-93d1-4c00-b37d-403168493fc3", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sanitized: [5.1, 3.5, 1.4, 0.2]\n", + "Original: [5.1, 3.5, 1.4, 0.2, 'setosa']\n" + ] + } + ], + "source": [ + "print(\"Sanitized:\",iris.sanitize(iris.examples[0]))\n", + "print(\"Original:\",iris.examples[0])" + ] + }, + { + "cell_type": "markdown", + "id": "2418c071-cab2-46a5-ac1b-739b7f1bae7f", + "metadata": {}, + "source": [ + "目前 \"iris \"数据集有三个类,Setosa, virginica和versicolor。我们想把它转换成一个二元类数据集(一个有两个类的数据集)。我们要删除的类是 \"virginica\"。为了达到这个目的,我们将利用辅助函数`remove_examples`。" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "b914eeee-4e3c-4b32-8d29-ab8ec2afa0bd", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['versicolor', 'setosa']\n" + ] + } + ], + "source": [ + "iris2 = DataSet(name=\"iris\")\n", + "\n", + "iris2.remove_examples(\"virginica\")\n", + "print(iris2.values[iris2.target])" + ] + }, + { + "cell_type": "markdown", + "id": "9ededc31-c6d2-4510-8791-44c80e425cf4", + "metadata": {}, + "source": [ + "我们也有`classes_to_numbers`。对于模块中的许多分类器(如神经网络),类应该有数字值。通过这个函数,我们将字符串类名映射为数字。" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "69dbcffa-6304-4e6c-90c6-91f69afe12cd", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Class of first example: setosa\n", + "Class of first example: 0\n" + ] + } + ], + "source": [ + "print(\"Class of first example:\",iris2.examples[0][iris2.target])\n", + "iris2.classes_to_numbers()\n", + "print(\"Class of first example:\",iris2.examples[0][iris2.target])" + ] + }, + { + "cell_type": "markdown", + "id": "c76f05da-d851-49bb-a98d-24438cd2edb5", + "metadata": {}, + "source": [ + "正如你所看到的,\"setosa \"被映射到了0。" + ] + }, + { + "cell_type": "markdown", + "id": "e544e16b-62ef-4e07-924f-45e81d76ce73", + "metadata": {}, + "source": [ + "最后,我们看一下`find_means_and_deviations`。它找出每个类的特征的平均值和标准偏差。" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "32a009e5-25bc-4651-835c-9114f2c9a9f0", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Setosa feature means: [5.006, 3.418, 1.464, 0.244]\n", + "Versicolor mean for first feature: 5.936\n", + "Setosa feature deviations: [0.3524896872134513, 0.38102439795469095, 0.17351115943644546, 0.10720950308167838]\n", + "Virginica deviation for second feature: 0.32249663817263746\n" + ] + } + ], + "source": [ + "means, deviations = iris.find_means_and_deviations()\n", + "\n", + "print(\"Setosa feature means:\", means[\"setosa\"])\n", + "print(\"Versicolor mean for first feature:\", means[\"versicolor\"][0])\n", + "\n", + "print(\"Setosa feature deviations:\", deviations[\"setosa\"])\n", + "print(\"Virginica deviation for second feature:\",deviations[\"virginica\"][1])" + ] + }, + { + "cell_type": "markdown", + "id": "ceaf8c9f-adef-43ea-a562-8654f273c1ca", + "metadata": {}, + "source": [ + "## iris的可视化\n", + "\n", + "由于我们将在这个笔记本中广泛使用iris数据集,下面我们提供一个可视化工具,帮助理解数据集,从而理解算法的工作原理。\n", + "\n", + "我们使用`matplotlib`和`notebook.py`中的`show_iris`函数在三维空间中绘制数据集。该函数接受三个参数,*i*、*j*和*k*,它们是指iris特征,\"萼片长度\"、\"萼片宽度\"、\"瓣片长度 \"和 \"瓣片宽度\"(0到3)。默认情况下,我们显示前三个特征。" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "fc7d235b-57a5-4af4-a6ca-1219c29d7210", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "\n", + "def show_iris(i=0, j=1, k=2):\n", + " \"\"\"Plots the iris dataset in a 3D plot.\n", + " The three axes are given by i, j and k,\n", + " which correspond to three of the four iris features.\"\"\"\n", + "\n", + " plt.rcParams.update(plt.rcParamsDefault)\n", + "\n", + " fig = plt.figure()\n", + " ax = fig.add_subplot(111, projection='3d')\n", + "\n", + " iris = DataSet(name=\"iris\")\n", + " buckets = iris.split_values_by_classes()\n", + "\n", + " features = [\"Sepal Length\", \"Sepal Width\", \"Petal Length\", \"Petal Width\"]\n", + " f1, f2, f3 = features[i], features[j], features[k]\n", + "\n", + " a_setosa = [v[i] for v in buckets[\"setosa\"]]\n", + " b_setosa = [v[j] for v in buckets[\"setosa\"]]\n", + " c_setosa = [v[k] for v in buckets[\"setosa\"]]\n", + "\n", + " a_virginica = [v[i] for v in buckets[\"virginica\"]]\n", + " b_virginica = [v[j] for v in buckets[\"virginica\"]]\n", + " c_virginica = [v[k] for v in buckets[\"virginica\"]]\n", + "\n", + " a_versicolor = [v[i] for v in buckets[\"versicolor\"]]\n", + " b_versicolor = [v[j] for v in buckets[\"versicolor\"]]\n", + " c_versicolor = [v[k] for v in buckets[\"versicolor\"]]\n", + "\n", + " for c, m, sl, sw, pl in [('b', 's', a_setosa, b_setosa, c_setosa),\n", + " ('g', '^', a_virginica, b_virginica, c_virginica),\n", + " ('r', 'o', a_versicolor, b_versicolor, c_versicolor)]:\n", + " ax.scatter(sl, sw, pl, c=c, marker=m)\n", + "\n", + " ax.set_xlabel(f1)\n", + " ax.set_ylabel(f2)\n", + " ax.set_zlabel(f3)\n", + "\n", + " plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "1590e14c-cc5e-41b1-9789-c8aa90becb3a", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAGQCAYAAABriI3PAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOz9eZwjd33njz+rdKtbrVbf9/Tc93gOj+ewARswRwwkJJybBXJnN5AQ2OwmkHxzLSFkIRvYH4k3EELIhoQAxglHMMY2xjb22HP0fd/3Kalbrfuo+v2hqZqSWt0ttaSeHrteefgRplv9qVJJ9XnV+/16v19vQZZlGR06dOjQoSMPiLf7BHTo0KFDx50PnUx06NChQ0fe0MlEhw4dOnTkDZ1MdOjQoUNH3tDJRIcOHTp05A2dTHTo0KFDR97QyUSHDh06dOQNnUx06NChQ0fe0MlEhw4dOnTkDZ1MdOjQoUNH3tDJRIcOHTp05A2dTHTo0KFDR97QyUSHDh06dOQNnUx06NChQ0fe0MlEhw4dOnTkDZ1MdOjQoUNH3tDJRIcOHTp05A2dTHTo0KFDR97QyUSHDh06dOQNnUx06NChQ0fe0MlEhw4dOnTkDZ1MdOjQoUNH3tDJRIcOHTp05A2dTHTo0KFDR97QyUSHDh06dOQNnUx06NChQ0fe0MlEhw4dOnTkDZ1MdOjQoUNH3tDJRIcOHTp05A2dTHTo0KFDR97QyUSHDh06dOQNnUx06NChQ0fe0MlEhw4dOnTkDZ1MdOjQoUNH3tDJRIcOHTp05A2dTHTo0KFDR94w3u4T0PHKgyRJxGIxRFHEYDAgivozjQ4ddzp0MtGxY5BlmUQiQSwWIxQKIQgCgiBgMBgwmUwYDAadXHTouEMhyLIs3+6T0PHyhyzLxGIxEomE+r8FQUCWZSRJQvkaiqKI0WjEaDTq5KJDxx0EnUx0FB2JRIKZmRnC4TAtLS3Iskw0Gl1HErIsp5CLErno5KJDx+6HnubSUTTIskw8Hicej7O2tkYgEGDPnj0qUSj/X4FCHgpZKMQSi8WIxWLqa7TkYjQaU9bQoUPH7YFOJjqKAoUEJEkCyBiFbEUCip6i/Rtl3Wg0ui5yUQhGJxcdOnYeOpnoKCi0G74kSYiiqG76+WZUtyKXUChEKBSirq4uRdDXyUWHjuJDJxMdBYNWZAdUIgHWkUk2kclWSCeX5eVlZmdnqaysVCOXTIK+Ti46dBQeOpnoKAiUCCGRSKSQiBZKJDE4OMjk5CQOhwOXy4XL5cLpdOYtrGvTXgpxSZJENBolEono5KJDRxGhk4mOvKD0jsTj8ZS0VjpEUSQej/Piiy8iSRKnTp0iFAqxsrLC7Ows8Xgcp9OpkovD4dgWuSgkopyDErno5KJDR3Ghk4mObWOztFY6VldX8Xq9NDc3c+jQIeLxOC6Xi8bGRmRZJhgM4vV68Xq9TE1NIUnSOnLJRrDf6ndaclH+i0QiRKNR9T3o5KJDR+7QyUTHtqB0sm8WjSiv6+vrY35+ntLSUo4dO6ZGMwoEQaCkpISSkhKampqQZZlAIKCSy8TEBADl5eUquZSUlGyYSssGSkoMkgSTTi7ayMVkMqnkstl71aHjlQydTHTkBG3vCGwejaytrdHR0YHRaGT//v243e6sjiEIAqWlpZSWltLc3Iwsy6ytreH1evF4PIyOjiKKYgq52O32vN7XZuQSDocBCIfDxGIxqqurdXLRoSMNOpnoyBrpvSPaDVgLWZaZnp6mv7+fPXv2cODAAebm5rZdGiwIAmVlZZSVlbFnzx4kSVLJZWlpieHhYYxGIzabjWg0SjAYxGaz5bXJZyKX1dVVlpaWcDgc6muUxkmdXHS80qGTiY4toe3n0NqcZEIsFqOnpwev18vZs2eprKwE1pcG5wNRFHE6nTidTlpbW5EkidXVVaanp/H7/bz44ouYzWY1anG5XFit1ryOqaS8BEHAZDKlWL8okYsoius0F51cdLxSoJOJjk2RLrJvRiQrKyt0dHRQUlLC5cuXsVgs6u8KSSbpEEURl8tFPB4nHA5z9uxZVfCfmZmhv78fi8WSQi7ac8sW2vPfKC0mSRKRSIRwOKyTi45XFHQy0bEhsukdgeQmOz4+zvDwMAcOHKC1tXXda4tJJukwGAxUVFRQUVEBQDweV8llamqK3t5e7Ha7Sizl5eWYzea8jplOsgq5JBIJEonEOkFf6yumk4uOlwN0MtGxDtn2jgBEIhG6uroIBAKcP3+e8vLyjK/bKTLJdAyj0UhlZaWacovFYqysrOD1ehkbGyMQCFBSUpJCLiaTKa/zyGRaqb2u2lku6b5iOrnouBOhk4mOFOTSO+J2u+ns7MTlcnH58uVNN+CdIJNsN2GTyUR1dTXV1dUARKNRlVxGRkYIBoPruvONxvxulY3IJR6Pp8x2CYfDVFRU6Hb7Ou446GSiQ4UkSXi9Xnp7ezl//vyGm7MkSYyMjDA+Ps6RI0doamrKaiPfraNzzGYzNTU11NTUAMloS+lxGRgYIBKJ4HA4MBqNKtFqPcG2g0zkEgwGuXr1Kq961av0KZQ67jjoZKIjpXckHo/j8/k2JIdQKERnZyexWIyLFy+qZbJbYadSN4UgLIvFQl1dHXV1dQCq7cvMzAyBQIBnnnkGp9Op9rkUylcsU7WYErmAPoVSx+6GTiavcEiSRDweV9NaSmVSJiwsLNDd3U1tbS1Hjx7N6elcEAS1P6VYKBZh2Ww2bDYbkiSxuLjI4cOH8Xq9RfMVg43TYhvNctHJRcfthk4mr1Bs1DsiiuK6TT+RSDAwMMDs7CzHjx+nvr4+5+PdzuquQkIQBOx2O3a7fVNfsfLycjVyycZXTLv+Rj/Xp1Dq2M3QyeQViHRLFG0FUfomHwgEaG9vRxRFLl++vG3bkttZzVVIZCLFdF8xv9+vCvrF8BVTjqtPodSxm6CTySsM2t4R7ZOuAu2T7+zsLL29varTbz4plN1UzVXsc3A4HDgcjnW+Ym63e0NfMS2Zb/e4W5GLKIrrBP3dcM10vDygk8krBNn2jig/6+zsZHl5mdOnT6sltPngTk1r5YtsfcVcLhc2m03VRgo9hVIhl4GBAQD27dun2+3rKCh0MnkFIJfekUAgACQdcu+99968Pa0UvFzSXPki3VcskUjg8/nUyEWSJJ5//vmC+opBKrko/1sfFKajkNDJ5GUOZcPYqpNdlmUmJyfVJ9czZ87kbTGixSslzZUrDAaDShrV1dXcuHGDo0ePpviKWa3WlLTYdnzFFMiyrJKEPoVSRyGhk8nLFEpaS6nW2oxIotEo3d3d+Hw+zp49y7Vr1wq+8b8cIpOdOH9RFNf5iq2srLCyslIQXzFJkjIWEYA+hVJHftDJ5GWIXNJaXq+Xjo4OysrKUixRirFx7vYU1O3GRr5iVVVVVFVVAZl9xUpLS9XIZStfsWz0mFynUCplyLoj8isbOpm8zJDtOF1ZlhkdHWV0dJRDhw7R0tKyzvW2kNCuLUkS09PTqi18vr5XmY5xp2Kr91AIX7Fcr1M2Uyi15KLb7b8yoZPJywS5jNMNh8N0dnYSDoe5cOECZWVlKb8vRre6kuYKBoO0t7cTj8fVzUjZ/CoqKigrK8vL9+pOjn62c+7Z+oop5JIpzZUrtiKX6elpSkpK1PHGOrm8MqCTycsA6eN0N+sHWVpaoquri6qqKs6ePZsxKhBFsSiRiVKp1NDQwL59+4Dkk7Uy272npyfFmqSioqIg3eN3EvJ9Dxv5inm9Xvr6+ohEIvh8PrVLv1C+YlpyWV1dxWQypUyhzGS3r5PLyws6mdzB0DambZXWkiSJwcFBpqamOHbsGI2NjRuuW2ixXHEZBjhx4gS1tbXqOVutVurr66mvr19nTTI5OQlk1z3+ckAxoirFV0y5vh0dHRiNRgKBADMzMwXzFdNCKfhQRHp9CuUrAzqZ3KHIRWQPBoN0dHQgSRKXLl2itLR007ULmeZSjq2cZ21t7abHTbcmURr8lpeXGRkZURv8lP9sNlvKGnd6mquYm6mia5SXl6vXN528ZVlOEfNziQwVKGSiPW66HrfRFEqdXO5c6GRyB0KSJObn55FlmcrKyk1vuLm5OXp6emhoaODw4cNZ6RGFSnMtLCzQ1dVFQ0MDra2tPPPMMzltmJm6x5Xxu3NzcwwMDGCxWKioqCiokL/VOd3J62uv/0a+Yooj8tjYGIIg5BwZbqXLbOSIrE+hvLOhk8kdBO0NNzc3h8lkUktG05FIJOjr62NhYYGTJ09uGhGkI980l2LbMTMzo7oMK70K+Tx9i6KobmpwqwdDMVT0+/0IgsDQ0JD6ZL0TBFMo7FQfzmbOxIqvWEtLC5IkqeSSja+Y9hi5pMo2IhftFEqdXHY/7pw77RWO9LSWYoeRCWtra2pu/PLly+tSQVshnzRXKBSivb1dTamVlJSsex+FQnoPhtvtpru7m0QiwdDQkFoppkQuhRCbi42djEy2giiKGX3FPB7POl8xhbwVf7F83kc25BIKhVRi02e57A7oZHIHIFPviCiKKrEokGWZ6elp+vv7aW1tZf/+/du6wbYbmSwuLtLV1UVdXR1HjhxJSalpN4ZiwWw2I4oiR44cAZLEpugB6UOscq0U2wnsdocAra8YkOIrpqQdzWYz8Xgcr9dLaWlpwXzF0sllfn6eeDyudv/rUyhvP3Qy2cXYrHdEFEV1KBIkO6N7enrwer2cPXuWysrKbR83V81EWyl24sSJTYdnaQdxFQPa81YqmRoaGlLEZo/Hs65SrKKiImPKZqexmyKTraD1FYMkuayurtLd3Y3b7WZ6ehqr1apGLfn6iilQzt9oNKaMOFbs9kEnl9sBnUx2KdJ7R9I3YO1ExJWVFTo6OigpKeHy5ct537C5pLm0aa3Lly+vS2tp17yd2KhSzOPxZKwUq6ioKJhjcrbYqcikWJuqwWCgoqICg8HAkSNHKC0tVTWtQviKaSFJkqqHZYpctppCqZNL4aGTyS7DRuN006GkucbGxhgeHubAgQO0trYWZNPONjJR0lrZzIRXzquYG2Yu711bKdba2ppSKTY7O8vAwID6VK38txOb/Z0UmWx2DCUyyMZXTCGWrXzFtFBSvpmw0SwXfcRxcaGTyS5Cusi+WTookUjg9Xrx+XycP3+e8vLygp3HVpqJNq11/PhxGhoasloTiv/0vd31tZVi+/btW1cp1tPTo2oyy8vLRakUu93VXIXCRqXBmXzFlDLkbH3FtjpGJmxGLvoUysJBJ5NdAu043a2atdxuN+Pj4xgMhhSn30JhszRXKBSio6ODeDyeVQOkdk24c5oK05+qo9Eow8PDeL1etVKsrKwsZeMrRNpkJ6KGYiPbVJrZbKa2tlYtW9/KV8zpdKqkIEnStj3csiUX3W4/N+hkcpuR7ThdSN5Aw8PDTExMUFtbSzgcLjiRwMZprqWlJTo7O7NKa2WCNuIpxo1ZzJvdbDbjcDiIx+OcOnVqw0oxpQx5u53jxcZOpbm2c4xMvmJK5NLX10c0GlWr8SKRSMGul5ZctIPCwuEwbW1t3HXXXZjNZp1ctoBOJrcRuViiaCOCixcvsra2xtTUVFHOKz3NJUkSQ0NDTE5OZp3W2mrdYm2cOxX5pFeKBQIBlVwmJiYAUvSWbCvFdutGnwsK4UwM66+xlsDX1tZYW1vD7XYX1FdMa1gpSRKBQED1F9OnUG4OnUxuE7IdpwtJW5Lu7u6UiCAQCBTcJl6BNs213bTWRuvuFgG+0MctLS2ltLSU5uZmtXM8U3OfErlkqhR7OUQmynsodKWUIAjY7XbsdjuNjY1cu3aN6upqDAZDQX3FtFDuAa04r41c9CmUqdDJZIeh7R1RcsubiewDAwPMzs6qtiQKirkxK2mufNNamXCnaCb5QNs53traqjb3eTyelLnuCrkoJbI7WWlVLGhL2YsJWZax2WzU1NSs8xVTqsW24yumRSKRWFcEo41cMk2h1JLLK20KpU4mOwhJkojH41mltfx+Px0dHYiiyOXLl7Hb7Sm/1/aZFAOLi4usrq5uaVefC3biZio2WW3nPaQ392krxbQlsmazmVgsRjweL5qn2E5V0xW7hyO9NHgrX7GRkRH1c1AIZqvUYzYiv5ZsXulTKHUy2QFk2zuiYGZmht7eXpqbmzl06FDGG7NYZBIOh/F4PAiCkHdaKx0v1zRXrshUKeb1epmZmSEcDvPss88WpVIMip/m2snIZLNrkslXTLF+WVxcXOcrpqQetee9WS/LRnglk4tOJkVGuiXKZkQSj8fp7e1laWmJ06dPq/X4mVAMMlHSWkajkfr6+oISCRSfTO5UKCWyyud59OjRdZVi2nRNPlrAnaqZpCPXjV4xhSwvL2fv3r0b+oppyUUp088H2ZBLIpEgFApRVVV1R5OLTiZFhLZ3RGv3kAk+n4/29nasViv33nvvllYehRytqy05PnbsGF6vt2ilu7u1aXE3QNnoN6sUGx8fRxCEbVWKaY9RLOxUZLKdqEGLTL5iKysrrKysqLqWMnp4fn6+oL5i6eSytrZGf38/99xzzx09hVInkyJA6R0ZHR2ltLSUioqKDb8IsiwzOTnJ4OAg+/btY9++fVmXkBYiMgmHw3R0dBCLxbh48SIOh4OVlZWipNC0ZBKJRIjH4+u0oHzXf7khU6WYMn1SWymmVIltVCmmYCcik53wvMqXTNJhMBiorKxUDVLj8TgTExPMzs4yOTm5zlfM5XIVpMdLuY8VoX6jKZR/9md/xpve9CZe97rX5X3MYkEnkwJD2zuytLSEIAgbOvhGo1G6u7vx+XycO3eOioqKrI9TiDTX8vIynZ2dVFVVce7cuRTjvGI84SvrKpqQMgNeuxEWownzTkE2G73WBl6pFFM8xbSVYhtd050gk50g9UKTSTqMRiMlJSXY7XbOnTu3zlesu7t7275i6dAK/ZlMK2VZ5qmnnuLcuXMFe3/FgE4mBUR678hmA6w8Hg+dnZ2UlZVx+fLlnN1T8yETSZIYGRlhfHyco0eP0tTUtG7tYqWLRkdHWV1d5dSpU5SUlKh5a+UGVYZZVVRU5Cw8a3sBirGh7UajR8WpV3kQUWaJZNr0KioqdiTNtRPi+04cR7vJb+Qr5vV6c/YVS0cikdiwakwhl0AgsKEj926BTiYFgBKWKtVa2gFW6Ru+LMuMjo4yOjrKoUOHaGlp2dZNoWz4uW4OSlorGo2qaa10CIKwbvBWvggGg0QiEbXUWSmD1VY1Kd5MHo+Hnp4eVXhWnrJLS0tflqksBYUgK6PRuG7T83g8qt8VQHd3N5WVlVRUVFBWVlbQJ/ydSHMp16kQfU+bYTMBPhtfsbKyMrVwQusrluk4m70XZRZPoQtiCg2dTPLEZpYo6dMQw+EwnZ2dhMNhLly4QFlZ2baPu52n8I3SWukodGSyuLhIZ2cnBoOBQ4cOYbPZMpKV1ptJuYGUjXBsbEx19lXIZaNxxDuVaikGCn3eZrNZvaaSJPH0009TU1PD2toaMzMzJBKJlAFh+RL2TkUMsDMif7aEtZGvmNfrXecr5nK5Ukg8m6oxPTJ5mSPTOF0ttGmupaUlurq6qKqq4uzZs3k3pSlfvmxyx7IsMzw8rKa1GhsbN70RC6WZaD29Tpw4wcjISNYbgCDcGmalFZ49Ho9aypmuDdypBKJgpyrd6uvr2bNnT8ErxZRj7ERZMBS//Dif0uDNfMWmp6dJJBIquQSDwazIJFMWYTdBJ5NtIL13ZKOyPSUy6e/vZ2pqqqDd5Foy2QxKNBSJRDZMa6WjEJVikUiE9vZ2YrGY2vw4Ojqq/l5J0WULrfC8d+/edV3kijYA4PV6cblcd+QkvZ3oAVGOsVmlmNLYZzKZ1jX2bYadjEx2Wy/LRkj3FVNIXPn+ut1uNcOhXGdthBiPx4lEInqa6+WG9HG6m33ZEokE8/PzWK3WgneTZ0Mmbrebjo4OKisrc4qG8k1zeTweOjo6qKioKFqVWHoXeSQSYWlpicHBQfr6+u5IvWWnIpONrsNWlWJ9fX1blsfuVGSylYtEoY5TDF1GS+JNTU309fUBUFJSss5XrKurS30ALSaZtLa2qm7XWvzGb/wGf/3Xf53VGjqZZAmtJUo2Tr9zc3PMzMxgs9m4ePFiwb+Uys2UiUxkWWZkZISxsTGOHDlCU1NTTjfedjd9WZYZHx9neHiYw4cP09zcvM4kr1gbpsVioba2lsHBQS5evEgkEtmW3nK7sZORyVZIrxTLVB6rrWAqLy9/WZhVKkgkEkXzSNNCkiRKSkpoaWlZ5yv2yCOP8Oyzz2I2m/nVX/1VXv/61/PAAw9w6NChgl7nq1evpuiY3d3dPPjgg7zzne/Meg2dTLJALnNHEokEfX19LCwsqKJnsapOMpFJJBKho6ODcDicdVorm3W3QiwWo6urC5/Pxz333IPT6cy47k6V1+aqt2TbI7ATm32x19/ue0gvj81UwaQUV6ysrBS8UkxBsXtMbsdxtHuE1lfsW9/6Fl1dXbzpTW/i1KlTfP3rX+e3f/u3+Yu/+As+/OEPF+wc0q2bPvWpT7F//35e85rXZL2GTiZbIJdxumtra3R0dGA0Grl8+TKLi4u43e6inVt6OsrtdtPZ2UlFRUVeIn+um75iBWO32zftmdkpMkk/RjZ6Sz79LYU8790UmWyF9Oq7cDjM2NgYbrebrq4uJElKmT5ZqFTj7drki4WtSoNjsRhms5n/7//7//jDP/xDwuGwanVfDESjUf7pn/6Jj370ozl9XjqZbACldySbcbqyLDM9PU1/fz+tra3s379f9dcpdL+GFkofS75prUzrZrvpT09P09fXl5UVzG5xDc6kt2TqbylUuexuQaHJRAvFU6y8vJxIJMLp06cJBAIpqUZtpVhFRQU2m21b57JTZFIIo8dCHMfv96eUBVut1i0LIfLBv/3bv7GyssIv/MIv5PR3OplkQC5prVgsRk9PD16vl7Nnz6ZYp2zWAV8IiKJIJBLh2rVrhEKhvHtXFGST5kokEqrDcfr73mzd3WjEuFl/y/j4uKq3KLpZsfBysDrRNu0qIrOiAyipxsXFRYaGhlJceisqKrI2UnylRSbKvJudeqD50pe+xJvf/Oacx3PrZJKGrXpHtFhZWaGjo4OSkhIuX7687mYo9gArWZbp6uqisrKSM2fOFEws3GrTDwQCtLe3YzAYuHz5ck5PScV8Ok4/xnawld4SDod54YUX7kg/sZ0gk41Kg9NTjdpKMSW6zdZIcSfKj2FnI5PNyCQYDO5Yw+LExARPPPEE3/rWt3L+W51MbiLb3hHltePj4wwNDXHw4EFaW1s3vIGKQSaKJUskEqGlpYWjR48W9ObaLM21sLBAV1cXTU1NGw7u2gi7Jc2VC7SboMFgYGVlhYaGhox6y1a2GVvh5RSZbAVtpdj+/fuzqhRTruvLTYDfikzS01zFxJe//GVqamp46KGHcv5bnUxY3zuyWQ17JBKhq6uLQCDAPffcQ3l5+YbrFkMziUQidHZ2EgqFsNvtVFVVFXyDyJTmkiSJwcFBpqenOXHihGobkeu6t0OAL+S6oihuqLf09vbuar3ldkYmW2GzSrH+/v4UO5Kdikx2U5prJ8hEkiS+/OUv84EPfGBbWY5XNJnkOk5XqZZyuVxcvnx5y/RGoTUTpRnQ5XJx5swZrl69WvS5I5A68+TSpUvb/mJr190tG2y+yFZvyaa/5ZUUmWyF9OuqtSNxu90kEgn1XihWU+pOpLmUPSgbzaTYeOKJJ5icnOSXfumXtvX3r1gyUSwNJEnCZDJtSiTaSYS5VEsVKs2ldRrWNgMWK42mTXMpXfRVVVXcfffdeT2pFXsju90EVYj+ljv9GhUjaki3I5mcnGR5eRmXy5XSlKp1PNhupZgWOxGZKLZCu0EzecMb3pBXVP+KJBMlGunr68Nms3Hw4MENXxsKhejo6CAej+fcBFiIzT4ajdLZ2amm1bTNgIXw0MoExYJ+ZGSE0dHRgpQbK+tqv6zF2th2S8VYLv0tSvqmmLiTIpOtYDabUzrGFdJeWFhgcHBQrRRTru12Ru7uhGairRjdCDupmeSDVxSZpPeObJWGWlhYoLu7m9raWo4ePZrzU0q+momS1iovL8+YVivWECtlyNf09PSG3ezbwW4tDd4pbNbf0tvbqzanKdFLoVM3OzW4qtjHSE8/bVQp5vF4mJqaShm5W1FRkdVURCX9tFNkslWaK5cprLcLrxgyydQ7YjQaM272iUSCgYEBZmdnOX78OPX19ds6psFg2NYAq2wHaBUjzbW6ukp3dzdAVrpQLrgTq7mKiXRdoK+vj3A4zOrqas56SzbYKQF+J1JDm23yG3mKeTyejFMRtZVi2vehrFVMZOOscScMxoJXCJmkj9NVPjhRFInFYimv9fv9dHR0qBMB7Xb7to+rdfbN9ku5WVor0/qFIhNtF39jYyOzs7MF75+406u5oHiEJQgCJpMJk8nEwYMHC+onpuDlkubKNWLYqFLM4/GsqxRTBldlk34qBLaq5IKdE+DzxcuaTDYap6vAYDCkRCYzMzP09vbS3Nyccw9FJuRKJl6vl/b29g3TWpnWLwSZxONxent7WV5e5uzZs5jNZmZnZ/NeNx1aMolGo/h8voKaAd5pkUk6tJt9rnpLtv0td6IAn+kY+XxnNqsUm56eRpIkVRtVooJivadsyUTXTG4jJEkiHo9vaomikImymS4tLXH69Ol1DprbhXYs52bEIMsyY2NjjIyM5DQXvhACvN/vp729HZPJpHaz+/3+ovZqrKys0NbWRiwWS0nlKH5NhTjGnYqNPvet9JZs+lteTpFJodJPmQZX+f1+FhYWWFlZ4caNG+p3VPmvEJViCrJ5L3fClEV4GZJJLr0jBoOBSCTC888/j9Vq5d577y2ogZpCYJtt+NFolK6uLvx+f85id74C/Pz8PF1dXbS0tHDw4EF1EyhWlRgkNZmJiQkOHDhAXV2dagaoVOFYLBYqKyuzFkpfTsjls9xOf8tublrM9RjFIixBENSNe3Z2lvvuuw+fz4fX602pFNOmG7dTKaZAj0x2KdItUTYjElmW8Xg8rK6ucvDgwS0db7eLzVJRXq+Xjo4OnE7ntsTu7aa5JEliYGCAmZkZTp06RW1tbcrvi6FtJBIJvF4v4XCYc+fOUV5eTjQaVec2tLa2qqkcRSgNhUIp1vDZpMR2q+dXttjO+Wfb32Kz2YjH48RisaKR9G7UTLZ7DIPBoPavlJeXq5ViSrpRWymmkEuuD0BbNUYq/XC6ZrKD0M4dURr6NkI0GqW7uxuv14vdbmf//v1FO69MG77W2+vQoUPs2bNnW5vIdkqPw+Ew7e3tJBKJDQsMlGtXqCfZYDBIe3s78Xicuro6KioqMpJgeionHA7j8XjweDzMzMwgSdK6lFim87tT01yFOu+N9Jbp6WkikQjPPvtswfzE0rFT0c9OCOOZjmEwGKisrFRdsmOxmKq3pFeKKbNxNru2emSyi5DL3BFI9m50dnZSVlbG8ePHGRoaKur5pW/4SlprbW1tS2+vbNZOr0bbDMvLy3R0dFBTU8OxY8c2/BIr168QG8PS0hKdnZ3U19fnvJbVaqWhoYGGhgY1l+12u1laWmJoaAiLxaISy53k3rsZirERKySt3CPHjx8vmp/YnZ7myvUYJpOJmpoaampqAFLGRff19aVUilVUVOBwOFLWzZZMdM2kyMhl7kim3o2VlZWiDq+CVH+ulZUV2tvbKSsr4957781788s2HaUdnnX06FGampq2XBfyu2m1xzx27BiNjY0MDAxs+3oruWyHw0Fra6uabvB4PIyNjdHT04PD4UCWZXw+H1ar9bZMS8wHOzG2VxCEgvqJZTrGbtno8z3GdqI1i8VCfX099fX1KZViHo9HrRTTEnc8Ht/0ONFolFgspqe5iomNekcyIRwO09nZSTgcThkgVexJiNpjjI2NMTw8zMGDB7ed1sq09laaidK3EgwGsx6epU1zbQexWIzOzk78fn/KMQupxaSnG5QnQp/Px+DgIAMDA1mlxHYbdtroMVu9RUsumz0EvVwik0KYPG5UKaaQy+joKJAkoJmZmYyVYoFAAEAnk2Jgq96RdChplurq6nVz0Ys9CRGSX6jh4WGi0Sjnz5/PK62Vjq3IRBsJXbp0KetISJvmyhVra2u0tbWpA8O0xyxm06LyRDgwMMDZs2fVAoulpSWGh4fVCpzdnBLbqchkM+Tb37ITkcmdVn6sQBtdK55i3d3dxGIx5ufn1WpGpUrMbDbj9/sBiqqZzMzM8Lu/+7t8//vfJxQKcejQIb70pS9x7ty5nNa5o8gkl7SWMn9jampKTbOkQ+kzKZZouLKyoopnly9fxmw2F3T9jchElmWmpqYYGBjgwIEDGw7v2gjaNFcumJ2dpaenh71797J///7bIo4rFXylpaU4HA727NmzLiXW3d1NWVlZiki6W1Jiu82CPpv+FqfTqRK1UgBTTOwmzSQfKJZOSjWjtlJscnKS9773vdhsNurq6vjOd77DAw88UNCHUUhWlN5777088MADfP/736empoaRkZFtHeeOIZNcxukGg0E6OjqQJIlLly5tGCJqJ7cV8ilElmUmJiYYHBzEarXS0tJScCKBzP0g8Xicnp4ePB4P586d25ZBXK6RibbUeLOmz2IZU6Yj/RgbpcQ8Hg/d3d1qlZhCLna7/bakxHZDZLIVttJbEokEk5OTRKPRgviJZcKdkubK9jjK3pP+PX3++ef5yle+wt/8zd/wsY99jKGhIc6dO8enP/1pXvOa1xTk+H/xF39Bc3MzX/7yl9Wftba2bmutXU8muYzTBZibm6Onp4eGhgYOHz68KUkov8umoiJbxGIxurq68Pl8nD9/ntHR0aJtEumbs9LNbjabM86kzxbK0302kYkyOCsej2flZbYTkclWSBdJlTz28vIyIyMjmEwm9Um7oqJix1Jid9pwrEx6i9IAvB29JVtkIpNgLMhXe77K61pfx77yfQU5xk5NWdyItGpra7lw4QL/9E//RF9fHzMzMzz11FMZsyzbxbe//W3e+MY38s53vpMf//jHNDY28hu/8Rv86q/+as5r7WoySR+nu9mTQiKRoK+vj4WFBU6ePLmuES8TtN5ZhYCiUTgcDjWtVUyRX5vmmpubo7u7mz179nDgwIGC+IpttfErXmKVlZUcP358y5tvN1rQp+extamGiYkJtUqsoqKCaDS6I+dTLBSbrJQHvcbGRpxOZ8H8xNKRiUyuzV3j+ennkWSJXzv9a3m/z9sRmWSCMstEEASampp4//vfX9Djj46O8vDDD/PRj36Uj3/847z00kv81m/9FhaLJedj7UoyUbSRcDiM0WjcMhpZW1ujo6MDo9HI5cuXsw6tlebGfDd7Ja01NDS0TqMopsivnHtvby+zs7Pcddddar17vths49e+31y9xHYC+RBWppSYohEsLS2pnm9K1FLIlNhOpLmKDS1hbaa39PX1EYvFUvSWbPtb0ivGgrEgP5r8EYIg0LnUyZB3iEMVh/J6H7tl/nuxpyxKksTdd9/NJz/5SQDOnDlDT08PDz/88J1PJgqRzM7OMjY2xqVLlzbtHVFs01tbW9m/f3/OTxPpzsG5IhaL0d3dzerqKnfffTculyvl98UarQtJfSQQCCAIQt52+enYKM2l1WQyvd+t1twNaa5coNUIxsbG8Pl8uFwu3G73upSYUoGTD+7kyAQ2Lw3OpLco5KLtb1HSYhs9FKZHJtfmrjGxOsGhikMMeYb40cSPOOg6mNd7lSSpKDpnOrKNTIqF+vp6jh07lvKzo0eP8sgjj+S81q4iE60lijK4aqMvRCwWo6enB6/Xy9mzZ9UnyVyRD5msrq7S3t5OaWnphtVaxSKTpaUl+vr6EEWRCxcuFPwpKlOaKxAI0NbWpjoM56rJ7MY0Vy5QZo4o42K1E/3SU2LbqRK7EwT4bI6RzXvW6i1NTU0p/S1KmWwmvUUZNqccQ4lKSswlmAwmmsqaChKd7JY0V7F9ue69914GBgZSfjY4OMiePXtyXmtXkEkmS5SNpiBCUpvo6OhQS27zce3czmYvyzKTk5MMDg6yf/9+9u7du+FNWmjNRJZlhoeHGR8fZ8+ePczNzRUlHE/f+BcXF+ns7KSxsZHDhw9v60a704djZaoS0070i0ajapVYT08P8Xg8pXEym5RYsSOTnSip3a7PXDb9Leklq9qoBKDMUsa0bzrv6GSn0lxbHafYUxY/8pGPcPnyZT75yU/yrne9i5deeokvfOELfOELX8h5rdtOJhv1jhiNRrWCS/va8fFxhoeHt9U/kQm5RiZKWmtlZSWrNI/BYFj3PraLaDRKR0cHoVCIixcvkkgkmJmZKcja6VDSXFryOnHixLZHGCtrajdk5d+FrjC6XTCbzSlpHMVeX0mJGY3GlCqx9Ej2To9M0qOGfLCR3uJ2uwG4cuUKJY4SvrPwHfxhP0OeIbj51qJSlD53H2OrY9uu7NpNkUkx01znz5/n0Ucf5WMf+xh/+qd/yt69e/nsZz/Lz//8z+e81m0lk816R9I3+UgkQldXF4FAoKCd5LmQiZLWKikp4d57780qp1qoNJdSKabY1RuNRnw+X9H0GEEQiMViXL9+nWAwyMWLF/M2m0snk51Iu9wuKI2TpaWl61Jik5OT9Pb2UlpampISU/6uWNgpgb8Y70HRWyoqKlhYWODcuXN4V7yc9J+kPl6PIAs4Shxqs6rVYqXStr3UN+xcY6Qsy1tqJsW2UnnLW97CW97ylrzXuS1kkk3viMFgUAddeb1eOjs7cblc25r7sRmyIZNc0lrpyJdMtMdO9/UqprgvyzK9vb2Ul5fnZMWSzbrFxm7UZTZLiSmVTcr8DKfTqZaDFhI7EZlAceemK993hTTe3/z+FL3F4/Hgm00afc6F54hURLbV37ITaa5s5swHAoGCVWgWG7eFTLSVQhsNsFI+yIGBAaanpzly5AhNTU0Fvxm20jS0aa3tdJTno5nE43F17kqmlFqxNIiZmRlCoRANDQ2cPHmyYNf8TqzmKhbSU2LBYJCuri6CwSDXrl3bMiW2HRRbMylmZKIgUxYjVz+xUkcpZtPm13Mn0lzKvnA7S4MLiduW5tqK9ZUGsaWlpYKkWDY7j42e7ldXV+no6MBms21b6N9un8na2hrt7e1YrdYNj61EJoV64pQkib6+Pubn57Hb7dTU1BRcz9iNUcPthlLZZLFYqK2tpa6uTk2JKdP80lNi23lqLnZkon1ALOYxttrkt/IT+/bct9lbvpefOvRTG/a37KRly2bX606ZZQK7QIDPhIWFBbq7uxEEgbvuuquoFzNTmktrlLhv3768RvpuJxWlGCa2trZy4MCBTSvFlPPN9wYOh8O0tbUhyzKXLl2is7Oz4Bu/lkwCgQCTk5Nq05rWzTlf3MmEpTTSKv0W+/fvJxqNrmv2Ky8vV8kl25TYyyXNlev62v6W3qVePCse/GE/J5dPbtjfslNprpfLlEXYZWSSSCQYGBhgdnaW48eP09fXV/RjppOJNrW0XaNELXIhE21ksJlhonZt5e/yuYHdbjcdHR1UV1erExiLEUUoG9nS0hIdHR04nU7cbjc9PT2UlZVRUVFBZWUlDodj25venZLmyoSNrrfZbKa2tpba2toUc0XFBVmrx7hcrg0j6JdLZLLd9WVZ5kdTPyJOnJgQw+f08aYzb8rY3xKLxVhZWcFmsxXNmy1bMrkTZpnAbSST9C+E3++no6MDURTVbu6hoaGCldVuBK2m4fP5aG9vzyuttdn6m0GZk650s2djCZOvt5i21Dpdk8rW6DFXKDPojx07lpKGUEpop6amEAQBl8tFZWUlFRUVOX8Od3pkstXv080VN0qJuVwuysvL1Q1rJyKT3TxnpN/dT9diFy1lLfiiPn489WMuNV3C5XSt01s6OzuZmZlheHi4IH5i23kvyoODHpnkgJmZGXp7e2lububQoUPqFzJfq5NsYDAYiMViTE5OFiStlWn9rTZl7Zz0I0eOZH1D5jPESonAVlZWMpZaF9ouPh6PMz4+TjQa5dKlSzgcDlUX0856Vypz3G43MzMz9Pf3Y7fbVWIp5M28Hey20t30lFgsFlOjlv7+/hT/q2g0WvCNyR1ysxZdo9XZumNTFrdzDFmWeWryKaKJKA6LgxJzCb3Lvbww/QI/deCn1NcphQ8AZ8+eBSiIn1gmZCPy+/1+XTPJBvF4nN7eXpaWljKmdXaCTCCZ5lleXs7LlmUjbJbmkmWZoaEhJiYmOH78OA0NDTmtvd0hVn6/n7a2NiwWC5cuXcr45F/IyESxYRGE5BjTsrKyDTdObWXOvn37iMViarOacjNv1lW+G9Jc02vTdC9182DrgxjE3Igv3/M3mUwZU2Jer5eVlRV8Ph+BQEC9fvlE37Is853h77AQWOCDZz+4q+e/K1FJk6MJAFEQcVldt6IT661KSa1LuclkKoifWCbomkmBsLa2xtWrV7Fardx7771YrdZ1ryk2mfh8PqamppBlmfvuu68gaa10bJTmikQidHR0EIlEtl2tpoi1uWz6CwsLdHV10dzczMGDBze8MQsVmSj6SFNTE5WVlfT39+f09yaTiZqaGmpqajJ2lWvH8SpPlLczzSXLMs9OPUvXYhd7yvZwtOpoTn9bSKSnxJTKRKPRyMzMDH19fZSUlKjXTpsSywbDK8P0LPUQjAVpX2jnlPPUrp2y+NLcS/hjfqJrUVhL/kxGxigYaV9o54E9D6QcA9ZXnG7XT2wjbEUmCnnpkUkWaGxs3DSllMlSpRDQug0ro0aLQSSQOc2lzAFxuVzr5tLnimzJRJZlBgcHmZyc5OTJk9TV1W36+nwFeFmWGR0dZXR0VI263G53Xmtm6irXjuPt6elBEATm5+cxGo15CfnbxejKaHKDjQd5YeYFDlUcyik6Kfb52u12mpqaUqI+j8fDwMAA0Wg06xSOLMv8ZPonRBNRnBYnz00/x15r9s2828V2yeTepns3NH7cX74/5d/Kw99W7yXX/pb0FO1WZBKNRonH47oAvxXKysq2DAmLEZkoFuput5uzZ88SiUSYmpoq6DG00G72250DshmySUcpnl7hcHjTMca5rrsR4vE4XV1drK6ucuHCBcrKytTfFfLpO332SDgc5tq1a4RCIdrb2wHUjbGysjLvB4atzl2WZV6ae4loIsrBioMMegcZ8AxwrOrYpn+X7fr5Il2AT4/6QqGQqrcoKRxt1Ke9fkpU0uhoxGa0MeAZoGOpA5eY/UiC7b6H7ZDJAdcBDrgOZPVaRRjP9d7MdX5LPB7f0koF0MlkK2TzQRWaTNbW1mhra1NTaxaLhYWFhaKm0pR0UTQapaenh9XV1YJ6i22VjlpdXaWtrQ2n08mlS5eyjoK2G5ko+ogyOljbuV3spkWr1YrRaKS1tRWXy6UK+bOzswwMDGC321ViKYaQr0QljWXJDVZE5MrMFQ5XHM46Orld80wUPUuJXCRJwufz4fF41qXEXC4Xz848mxSzzckUTJm5jCtzV3jA+kDG9QuFnfLMKsQxtprfIssyFouFmZmZjHqL3+9XP5c7AbuimmsjFMpxN32IlrYRsNi6jPKlvHLlilpyXMihO5uluaampujv79+Ww/J2NBOtPqKtylOwEyknrW9ZPkJ+rtBGJcoG21jWqEYnB10HMRk271fYTTPgRVGkvLyc8vLydSmxpzqe4oczP8RqtTIUGcJut2M0GplcmWRYGuYBikcoO9mZXkhk0lu6u7uJxWKq3mKxWNTvo9VqVcuCi/Wd+OM//mP+5E/+JOVntbW1zM/Pb2u9XU0mRqORUCiU1xpKxdhG1VrFJpOFhQUg+SEdOnSoKN5i6WSSSCTo6+tjYWFh2xVquaS5MukjG615u4we01M6wWAQt9udMjFRKT/ejjHghG+CnqUewokwfe5bzba+iI8nx5+ke7Gb+5rvo8GRW8VeIZEPWWmv36pjlcPGwwSCAUKhED6fL/kEbbLjDrkJh8MZC2oKgZ2KTIpdfq7MbHI4HOv0lpGREd71rndRXV1NeXk5Tz31FPfdd19Rrunx48d54okn1H/n8753NZnku9Er/lYWi4XLly9n/DCK5byr3dCBFLffQiL9/EOhkFqGm23zYyZku/Fvpo9sd81iQ/uUuJGQr3TkV1RUUFZWtuVnV2oq5f4992d8fyPeEfo9/TgsDupL6zcVtXdLZLIZLjRe4Fz9OfXfSlXT5OQkq95VXnjhBWw2W0pXfiEb/e7EyGSj4yjXRau3HDx4kOeee46///u/5x//8R/5hV/4BdxuN/fddx+f+cxnuOuuuwp2DkajcctinKzXKsgq20AxNRNZltU871b+VsWITNK72Z955pmizh1RNrDl5WU6Ojqoq6vj6NGjed0Q2aS5NtNHtjrXYmE7m2UmIV8RopXiDKXqb6NNscpexetbX7/u556Qh5GVEcqt5Qx5hzjlP0WjozHncywECklWRlGzdYhQ6aokGo4iJ2ROnjypkvPg4CCRSCRFeM6nym6nGiNv92CsPXv2cP78eZ544gk6OjoYGBjgiSeeKHgf3NDQEA0NDVgsFi5cuMAnP/lJ9u3b3kCx2xqZbLW5bKc0WJvWOnPmjFpZsREUMinUjZZpvG2hR/dqoaw9OjrKyMgIR48epampKe91t0pzbaWPbLRmpkmLhUa+a2o78mVZVoXo2dlZIpEIL7744oZCfiAWoH2hndO1pykxJbusVyIrHHYdZtg7TOdiJw2lDRm/a3dKZLLV+iaTierqarUJWSs8T05OqnY5Wn0gW+xECmqnRvZmO2VREASOHDnCkSNHCnr8Cxcu8I//+I8cOnSIhYUFPvGJT3D58mV6enq2RVovqzSXktZSnpSz+ZIqH2a+XyBJkhgaGmJycnLdeNvt2tBnA0EQVJuSe+65R53YV4h1M51ztvrIRmvuhjRXLhAEQRXylfLZ6urqFCG/vLxc1VsG1ga4Nn8Nu9HOHuceupe7qbZVIwgCdaV1tzU62Qmjx0wPFUqVWGNjY0qj39zcHAMDAykpsfLy8k0rDiVJKprxooLdMrLX7/cXtfv9zW9+s/q/T548yaVLl9i/fz9f+cpX+OhHP5rzei8bMpmenqavr489e/Zw4MCBrL8MyuuysTbYCEo3u+I7lV4XXixdxu/34/P51Jknha4SS9/4c9FHNsJuTHPlAoPBsKGQ3z3YzRXfFaKmKC9EX2C+el6NSgAcZgfz/vkNo5OXS2SyGTI1+ilRy9DQEOFweNOU2MtFgM/mODvtGFxSUsLJkycZGhra1t/v6jRXNqXBW/l7bQVtZLIdeDweOjo6qKio2LCbvRhkMj8/T1dXFxaLhebm5oISCaz/bHLVR7JZs1jYqehnnZA/m8A6ZKVRaKR7ppvro9cpsZWw6l3FZrNhtVhBgFn/LJ6wZ92M8s3Oezm4TJV985RttudcLGynodBoNKakxLSNk5OTkwApQv7LVYDPhJ12DI5EIvT19fGqV71qW3+/qyMTo9G4aWTi9/tpb2/HZDJt6O+1FRR/q1w1Da19++HDh2lubt50iFWhNBNJkhgcHGR6eppTp04xOztbkHXToU1zKfqIVgfa7pp3WporW/ijfno9vTRWNFJtr8ZUbsIb9HLacZpoIMrqyipEwVnupMJWgZXM39VM36Fh7zAfeeIjHK86zscufwynZXupzGKL14VY32az0djYuGFKTBRFYrEYdrt9y5TYdrEbBHhI7m/FjEx+53d+h7e+9a20tLSwuLjIJz7xCXw+Hx/4wAe2td6uJpPNxHHFtj7XtNZmx8kWsViMrq4ufD5fVjpFoTSTSCRCe3s7sViMS5cuUVJSwvz8fFFSaEo0NTIysi19JBPysczP9Rg7jSHPEJ6QR7XsqLHXsBpZxVXp4szxMylCvtvt5qUrL6kd+YpWsNF1+Zfef6FnuYdBzyD3NNzDzx7+2W2d4502zyRTSuz69esAakos1xLubLCTAvxm1ysQCBRMA82E6elp3vve97K8vEx1dTUXL17kypUr7NmzZ1vr7XoykWU55SZIJBL09vayuLi4rbRWJuQSOSgDtOx2e9bpnkKkuVZWVmhra8PlcnHu3Dn1iaxYeowsy6pl+Xb1kXTs1EZfLLLaaN1gLEjPcg+RRISRlRH159FElN7lXg5XHsZusqdsjNqOfGXuiCzLLC4uYjKZ1I78Ye8wPxz7IZIssRpZ5StdX+G1e15LubV8W+ef72cw5UuWSjeXNa/7XbEjH6PRiMlkoqGhgbq6upSUmFLCra0S226PVSKRKLrIL0kSsixvmeZqbCxeocbXvva1gq532zWTzaBc6Hg8jtlsLkhaa6PjZLMhKyJ/rgO08tnwtfPoDx48uK75sRhkEggEVO+gQgr72sikWJvO7YhMjKKRkzUnMxo6GkVjak/GTWTqyL927Ro+n4/Z2VlMJhMVFRV8ZeorzPnnMIpGrAYrPcs9PDXx1Laik/TrHoqFmPHPZG2AGEvE+PATH0aWZb7+M19fZw+z0/NMtCkxWZZVL7aFhQXVDl6rt2SbEtspXQY27zgPBAJ3jC8X3AGRCSQvvJLWamlp2XQOx3aPs1lkokRDS0tL27In2a5mkkgk6OnpYXl5ecN59IUer6udzy7LckGF/XQyiUajGAyGgqcUdlqXMRvM3FWz/a5kRcg3GAwcOnSIkpISVldXuTFxgx+O/pBIJILRYMRmtOGL+fKKTrRk8sjAIzw9+TSfePUnqCvdugv6h+M/TFrFyPD42OM8dOChlN/fzoZCQRAoKyujrKwsxZ7E4/EwMjJCKBRKSYk5HI4N95CdSHNlSyZ3imMw7HIyUcTx/v5+PB5PwdJa6dhssw8EArS3t2MwGLLuXUnHdjSTYDBIW1vblsct1BCr9P4RxRyzkNBOhlTGJBsMBrUBMNcGtt0EWZYzVmjlugagXpMX+l5gjTVMZhM20YYsycgJmbbZNv7hmX/gXSfeldM100Ymi4FFfjD2A6Z8U/xg7Ad84OTmomssEePvOv5OPce/6/g73rD3DSnRyVZpm0IgW8JKt4PP5GqwUUpsJ6q5lGNs9l50MskBW30p/H6/mgLIx2dqK2wUmShTCfOtYso1FZXLTHiluiUfZOofmZ2dLcrkP4De3l48Hg/nzp1DlmW8Xq9araOd915eXp7zNb9dAvyV2St8o+8b/PY9v01LWcu211HO3xf28fzU84TiIWKJGCHhpuGpCAkhwZPLT3Jq8hRd17o4WXmS5prmLaclasnk8bHHWQouUV9azw/Hf8gb975x0+hEiUoqrMnouN/dvy462c1WJ+muBkqVWKaUWD49Z9kim+gnGAzqZFIIzM7O0tPTo4b+xSISWB85aMtvT5w4kbcRWrZpLlmWGRkZYWxsLOvqqXw1k2AwyI0bN9b1jxQq4tEiGo0CySeuixcvqte9vLxcFaWVp8fe3l4SiQQul0sll2y/Azud5oolYny97+v0LPfwxNgT/NJdv7StdbTnXWou5SMXPsJjI48RlaIprxMFkaOVRzFVm+jx9FBuLqch0aAK+eXl5Wq0p7XWV8hEiUpcVhe1JbX0ufs2jU60UYnVmIyCZOR10Umxq8WgMHqGNiXW2tq6LiUWDAaJRCKEw+EtU2LbRTYje++k+e+wC8lE67Z71113MTQ0VPTNQRuZhMNhOjo6UspvC7H+Vht+LBajs7MTv9+fU/VUPmSiREANDQ3rIq9C94Ssrq5y48YNAE6fPo3Val0XUZlMJmpra6mtrUWWZfx+f4qgarPZUqKWnSjfTEemzfJ7I9/jRxM/oqmsieemn+P1e19PqakUg2jIuSdEO4/lDXvfwBv2viHj6xJSgs+89BnmAnNcN13nLSfewuHDhwkGg2r58ejoqCrkV1ZWqpHD46PJqORI5RFEQaTSVrlpdPLD8R/St9yH3WgnFEtGSHajfV10stMCfKGQnhK7cuUKLpeLQCDA9PQ0sizjcrnUtFghRPFsoh+/33/HzH+HXZbmUqq1jEajmtYaHR0t6rwRuEUmbrebjo4OqqqquPvuuwu2WW0VmSgTIEtKSrh8+XJOZYnb2fRlWWZsbIyRkZENI6BCCvtKlLl//34GBwezNoV0OBw4HA716TG9lFa5uZUncOXvdhKxRIyv9nwVf8xPOB5mObTMD0Z/gCiIWI1W3nvsvYjC5u93KbiEQG6fY+dSJ71LvRyrOsb46jjPTj3Lzxz6GbUjv7m5mWgsyqd/8mnuDt2Nb8wHwDPXn+Fb09/CgAF/NDkW1ma0Mb46njE6kWWZv7n+NwDE5XiKI4XZYOaFmRdUMtnNaa5cUV1dTUVFRUpKbHFxkaGhoZQhVtuZfQPZ6TK6ZrJNKBtOc3NzigttoaYtbgZRFHG73YyNjXHkyBGampoKelNspmso73vv3r3s378/5+PmGpnE43G6u7tZWVnZtOGyEGkuWZYZGBhgenqa06dPU1VVxeDg4LbW0tpuaD2xlpeXGR4eVnPesVis6A8fWnxv5Hv0ufuotdfii/ooM5fx/dHvU19ST4WtgtGV0U1Lb+NSnC+2fxFRELlHvierzz8hJXh87HEScoJScykVtgqemniKVzW/KqUA4MfTP+abo99kpnGGzz/4eZ5++mn8Nj+JRIJIOMJ4YByTyYTZbMZldtGx2LHuWD+Z/gkdSx0cqTjC/33T/113fvUltwxNdyIy2YljaDf69JRYptk3DocjpXEym/PLJs2103Yq+eK2k0l6Wqumpibl91tZquSLWCzG8vIysVisoK67WmSKTCRJYmBggJmZmbyq1HIhk430kUzIN80Vi8Xo6OggFAqtSxfmG/Gke2IpOW+3200wGGRoaIilpSV1Rkm2I3llWeaJ8Sc4UX2C+tL6LV+vRCWxRIxScymrkVW8YS+r0VXiUhyHxcGLsy+yr3zfhtFJ+0I7/e5+ACrFSi5wYcvjKlFJU1ly1EBdSR29y71qdAJJwvmHrn/AH/VzZfYKV+euAvCWE2/hviP3IUkS/oAfr9fLineFNf8aFdYKhoaGUoofPnvts4TjYXrdvUz4JnjtntdueF47YdeyU6m0jTb6TLNvFKPKrq4uJElaVyWW6ZpsRSahUAhJkvQ0V7YIhUK8+OKLiKK4YbVWMcfqrq6uqkOsampqimZdkK6ZhMNh2tvbSSQSXL58Oa8cbLZkspk+kgn5pLn8fj83btygpKSEixcvpqQBirHZaHPewWBQnezn8XgYHR3FbDar6bDNmtcGPAN8rfdrnKs7x389+1+3PNefzPyE0ZVRbCYb3rCXuBxneW0ZURAJxUJU2arod/czsjLCQdfBdX8fl+I8PvY48s3/u+67znvk92x6TCUq8Ua8WE1WvGEva9E1+pb7+I+R/1Cjk6cmnqJ7qZumsiYWA4v8Q9c/8A7TOxBFkdqS2uRiDvi69+t0xjr5owf+SN0UlTTiuDzO89PPU2IqIZKI8FdX/4oHWh7YdFJkMTd65eFmN+kyVquV+vp66uvrVZ3P4/GwtLS0aUosm1kmgJ7myvrgRiO1tbXs27dvww+vGGkupYeiv7+f/fv3q1+CYkG74Xu9Xtrb26msrOT48eN56zJbkUk2+shG624nMlGaHpV0ZfrGo0Q8giAUhVi8MS8W2cKx5mM0NzeraQm3283UtWssdnRQ7fFgrarCcvEi5kuXEEwmZFnm8bHHWQgs8NLcSzy48mBGAtBidm2WVmcrLWUtLAWXGFkZocJagUk08brW11FhrWA5tMyLMy+yv3z/uuhEiUqaHE3IyLQvtdO13MXl0ssbHjMmxTAIBk5Wn1R/NuodxR/z4w65CcaClFvK+Yeuf0CSJWxGG5W2Sl6cfZFT1ad4jfAa9e/Womt89MmP4ov6+OlDP835+vNqR34gEOAz3/5M8kFOBoNs4NrsNR7tfJS3HntrRp1gJyITKC6ZyLK87T4Trc63Z8+edSmx7u5utXEyHA5v6ctlMBjuqL6r20omZrOZAwc2t3IodJpL21WudLOPj48XNZWmpLnGx8cZGhri0KFDtLS0FOTG2ywdla0+kuu6mZAtaRXTOTiaiPLjxR9TG63lSGuyUklJS1TF4xh7epCmpgiazUQGBli7cQPv888Tf9vb8Bi9XJ29yn7XfmbWZvjh6A85cPYAAiCMjyMODVE+MEDE6URwOAjXVDK+Ok6rsxVJlpham2IxuIhZNNNc1kwgGmDNuEaZuYwBzwAjKyM0ljYiyRKl5tKUqMRusqvX8MnJJ7nQcgGDmPkhw2q08jsXfkf995BniLaFtuRseQRsRpsalSiW9SWmEtwhN4+7H+dX+VX1b7/Q/gXWomsYBAOffP6TPPpzj6qfUbu3nXZPO3azHYvRQiKRYC26xueufY5yd7m6KVZWVqqls8WOTHaKTGDzzvRskZ4Si0Qiaun70tISsiwTj8czpsQUK5WdKDYoFHZVNVcmGAwGtT8hXygzOUwmU0pXeTFTaQrW1tYIBALcfffduFyugq27UWSSiz6SCbmkuRKJBN3d3Xi93i1Jq5hk0ufuYyY0g1/0rxO+DVeuIE5OwsmTlIoipQBeLxULC0wuLfGt5W8z7Z3G7rTjMDh4fup5Htz7IIcX4hh+/GNkSUIIhbAtLWGIxbC+9rV88NwHiSaitC20MeOfwSya8UV9HKs8hsVoYS26BiSrpWZ8M1yfu05civOfjv8n2hfa6XP3UWYuYzWyCoDdYKfP00fHYgdn685m9Z4fHXyUtegah1yHGPYO852h7/DExBP4Ir6Uvo+YFKMv0EfbYhvnG86zFl3jr176K2RkEnKCJyee5OrcVc7Xnwfgs9c+SygeAiPEosniEVmQGQ4NE2uJ0WBvwOPxqC4JLpeLSCSSdwPtZlAin2JGP8o+UIxN3GKxqCmx/v5+ZFnGbreztLTE8PBwshDC5aK/vx+73b5jKa4///M/5+Mf/zgf/vCH+exnP7vtdW67AL8VCrXRz8/P093dTXNz8zpvr2KO1Q0EAgwPDyNJEvfddx8Wi6Wg62cik1z1kY3WzWbTD4fD3LhxA1EUuXTp0pbvr1hkEk1EeWn2JcwGMzEpxtW5q7eEb1lG7O9HrqwE7bVwubDNzRFNzLNkXuJ403GEhMD48jiRSIS/ffyv+Z2pBspMZmz79hFfWkJyuSAWQ+zowNnyVuImmauzV7EarFxuukzXYhena0+rQriCpeAST008RUJOMOIdYWRlhFJTKQk5gT+adHqQZIkSUwkj3pGsyGTIM8TTk09TbavGIBoos5Tx3eHvste1l1c1v4q2hTb2l++n2l6dLH1fcmMxJj8fJSpRkB6dWAyWjFVooiCCSEo3+Zeuf4nxmXFeb309AwMDTE1NFaUfaKemLMLO6DI2m409e/akpMQWFxf5n//zfzI6OorD4eAP/uAPeOMb38jFixcLPgAP4OrVq3zhC1/g1KlTea9128mkENMWN4O2aurkyZPU1tZmPEYxIpPFxUU6OzupqKggEAgUnEgglUy2q49kQjaRidfrpa2tjZqaGo4dO5Z1/0gxyKTP3cfY6hi1llrMNjP97v5b0YkgIFssCGtrqX8kSSAIPL56jcXoImKJSPdyN5OrkxxwHWBemmQuJBMSWwj29mIymTAajQRdLuzLy+DzcSM0RL+nn+ayZoyikQZDOdfbvstbZkqp2XsC6eBBZFHksdHHiCQiiILIi7Mv8s4j7+T1ra8Hkp/bN/u/STQW5R33voNqR+bKvmHvMG0Lbfzc4Z9DFEQ1Kql3JSvPauw1DHuHeevBtxKviNPn7mOPcw9/+dq/JBaN8fzzz3Oy+mRKVKIgPTr56tu+mtV1Xwou8Y/9/0goHuJUyynuP3E/kJxAOjAwQDQaVTvyKyoqKCkp2XZksVN9LIonYDGRLsBrU2I3btzg//2//8dnPvMZJicnede73oXf7+e5557j9OnTBTsHv9/Pz//8z/PFL36RT3ziE3mvd9vJZCvko5lkWzVVyEmIkNwchoeHGR8f58SJE5jNZnp6egq2vhbKpp+PPrLRuptt+oodf676j3bdQm0MSlRiES2YRBM2o42AFEiJTqSzZzE+8ghyMAh2O8gywtQUclUV5ho7l2UHgWiAJwNPEpNieEIe3tz6aspW7Ryo3UvcaGRqaopoNMrkyAiGWAz/0ACPev8dSZKwGCwwNUXtyBg90jzP9n6J9zx+hMTddzP04N30LvfSUNqAQTDQ5+5j0jfJwYqkwN+52MlTE09hC9j4Zcsvr7N2B5BkiW8NfIvupW4OVxzGaXHy3PRzRBNRhleG1ddFEhEeHXyUhJygxFRCx2IHV2avcLbyrJoi+lLHl/BFfRmv5ade+BSP/OwjWV/7r/V9jeXQMjIy31/4Pg+eeRCXy5Vira/oBNqOfOW/XBr+Xkkje41GI/X19fy///f/kGWZrq4uDh06VNBz+OAHP8hDDz3E61//+lcGmWw3alheXqazs5Pq6mqOHTu26QdXyMgkGo3S2dlJMBjk4sWLOBwOvF5v0TQZJTK5cuVKXvPZM60L6/2WlEhvdnZ2W3b8ypra/58JS8El1epDi2AsqIrYCvrcfYysjFBprcQdcxOLJse6aqOTxPnzCFNTGNraIB4HWUauqiL+0EP82s2nvT997k+RZIm6kjpC8RAHm+7iUMyBODqKce9erFYrFkGg0eVi7eBBnjOvMbo8ijfsZWVxAfv8LCbRiLHUyUtlMm+LVGN74XmulowQcUYosyQtciS/xIuzL7LftR8Bge+NfI+16BqLoUXaFtu4b899665H91I3nYud+GN+fjD6A375rl/mnUfeSVxaH7W/OPsiHYsdHK44zLB3mD957k/44uu/qH6O9aX1nK3NnEY7Unlkk08uFYuBRb7Z/03sRjsm0cRL3pcY8Y1wt+tuILUfSKmsW11dxePxMD4+Tm9vr9rwpxXyN8JOpbl2aspitvbzgiAUJA2lxde+9jVu3LjB1atXC7bmbSeTQqe5tFbqR48epampacu/KZRm4vP5aGtrw+FwcOnSJfWpq5iazOrqKrFYLC99JBMyDbKKRqN0dHQQiUS4dOlS9v0xkgRjYwgjI9TcuIHRZILz51P1Cw2UZkCLwcKvn/l1taRWlmX+8qW/JBAN8Mn7P6n+fDGwSIW1AhmZmBzDkDBgE22YDWaWgkvJVJfNRvzd70Y6dw5hfh4slmQK6qYf01JwiUcGHsEkmnBYHKxF1/hS59/zztf/E+ZIBHFsDIvbjQzI589je9WreE2JlfrGeiLRCF1P/SsvBeZ5HcdxrFkot5QStgnMWMP0TrxExYVXEY6HAai0VSYJ0DtCKB6ibb6Nvc699Ph6+P7o97nYfDFloJYkSzw2+hgxKcb+8v10LnUyvTbN+068b921Wwws8u9D/47L6lKvz435G3zqpU/xnpJkD8u7j76bdx99d3af3SZQopImRxMiIgu+Bb428DXu3nN3xtcr1voVFRUcOHBArW5yu93MzMyoHlgbjSN4JUUmxTR5nJqa4sMf/jCPP/54QUuPbzuZbIVcogZtVJCLWWIhIhNleFemKYzFmIao6CPDw8MIgsDRo0cLur529ogoiqytrXHjxg0cDgcXL17Memodsozw9NOIP/oRRCKsLPZhWhzAOTkJb3kLZIiiupa6GPQMYhAM9Lv71QmGPcs9XJm9QlyKc3XuKhcakt3ir2t9Hfc1JZ/me3p6KHOW0dyUHCtrMVpupdWMRqQjR+DIEV6ceZFIsJdX82oAHr7xMMuhZbWhr9xSzpB3iG8s/Yj3PPQzSNPTrA0OErfZKL/vPjCZMAMnqk8Ql+I8Zfwi3hKJEmcFr4vvJxAMsrbq42pgioAniGdxBo/No+pmJtHEsHeY9sV2IokIDrODanM1vZ5ers9fV98b3IpKmhxNOMwOptem+cHoDzhRfWJd78q3h77NUnCJg66DJKQEM/4Z4nKc745+l9cfeX12n1kW0EYlBiG5KZYaSvnR9I8YcA9wuPLwlmtoq5u0kxKVcQTpxp53qpHkRse5XWRy/fp1FhcXOXfunPqzRCLBM888w+c//3kikci2orNdTybZaiarq6u0tbVRVlaWEhVkA0Uz2Y6FtiRJ9PX1MT8/z5kzZ1Tn0fT1C0kmWn3k9OnTqhtvIaFNcy0sLNDZ2UlraysHDhzI7RpNTSE+8wyy00mwwsF3bTeosq+wr+0aYksLibtTn2JjiRjPTD2DQTCQkBP8ePLHHKk8goDAo4OPEowFERF5dOBRztefRxREREHEZkq6J1gMFqwGq/pvgCfHnwSSpAOwEl7h6amniUtxjlUdQ0bmkYFHQEZ1xQUIx8N8qeNLvOPIOzAeOECYm9U+ad+t9oV2+gQ35Vh5KtLLvfa9VForqHQ6efvKMncdPMds0zFWVlZIRBOIdpHm6mYChgBt821q06JFtBCUg3xv+HucqzuHUTSmRCUOc9Jao8nRROdSJ91L3ZyquZX+CMaCPDH+BJIsMbIykrR2iaxiEk0E40EemXuEn+Pnsv/sNsG3Br/FfGAeg2Ag6Asmr1cijByW+Vrf1/ij+/4op/XSJyXGYjG1I18R8m02G/F4HL/fn5eQvxl2Y5qr0Hjd615HV1dXys9+8Rd/kSNHjvC7v/u7237/t51MspkDv9lGr52RfuDAAVpbW3P+kikXL1cyCYfDtLW1Icvypmkfpcy2EE896f0j2kquQt5cylqjo6NMTk5y8uTJbc11ESYmwO+HPXvoiI4wL64RMMh0laxxur8f7r6bBf8CX+39Kr925P2MvPg9hvu+Qysu4tWV9CY66Hf3I8kSV2avUF9Sj0E00LbYlhKdpJ+3gqXgEtfnryPLMierT1JTUkP7QjuLgUUArs1dw2gwkpAT2E32FA3CYXawFFpizj9Hc1lzxvcXl+I8NfEUoqOM1tqj9C508sLKC7wlcRDB58Oy/zAH3/xODlRWIssy/XP9/J+r/4f7ovfx7OyzjK6NsmJbwWA0EAgHsJvsdC93c2P+Bvc03EPvci+di51EE1EG3APqcT0RD4+PPZ5CJlajlQ+e+yD+mJ9oIsonfvIJysxluKwufGEf7WvtzPvnsxrRuxVOVJ3gF0/9YsrPxsfGaWxq5O76zGmuXGAymaipqUkR8sfHx3G73Vy7di0vIX8z7GSaa7Pj+P3+bemR2cDhcHDixImUn5WUlFBZWbnu57ngtpPJVjAYDMiynHGzjMfj9PT0qFP7Ms1Iz/YYkNsXSbGrz1bgh/xD6Ez9I0pDZ6GfqJRocHZ2NqeU4TrIMggCQTnKc7FhSrBgxshPxBmOJiIAfKnzS3y9918pu9FDaGoEgzWALWGABTdSrcDTrsdZTvgIxoI0O5oRBIHZtdmU6CT1kLc0uBvzN1gJryAj07bQxoWGC7w49yIVtgqMopHrC9f59dO/zpPveXLdECpIdo9X2Db+XimWKC1lLVBuprwkwRPuFS4lyii//36k8+eT/S0kie7F5ReZjEwybh7nwTMPcsR9BH/Aj9/vJ2FMYLfbcZQ6sGNXj//AngeQZZnF4CLLwWWOVh5FEAQq7ambjSiIvKr5VQB8rfdrrEZWaSxtxGQwYTfYmfZN89Wer/LfLvy3bXyQqbiv+T7ua75VKCDLMj8K/YjL92xvtPVmUIR8l8tFLBbj1KlTqk3J+Ph4MrWZoSN/O9ipNNdWkUkwGKSlZfsTO28Hdj2ZKLn5eDyeUqWkzD5RntDz6eHQkslWTziyLDM+Ps7w8HDWdvXKl3O7qS5t/8ixY8dobGzMuHahyCQUCqmps7Nnz26fSAC5uRlsNjq8fUwZvdTIpZQmrIzFp+lqtlDum+K7I99lJejmH4JPcLSikVpjNV5MkDDjXF7k+YEfMGMKU2WvUiOHmpKaDaMTBUvBJToWO1QdpH2hnVAsxGJgkaNVRxEQ6F3u5drcNd60/005vzc1KtGk2er23UVvWS/PHL2Lhw6krjmxOsHzM89TX1pP73Ivb9j7Bn7uaDLtFI1Gee6559i3bx9er5elgSVemHiByspKfrrxp3E4HXz2+mcJJ8K8fu/r2Vu+d8PzCsfD/H3n3xOOh9USYEmSSMgJ/rX/X/n54z9fkOhEi0KXe2eCstGLopiVkK+QSy7kthNpLsX9+HZpJpnw9NNP573GricTZbPU6iZzc3N0d3ezZ88eDhw4UJAxntn0mmi1ivPnz1NeXp7V+vmQyVb9I/kSVTo8Hg9tbW3U19eztraWf/pgzx78F8/xk5c+j4MQ9hWwBsMY66v4F087nseeY8G3wB4qmBBncYkhqnASIAoGEAwGVn2LhGwyvoUlfOEwmM3IFRUIJSU8P/18CploqwOVqORoZbI4oW2hjWHvMK3lrYhCMvXosrq4vnCdu+vvVr2sssWwd5hZ/yzRRJSe5Vt9RAICL86+mDLSFuBHEz9iJbzC8arjDHoHeXzscc7Unkmp3Gpubk4ZJet2uxkcHKTH28Pz3ueRDTI/GPoBv3bu1zb83ofiIZwWJ4crbong8UQcBw5qSmpYi61RR3HIpNjeXJnW30jIn5+fVyd0KsSyVUf+TqS5lH3mdmkmxcJtJ5OtnmQEQVB1E0mS6O/vZ3Z2NuPsk3ywlUju9/tpa2vDYrHk3MuRLVmlIxt/LW3VVb6YnJxkYGCAw4cP09LSwvT0dP7riiLtJ6oYXa2hbC3KuMmNsbqKRQEizNGz1oPVYMUQBltMwCcH+BX5DFW2ckwmI+LKMB5TKatPPo3B60cWDQhSDLk0QPRXf576E+/MeFhtVKKWOSMz5B3CbDCzEvYy55+jz93PoYpDdCx2qAI9JDfHCd8Erc7WDd9aq7OVXzz1i0iy5hr5/Rh6eii5MYW9+4tIp0+TOH+eieAsz0/8mHpPFMN0Gy1WI73RDtoW2lQ/LLj1eWpt9WOJGE89/xSWoAWbZOOxnseoXqvmeMPxjLb6LquLb7z9Gynn6vF4GBwc5OLFi1l/dLlA+Z4UOzLJZr/QCvnaCZ2KkO90OtUqsXQhf6fKj0Enk9sCo9FIIBCgq6sLWZbzngGSCZuVBy8sLNDV1ZXR1ytb5FrRla2/ltLVnI9FiVKRtrCwkKI9FWLaIoAkwLGD9wIwPDxMPB7nRF0dLyy/wFpsjcMVhzHaojROB5gW1vh2pIu3Lx7EHo1SFgjR0jOMeTaIvOcgCEKy3HhmBvn//huhN34g4zH7lvtYjawiCAJLoSUgOXa2xVbP8ckwJ4fm+UJpP0ulbkoFCw2lqdYzj489zp/+5E/59AOf5nJT0hI+/VpYjVbuqrnr1g9WVzF/5W8Ru7uRbTaExAKGa9cQBgf5UcsSKy89Q8NsjARglUGoNfBDwz9x5qfPbHqdO5c6GV4d5lB5K3bBTLc/xqx1lhPCCUZGRgiFQuoGWVlZmbHSqVjmmunr347IZDNkmtC5WUd+NrPZ84US/WxGjHfalEW4Q8gEoKuri7q6Oo4ePVqUDzsTmUiSxNDQUF7VTAqyJZPN9JF8186EaDRKW1sb8XicS5cupQwoK5SP1r3N93Kp8RIDAwPsd+1n7969lNSV8Og3HkUUReYD88njlRmIhhM8bujlNyJ1mGQTKy0tWF96ibDVihgIYDSZMJlMiDU1CAsLGNrbSVy+Nf9DOefDlYdxWp1JU8b+AYTFBWRTHYaOEA0DMwxWiUyVrSFGo8wujTA61c7RqmQ6LCEl+LuOv2NidYIvdnyRS42XsnO4fuEFxO5upEOHkqk4AJ+PwE+eoq+/A1soyFi1LdmsKcsYgkEWX3yCsfuGaSnbq56/FnEpzpOd30Ls7sY5GQBJprnGQe/hGG859hYuHr5IKBTC7XarczOUDVKJWkw357UUM2rYSc1ku0jvyJckSRXyJyYm6OnpwWQyYbPZWFlZyXoEb67YSpdR5sncSVMWYReQyWZfPsXjKhKJsGfPnoI35mmRTiZKt3c4HObSpUt5h5y5ajK5+Gttl0x8Ph83btygvLycc+fOrWtEzGfaohbaEb4Oh4PS0lJWIivUldbhMDsQuPkdKJWpC4dxRkXWzjxI9cEzVMoytv/4DxJWK9Gb4wiCwSBGWcYejeL3+bBk2GRqS2qpCxsx/eX/wtDRAYkEwuoqgtdL6DX38WjjEIvmOKWylbVEmL+78v/jwVM/h9lg5onxJ+hZ7qHKXsW1uWu8MPOCGp1sBkNXF7LdntqIWVZG2UCQj/ZECdbsA+33KBLBuOxhz/deIHrYi5hh1ELn2Av0P/sIsbVVBuyWZGS24sPTM88zVV9j7xs/hs1mo6mpiaamppQNUplRXlZWhtVqVYXfYvVnFNsevtApKK2QD8l5Iz09PcRiMTULsl0hfzNko8voaa4CQtnMlQ2oWCN1FWg3e6UB0ul0cunSpey7vTfBVpYq2c4f8Uf9Kb5UyrnnuunPz8/T1dWVsWNfu26uA7KCsSAl5lvheSAQ4MaNG9hsNi5evKj25RyrPsa/vePfiMViG95YMkAigdzUhLG7G0NtLVabDdluR56eJlJRQZsoEn/uOfWG1xK28Z/+CcPVq0h794LNhtjbizA3x4vzV3nhYIgEMk6sCFKC7uAYj489zpv3vZm/6/g7JFmi0lbJ9Nq0Gp1s+f5NJkRJIv2KCfEE+3wGJGcZGG6lZ4XpFcSJRaTP/zUmp5PTNhuGujokjTNsWVs3rx6Jk6hoAvnW06w4vkS9YwDeqDnO0BCWJ56gbmaG2j17iL/2tYQaG9Wu8lAoxE9+8hN1A62srCxYf0axIx/lGMVMQVksFnXM7p49e/IS8jdDNqm0YDCok0khsLKyQnt7O06nk8uXL6vOv8WEstlPTU3R39+/7QbIjbDZhr+8vExHR8eW+sicf45HBx7lDXvfwIGKW7MmcokgtI7GWxUx5BqZPDn+JI+NPMYfvuoPKbOU4Xa7aW9vp6GhgSNHjqzTd+S1NYyPPYbpyhWIxUjcfTfS0aNJq/hYDLmxETwepOpqxHgcYWQEwWhMppCqqzH+3u9x6cEH1Zt+ZmaG6Ogo+HzM1tXR8vTTyR4Pm41lMcx4dZSzMxb+rXqZRVnGjBEDAo64AZ81mdoSEOhZ7lENJl1WlxqdVJPZGl6BdOYMhvZ2CATgZr5bWFxErqlBCgYR3O5k5AIIHg/CxASy0Yh07BgJoKS3F/OnP03kc59TPcOOzkQ5NeRCSptIKsxakIkSvvlvw09+gvkzn0kew2xGeOYZDI89hvB7v4f1/HmMRiOTk5McOHAAj8fD5OQkvb29lJWVqWJ0WVnZrraHTyQSRZnpoYWSgspXyN/qfWxGJpIk7XhpcCFw28lE+wHIsszk5CSDg4McPHiQPXv2pFRzFROiKDI1NUUwGNy2G+5W66dvzLnqI9fnrtOz1IPT4mSfa5/arJdtBBGPx+ns7MTv96uOxpshF80kFAvxncHv0Ofu4+mJpzljO0N/f/86s011zXAY0+c+h/DCC8mN12DA9PDDYDCQOH06aRMfDCLOzyMdOEDsF38RsbsbcXoaTCaiH/oQ8be8BQGSN73JxKHvf5/I176GaXERQZYRgkG8e/eC2cwjtTN0713FvSrR7oqyJoIJmSARMINgtNKz1MNfvvSXxKU4BsFALBHDLJpxx918seOLfOzgxzbdMBIXLiAODGB48cVbzsSlpSQeeogEYP6Lv0AYG4PSUoTxcQRJQjp8OPle43H8DQ04Z2cx3BwlDEB5ebLx82bzp3odw2Ek5UEgHMb0d38HPl9yvZvXWBwexvSlLxE5c0b1tXK5XLhcLvbv35/SnzE1NYUgCClRSy4b9074Zt1Ob650IV+rU42OjmI0GtXrtlVHfjYNi7Is65rJdqHoBV6vd91o22KTSSgUYmVlBaPRuE6ELhTSNZNc9ZE5/xzt8+3UldTRt9zHqHdUjU6ySXMpaTSLxZL11LZc0lzPTT2nTg/86rWvIlQJ3Hv3vetcCRQyEa5eRbx6lXhrK2JJCfj9CNPTiB4PkiwjHT2K2NGRtGM5cAC5ro5EXR0JQBwYgLTvg/Gb38T013+N0e1Ozm03GBCCQaoGBujc4+R58yJLhjCjTpEL80ZEmxlBFMFoRK6oRK6tIRyPMOOfwSSacIfc6tpmQ3LY1mzTLI32TQjfaiX2i79I4vx5xLExMBpJHD2KfODmgK6yMoxf/zri8DBCaSlSY2NKxCGIIjIgeL3qz+L334/xX/4FcWICqakJRBFheRkMBuJvSjZFisPDyeq2hoZbhCMISPX1iBMTCGNjyKWl64hQ258hSZIa4SmzarT28FuJ0Ts1uGo3uAYLgoDdbsdut28o5Csd+UrEp10zG18uQE9zbQdra2u0t7djtVozdrPnO21xMygpJrPZTH19fVGIBFI1k+3MZ78+d53V6Gqy4c09yIuzL6rRyVZkok035WJTn21kEoqF+O7Qd7GIFmxRG2O+McInwxntbVQyGRpCjseRrdbkzbW6Cjf/Lba3IwSD4PEkhfP5eeQ9e9Q1ZLM5mUpS4Pdj+uIXERYWkAUB2WBAAGS7HSEY5KqvnWizlbKYxLONcX7b9HbeWXEIETCeOoXj1ClcFRUYjAa6lrqIxCPrzttmsmHxZuGyYDIhnTmDdObMul8l7r+fxP33A2D+sz/D8MQTqdcmFkuSgGZCprxnD9H/8T8wf/aziBMTyWjH6ST2C79A4o1vJCso13yTzV4URZxOJ06nk3379hGNRtWoRStGK+XH6ffoy8XRdzsd8JmEfKX8OF3Ir6io2PIYwWAQo9FYlMmsxcRtJ5NIJMKLL76odrNn+sLnM21xI2hTTEePHmV1dbVoM0fgVvSQrT6ihRKV1Jckx7M2OhpTopONyESWZSYmJhgaGsp6tosW2Womz009x6B7EEfUgdFspKmmiSennuSNh96oDoTSrinLMgmzGaNmg5NlGSKR5FO32Zwcs+vzIS4vI8/O3logkUAIBpH371d/ZGhvTz6B3xzPq85JiccZqzTwQm0Mp+TgmitKpNROx6sv8Z9Pvp/V1VXcbjdj4+P09PZu2asxtDKU0/XbDPGHHsJw9Sri8DBSTQ1CJELp3BzSxYskLqWK/YnXvpbQuXMYrl2DaBTp5ElkzWcpHTqE3NyMODaGtH9/MjqRJMS5ORLHjiHv3Ys8P59T5GA2m6mrq6Ouri6lq3x2dpaBgQFKSkrUqMXpdL6iIpOtkKkj3+PxqEK+wWDAbDazvLyMy+VaRyzFdEUuJm47mVitVi5durSp2GS4WRJaKCjagc/nU1NMfr+/qGQiCAJLS0sMDg5m3T+i4PrcdbwRL7UltcSlOBajhVgipkYnmSIISZLo6elheXl5XdowW2ST5grFQnyz65uEfCEaKxpxVbiQZIl+dz9PTzzN2w69TX2tslYwGCR24gRGpxPj4iJyXR2UlSGsrkIkQuLAAeTq6qRe4PEgTE/D8DCC3Y7g8SDt30/irlvNgsLCAhgMSV3h1smDJPHkXpnVihJil06zutQJwH+MPcb9e19Li6sFl8vFgQMHCIfDuN1u3G434+PjGI1GlVjSO8wLAen0aaK/+7uYvvhFxKEhZIOBpdOnqf/9308tH1bgdJJ43evW/xzAbCb2q7+K6dOfRhwYQDYaEeJxpNpaYr/yK3DTLHW7m1Mme3glaunp6SGRSFBSUkIikSAUChUtur8TCUt77RSbnN7eXkKhEIODg0QiEcrLy1OEfIVMioWHH36Yhx9+mPHxcQCOHz/OH/7hH/LmN785r3VvO5lAMjdYyGmLm8Hv96ulqtoUk8FgIBJZn94oBOLxOKurq8Tj8Zzns/ujfkZXRrEarIyvjqs/N4pG5v3zLAQW1kUmkUiEtrY2JEni0qVL266PzybN9Xjn44zMj2C0GZmX5plfvtmAiMDTE0/z1oNvVdeRJImKigpGR0eZicU4eM89NDz7LNaeHoREAiGRAKczeTOvrCDHYkh33QV+P3IigWSxkHjwQaQLF5Kvu3WiJJqbMfb2IkSjyT4PSWKiJMaVJplobRU9yz2IiESlqFok8P6T71ffh9VqpbGxkcbGRjUH7na71Q7z8vJyJEnCZrMVpgw2Hk9qP4EAstGILIpYbvbByDk8aChIXLyI9L/+F8Yf/Qhhdha5uZn4a1+rpgcLWbprMpmora2ltrYWWZbx+/1MTU3h9/u5cuWKOtRKiVoKVc67W9NcuUBJX9ntdvbv358i5A8NDfHbv/3b7Nu3j9LSUpaXl6mu3ryCcDtoamriU5/6FAdu6nVf+cpX+Omf/mna2to4fvz4ttfdFWSyFQolwGsNIg8ePLhuGmIxRH5FH5FlmcbGxpz7ZUpMJfzckZ8jloit+50oiNSW1DIrzqpkovTIuFwuTpw4kdeNkTHN5fEgfvvbCE8/jW9lhT37mvn9d/1XSlrWz/twWpy30lo3Z9I0NDTQ2NiI3+9nqbWVntZWDN3d2CWJ+pISLHY7ZoMhSSwVFVBZiTA+Tvz97ydx/DjSzXEE3Hy4EEURYc8ejEeOkIhGEUZGIBQCgwG/U6S6tJqpuhri/lEqbZXEpTj+mJ8Bz8C681WvqyYHfvDgQfWGn5ycxOfzsbKykhK1GAIBTI8+iuFHPwJJIvHqVxN7+9vB6UyeS2lpSiUWgOGHP8T4jW8gl5cjnTpFPBCgpLcX0+c+R+Qzn0n+bY6Q9+8npkn/pfyuSH0ggiDgcDioqqoiEAhw5swZVS/o6+sjFouljOLNxwbpTklzZXMMs9m8TsiPxWL8+Z//Of/6r//K1atXqaur4+zZs7zxjW/kt37rtwrmRfjWt7415d9/9md/xsMPP8yVK1de/mSSr2YiSRKDg4NMT09v2FtRjDntWn1kuzezIAg0OjZ/UlUik9nZWXp6egrWI7MuzeX3Y/iTP0G4dg0fyQ3q3OIiBKIkPvlJyFBOrUQkykagnJPD4UiWPu7fT/Shh3C73ax88YtYr1zB19JCicORnO0xNwf19RgOHcJgsahd3IrxpyRJRA4fZvnMYSoTUeIOB5bFRSyCwOEzZ/jl3/wQI0Of556yKupKknY4A3Od2AZGMDz9R4iuShKXLyOdO7duw1egdJiHQiEkSaKqqkp18034fJz9+7/H1d8PViuiwYDY1YXxq19Frq9PlvA2NxP/mZ9Jiu+KkeMTTyTTT8p30WLBV9tE6fgU4aevErn/1ohdo1FtW9k2it1UqHy+RqMxZahVIBDA4/GwtLTE0NAQVqtVJZZMekE2xygmdsKCfqNqLpPJxFve8hZVy/vGN77BD3/4Q37wgx8U7X0nEgm+8Y1vEAgEuHRp68bczbAryGSrdEo+kUkkEqG9vZ1YLLapNlPI8uNM/SMDAwNFK28WBIGFhQXW1tY4ffp0wULj9M9F/PGPka9fZ7GsDMNNAVaIxxF6e5GfeALp3e++9cdra8kekqkphJISDBcuQPqwH1lGGBjA1tlJUzgMp05BNEr56Chhr5doIMC0zcbKxYtYlpaokmVKS0sRRTFl4NiaHOV36ts5+3oX7+qsx2I04nzVq5DuuYcnpr7DlG+KmpIapnxTEAxiGBqmKxiifWqeezw2jE8/Tew//Sfi73jH+osQCmF49lnEmRnKEgn899yjRiUHDx4k8fWvYx8YIFBZSVQQMIgipT4flu5upJUVsNsxtrdj/I//IPaf/zPRj30MTCaE5WVkzXcxFof55VJMa2t0/DDA0PKtW7O0VOYtb0nkRSg74c2Vvr4gCJSWllJaWkpLS8s6W/1oNJqiF9jt9k3PsdhkUqhpqFshm9LgkpIS6urqeN/73sf73ve+gp9DV1cXly5dIhwOU1payqOPPsqxY8fyWnNXkMlW2K5msrKyoqZ8MnlPpR+jEJv9Rv0j2qmIhYQyK1uSJC5evLj92nRJSqaIxsdBFJFvVtZpo7XgtWtIfj/WpibKnM6ko5bJBGYzQns7KGQyN4f46U8j9PWBJGEA+N73iP/6ryMpTz+yjOG738Xw6KPJsb6CkCyLPXIE8b3vpcTrpaS8HOuRI8TtdpaXlxkbG0uxZq+srMRgMPD09NMMro4wFjex76f+Ow+cfYD4TXPDcCzMyaqTyU1KAONwB4JbQiivJ9xUj1RThzA3h/Fb3yJx773I9fXq+xUmJrD89/+OYXAQZJmmWIxIUxPCX/818s00qb23F6PRSFlVFZIsk1hbw+jzIQFMTyNIEggCgiRh/vznEWZniXz+80gHD2J47jnk2tpk6a4EQjiaDEOaGykvT5J4OCzg9wvkKxnuBJlstQlrPztt45+iTZnN5pSoJf1+LfZGr3zXdwuZFBOHDx+mvb2dlZUVHnnkET7wgQ/w4x//OC9CuSPIJNc0l3YuvLaTfjMUQjMJBoO0tbWpzY/aOvFipNEU3ytBEGhsbNw+kcTjiI8+ivDssxAOJ6uinE5cBw4gv/GNqjNB1O3mgNWKKT2fH4+rFUiyLCN+/esI3d1IR44gmEzJCGRkBMM//APSiRPgcCBMTWH49rfBYkFubU2uEw4j9vQgnzlD4v1JcdwMNIFqYuj1elleXmZoaIjOzk4qV5f51xt/iC28SNgkMOX+Eo6zD8BNY8NfuuuX1NSYHAxS+ve/AVI1CLVqykmurUUcHETs6SGhkIksY/mjP8Lw0kvJDd5sRjKZsE5MIP7pnxL+yleSFWNms1pFJgoChlgM4Wa3uhCNItlsJAQhWdKcSCB8//uEvvEN+KmfwtrVhTg4iFRbi2HFT/XqAit3vZrQsTPY1TtTJhzOnwR2Is2Vy/rpekEikVhX9JBeql3sXhbl/tyJSYu3e5aJ2WxWBfi7776bq1ev8rnPfY6//du/3faau4JMtvoS5hI1JBIJtSQ2l7nw+W72W/WP5GMTnwlLS0t0dHTQ3NxMPB7Pa6MQ2tsRf/SjZImuMj1yfp7y559n7fBheoNBFhYWOP/2t2MeHEReWAAl1+/xJBv17r03qWWsrGC4ehW5tjZJJJB88t6zB2FkBLGnB+niRYTeXlhdRdYKflYrOByIV6+S+OmfXneeoiiqm8vhw4cJt7fzw099gKlTq7SuCHit8NjaU/zMf34b9f/6fUSrNWUSpRSPJ7vMbxYDRKNyMiKQZcwxGX9QJOxNcoez9yUMTz4JSnVYOIw5kSBRWorY34/Y24t04gSJ++7D+J3vwMoKlJcny3JlOdmAKIqIRiMiN8uiRZGEJBH69rd5qbGRlp/5GZqee46SxUUQRNoOvJHYWz6IwVh4/6ndEJlsBoPBoH62QEZbfcUfy2KxFMygUgtlj3klRCbpkGU572rWXUEmW0Ehk61uCCUyMBgMXL58OaeS2O2mubL11yoUmWhn0B8/fpyGhgb6+/vzWlvo6kr+D+0Y4ro6xIEB1q5dY+Xy5aTNjNVKYnISw9e/Dn19ySf7khISb3870n33JTfsWAxBkpJd6loYDMkn9pv5GmGj8xVFssnpeDwQ/V9/xTdbfJgwYDYYqY7JDJTH+Xb4Ovd/+tNE3vpWNa1itVoRS0vh8mUMjz5KxFnF2KSFaFjGuTpF3FjF99pPERw2UFYGvzzyfYRwGLm0NMkugByJYPD7EUwmBL8fgMR99xF/+9sx/vu/w/JyMkoRxWTfiwJJQkgkkJ1ODLEY9TU1mM6exb1nD50HDhCemSFmcPHUcy0cw0aVJCOKhd34d1tkshXSbfW9Xi8dHR1MT08zPDysGlRWVlZSmsEqZjtILxIpFrIhk2KUBCv4+Mc/zpvf/Gaam5tZW1vja1/7Gk8//TSPPfZYXuveEWRiNBpVcWyjD0GZTFhfX8+RI0dyfrrYDpnk4q9VCE1GibrcbnfK8QRByG/tcFjdMBVEIhHCkQiGRIILFy6o+Wvp/e9HvnQJoa0NJAn51Cmko0dJSBJSIoHociEdPYrhJz9BqqhQU0nC3BxyRUVycBQkNYOSEnC7k0Tj8yVf6/EgbWET4nbL/MEfhHjV+NMMXYSGNQPBBIBASdjAfxyI85aVCaxOJ7Ozs/T391NaWkpVVRU1r389lQMDGHsGcC4YMAoJYiVOBi6/H0tzDVJIZnUVhJHhZDd9LHaLTIxGDKFQMsI4eDB5MgYD0f/+34nff38yJSZJSHV1mP/mbzB0dSWvrcGAXFJyc/pigsT996vVbK2trcROn6a3dwZZlhgZmWFyMkFpaSkORylGo4NC3Ka3Q4AvFBSDSoBz584hSZIatUxMTGAwGLI2WdwMOyG+w9blx4FAgL179xbt+AsLC7zvfe9jbm4Op9PJqVOneOyxx3jwwQfzWndXkEk2aS7IzOiyLDMyMsLY2Jj6pL4d5LrZb6aPZEK+kUk4HKatrQ1BENY1IuY7Xlc+fBhu3EhunCYTfr+f+dFRHGYztiNHUoVQQUA+dAhZIYWb5bnap7roz/4MlrExxN7eZMVSKAQWC4l3vENNj8kHDpB4zWsw/p//g7C0BJKUFP6bmpA0PlzpkCSJrq4h5uereW5vArMEi6Wa6yonv9RXnT5+du9e9u7dSzQaxe12s7y8zA23G+Mb30hN81Fmnglgralk7dR9rDYexQkYDDKxGFBSilRTg7iwoDYWitEoSBLxS5dSyqDXAiLxQxfg0IVbn8mZ+6n55XdjGh1S7V0Ev5/Eq19N/KGHUt6TyWSiqqqcsjIPdXWHCIfDBAIBJiZ8RCJLuFwGpqdlwLVtm/idKg0u5vqQ/K6bzeaUBtPV1VWVWHp7e3E4HGrU4nA4crKG3yky2cqbq5hpri996UtFWXdXkMlWUD7g9M0+FoupluoXLlygrKws059nfYxsUmmQ/fyR9PW3SyZKVVpVVRXHjx8vuB4j3303UkcHQk8PPmDV46HRbsd75gzhTZrglH4P5RwEQWA5uMxv9v9PPvTr7+DV3WuIg4PIVVXJNNj587cWSCQQ+vqSDrmKe+9Ny23jP/4jsVOnkhqKBsrAtEDAQHX1MV47+0v81+t/Q9xgQTKYkBMyxngYswX2feE31b9TTDwVr6TV1VVGG7x8e86MyeSnImzGsbhIWZkDsCIIIrFXPYB9rB+pogJhZgYCARJGI7GGBsK/9VvIsRiCIBAIiHzrWxZ8vvQr1ETVe/+D90j/TMkLP0q6/L7hDcny4wwPHjabxL33LnP27D6St6UTcBKLxfD5PIRCy3R0TKo28cpTeC428XdqZAK3yCT9GIW01d+pPpathnz5/f47zjEY7hAyUWaaaMuD19bWaGtro6SkhMuXL+ctyCkf7mY3xXbmsyvYbrXYzMwMvb29m1al5a3HlJUR/8AHmP7mN0m0tdFw4QKWixfxlpUhZyin1na0AynjWh/pf4S2hTa+YDBz8f1/j0HMfNOIN25geOIJ5PJyqK1Nag1ra2AwIA4MIHZ0JG1TbiIQCNDe3k5paSmnT5/kO98R6T36MS5PdNA89QIQRwYkROIPPYRl0YvU2Yl84sQt48eb51peXs7eveW0tBgpLY0iSWusrflYWloiGjUhSeXMPHCZkqF2jFevkqisJGi1YnC5MP7KryAcPaqmXSMRGa83gc0GNtutaxEKCSyHnfje/ysYPvQrWX0MNpuUofHdRFVVLZC0LvH5fOpGqdjEZ/MUvtsF+K2QbdluPrb62UxAzBfKHrBVZHKnzTKBXUIm2XzJteXBSqf33r172b9/f0FuEm0qLdMXdrvz2bXr57LhS5LEwMAAs7OznDlzhqqbk/cyIV8yiUajtA0NET9+nLPvex8Wmy05V2NgACmNADfqaAdYDi7zr33/ilk007nYyY8nf8xrW1+b8ZjC9etJoV25aQQBysrA602mvFZX1dcq4mtDQwMHDx5kfj55zIjZwVff8yiHB77LnqmfYFhbwbYyz4HZKYx//dfJ1NqlS8Q//OHkAKoM8PlMGI0VmEwVuFwyXm8IrzfMjSE3E/ffT0NLC6bhYVz19VQ+9BDy8eOYb75nSZIwGhMIgoDVKmGz3SJXSRIIhbLfXLNJUwqCkGITH4lE1D6NycnJlGq3dO2g2CaJO7V+LsfIxlZfG7XsVGQCm5NJIBDIy3bmdmFXkEk2MBgMxGIxent7mZubK2inN6Sm0tKjnFz1kY3Wz3bDV9I5kUiES5cubfnFymUiYjr8fj/Xr1+nrKxsXWNnetOi1sYkU9XLI/2PsBhYZI9zD1O+Kb7S9RVe0/KajNGJEI2Cy5U0NgyFkvNLDIbkzPfaWuTmpNfX3Nwcvb29HD58OMVCPxoFt1sgEDAzU/Gz2Epey7uf+TAkEkT3HsRYZgSfD8NTTyHv3UviPe9JOb7RmNRHHnvMkNbHYcJkcmA2V/Hudw8wbTZTevfdjPn92FZXqRocpKqqCpfLddM+JGkhYjRKGI2ymsZIEm4yFRuPC4iimNXQpVxgsVhoaGigoaFB1Q4U12PtSN6qqqqib/bFns9eiI1+K1t9JQXm9XpxKoajBYbysLpZBBkIBPTIpJgQBIH+/n5EUcxqg80VygecnopS9JHtVolp18+GTBRX49LSUi5evJiV9fl2IxOlV2WjWTKiKGIeG0v2W/T3I9XWIr32tYiXLq17rRKV2E12DKKBmpKazNGJLCOMjiKbzchOJ8LQEGIkgmwyQTyOkEggHTqEdPgwoyMjTE5Octddd6VEZqEQDA+LrK3dWvZSoA2DZ5bBkkM0cPPcysqQHQ4MTz5J4p3vTCnXdTjgoYcSdHWJWK2yGlVAsut8bm6VqakFXvva8zidTuLxOB6Ph+XlZXp6eojH4zef/muIxxsQBAOiKKjFECYTiKJ889+SmqIVBEH9rmm/S/kUUCiflaIdpFvqT0xMIMsyJSUllJaWFsdSfwcik2JZwyu2+sPDwym2+tphYNt13k5HNqm0nWhaLAbuCDLxeDwEg0GcTifnz58v2hOQNhWl7efIVR/JhGw0k8XFRTo7OzcdFLbR2rmQiXZo1mYVcLbeXlyf+xxiOIxkt2Po7kb8yU9I/NqvkfjZn015rTYqAbCb7CwGFlOiE2F0FOPnP4/Y1weBQHJOSSKRrPi6WUkmVVeDycTQk0+yYLVy/vz5dTdWSQkcOCAhirKq0Z+e9OPolHA4RExGzcZstUIwmEyppX1vSkqSWrjLJavZNkmSGRtbJByGM2fO4HQmD5BuYOj3+29avCwwNhZhaclAZWUJDkcpNpuNcFjAYEhWHpnNtyreJElK+R5oI5ZCbsbplvrt7e1q5aNiqa9slFt5YmWDO12TMZlMlJaWEo/HOXHiBH6/H7fbzcLCAoODg9jtdjUdVl5evu1zyaZirNjVXMXCriCTzUI+ZdOz2+00NDQUNZRWyoPz1Uc2WltJgaS/X1mWGR0dZXR0lJMnT1JXV5fT2rmQiSRJ9Pb2srS0xPnz5ynXNiqmvpDyRx/F6PWSOH06WRIsCIjT0xj++Z9JvOY1anlsOB7mWwPfIi7HmVqbUpdIyAkG3ANcm7vGBedxTJ/4BMLgYHJeR2UlwthYcgxtVRVyWRnU1JCoqSHY1obQ28s9/+W/ZEwprq7CwICIKCatwQDCwQPcHSkhHvMRjTqw2UlGQW43iVe9KmMFVTri8QRzc3MkEkaqq2uxWjOTv2K7nrReh5ERmJsL4nYHmJpKznOx2+3U1dmQJDOiaE7pxNfqTrIsE4/HicViahoxPWrJF0o5bVlZGS0tLSmeWKOjo6onlmqpv417bCdKg3fKfl77+SoDrRStJV9b/a0ik0QiQTAY1COTQiIej9PT04PH4+Huu+9mfHy8qJMQIbnhB4NBenp68tJHMkG5EdLJREtc2y1vzpZMotEobW1tJBKJLYdmyXNzWCYm8JWVEfb5sFqtWCyWpK368DArP+nDf+AuJKsdyWDhXXs+SCCWzDsZTTLK2zAIBg5WHER87gWEkRHkAweSDCDLScF9ZSVZFnzuHLF4nKXFRcpFkYOHD29IANFo8r+SElnV1RdsR7nhfTUXPI9hnLYhRKwIbjdyZWVGa5b1a8aYnZ3FYrFQV1eL1ysCW1ffORzwnvdAPG4H7GrFlcfjYWVlnLa2VcrKyqiurqaqqiqz6/HaGmNjY1RUVGSMWgqxiWq/d9rucq0nVrqTrxK15Lp+MbBT4nimY2Rrq69ELVuRxVbiO6BrJoVCIBCgra0Nk8nE5cuXsVgsTE1NFWza4kZQRt02NjbmpY9kgvbJVPnfoVCIGzdu5E1c2Qjwa2tr3LhxA6fTycmTJzf9QsuyTEIUMdtslAsCQUHA5/ORiMexCgIlS14mf/v/YIhGCRkdXKn8KZ6qfQ8xQ5Kcyspk/vf/jlJbqznHhYUkgWj9umpqEFZWEHw+wpEIy0tLuPx+SlpaiN11F1upCGZz8r8kRL7V/FuMyXv5varHwLBG4jWvIfG2tyGfPLnpOl5vlLW1eUpLHZSWVhAM5rYppt73AhUVTlpbncBewuEwy8vLyVnzaa7HFRUV6ne9vr5eNd7TzmrRlsQq1Uzb+V5utNlrPbG0g8DcbjfDw8NZb5Q7URq8G2aZbGarPzAwsKWtfjYNi4AemWwX2out6AZNTU0cOnRI/QLlOyBrMyj6SDgcpqWlJW9f/0zQlh4bjUY8Hg/t7e3U1tZy9OjRvG6UrSKTXLQYtWKrogL5zBnMTz2F6dChpAgdi8G1a+B2UxIvxWeuxBlf4O1zf0OjYZZHD/8eobCAzycQiQigoQPFal3RRgDkpiaEiQkS0Sjh69epsVqxVFeTeO97U6zgIenFpfjQLS8nl7n5EKfyUzBh5THnu3j377+NkqOJdRYx6TCbwWgMMDm5SllZFVCCx5P8ndOpJSpYWuLme1oPi0Vmo8JCq9W6zmNKcT0OhULIskxVVRVNTU3qd0QbtSjEoqTGYGMRfzNkEzlkcvL1er243W76+/tT0juVlZUps97vNAE+E7bTZ5Juqx8MBtWUWCZb/WwiE7PZXBQjy2JjV5AJJL/sw8PDjI+Pc+LECerTNpNCDq/SQptmKisrK4g+kgnKjSZJElNTU/T393P48GFa0gdGbQMbkYm2iCDTNU1/bUpHu8FA4pd+CXF6GmF4GGQZkywjhMNEaxuYWzuO0ZggKiQIh5c5Pvt9nql9AzO2Y0jS+vSZdPEi8v79tzQTsxkWFwnW1TF1113U7duHsbaW2N13Ix89mvK3Hg986lMmtfVkdRWWlwXcbgGTCRobZYzGpMZuNoPZImxJJAB+/yT33DPDz/7sYSorzcCt0ch2e3JqMCSJ5GMfM2tbX1LgdMKf/3l0Q0JRoO0DcTgc9Pb2Ul9fTzQa5YUXXsBqtarpMKX0OF1rUchlIxF/ow13O2kog8GgbpSHDh0iGAzidrtZXFxkaGgoZdb7TmgmxTZglCQpr01cEARKSkooKSlZZ6s/PDxMOBxOGo6KIn6/n5KSknXvaaOf3wnYFWSSSCS4fv06wWCQixcvZswXGgyGvC2S05HeP9Ld3V3UaYiCIDA4OJizPf5WyEQmSspueXl58yKCcBjZ50NyOpE0VUXCTdv46Gc+g+HZZxGmp5ENBozf+AYxiwv6k64ERqOBuKmeqtV+GsOTDEp78HpDdHaOIctlVFVVJStTHA5if/AHajWXHI3is1iYeNvbaPzgB7E4HBsqFJFIkkCSXebJCq6GBhlRlAGBe+9NUFqajFR8PmHL8emyLDM4OMjQ0BK9vZe4cWM9+ZWXy3zoQ3HKy5MRiXJ8uz01+RYMJn+XHoltduzx8XHGx8c5c+aMarmeSCRU/zBt6bHW9Rg2jlqyKT3Oa0yBZqNU0jvKE3hvby/RaFQ9n0KW0irYiXG6hT5Guq1+MBhkaGgIv9/PtWvXMJlMKdY4RqNRJZNi4M///M/51re+RX9/PzabjcuXL/MXf/EXHD58uCDr7woyEUWR6upqGhoaNnwyKHRkkql/pBADsjaCMmVxdXW14H0y6WSijCreVGgPBjH87d8i/Pu/J40MGxqI/ef/jPyGNyB2dUE8jnT8OFRUkHjb25J/EwhgfPxxxJVQylJGKYIsmhCcdVTaq5BlZU76nPoEW1VVRXV1NeWf/jTS0BCDHR0E6+o4efFi1lqRzZYs4RWEZMWvJKW61YfDyX/PzwtoP0azWUZpU0kkEnR3d+P3+zl58hxPP23FZpNTGuSDQVhZEUgfjGm3y6xPZcuEQtlt0rIsMzAwwMLCAufOnUsptjAYDBlLj+fm5ujv76ekpES9hkpDXcqsli1KjwstkKeL0i+99BJ2u535+XkGBwcpuTnWubKysiANgDsRmRTb6NFut1NSUoLVauXgwYNq1DI2NkZ3dzef+cxnaGlpoaysrCgFDT/+8Y/54Ac/yPnz54nH4/z+7/8+b3jDG+jt7S0Ige0KMhEEgdbW1k1F5EJpJtrUz9GjR1O6qosxDRFuid+CIHD8+PGCN1xqBfi1tTWuX79OeXn5xkK7LGP44z9G/Pa3wW5HsloRR0awfPzjyH/1V0mhXJKgqor4+96XdPtVZpe87nWIX/4q9riDiMGBUYpQGRhnpvQwPZazhELJ69jQ0EBLS736BLu0tERXVxcej4FoVMRmO8SxxmMsLVnxeiEaFbBYZK0ZL2Zz6r8VGAxJMllbE4hEwOtNRgdXr4rIssCnPiWoOkokkpyh/pGPxCktjdLT04MgCJw5cw8GQ/JFdjvrSCIUoqBQxgesra1xzz33pOgN6dCWpioNdUrU0t7eDqB2tivmhZuVHsdiMWKxGIlEoiilx0okW1NTQ3V1NbFYTI1auru7VdsS5Ql8O4Umu0WAzxeKZiKKomrlAsn79sEHH+R73/seAwMDNDQ08KY3vYk3velNvO1tbyvInpE+r+TLX/4yNTU1XL9+nVe/+tV5r78ryAS2rkgq1DyQ7u5uvF5vxtRPMXSZ+fl5urq62Lt3L9PT00V5ulIik8XFRTo6Orb0LBP6+xGffBK5oiLZhQ7IFgtiezvCygqJ++5LzhhZXsb48MNJW/ibs9vj73sf0eFZyr9+FXFlFlkQGbEe4iuVH2NqMRkBOZ0yFkvyszQuL1Pr91PT0sKgqYH/9b9kIpFkWLHXfwOjDE97LxKVrVgscPr0rUZEp1PmD39jDvu1axxZcLJcfgkwYrPBhQsJfL4kkXzoQzEkScDtNmG3S+qMr3AY2ttFwmGR5WWJcDiEyXSYsrIyvvtd+K3filFMJEnyVpl7IpHg5Ml78PlMhMNwc0THljCZTCk2IKurqywvLzMxMUFPTw9lZWVqOszhcKSUHsfjcfr7+4nH4zidzqKWHivrmEwmamtrqa2tTbEtmZmZUWfLKOmfbC31d7LPpNjHyORc7HA4+MhHPkJVVRVms5lPfOITfP/73+dTn/oUr3vd64ri1bV6UwQsVLp915DJVsh3o8/GX6uQZKItKDh16hS1tbXMzc1lH/nMziJeuQJjY1BRgXz+fNIBNwMUIu7o6Niy6VGWZRgcTKa2qqsV45GkFbwsJ3e/F6+BKJBwlAMQ/OYP8Oy5fDNd5GDlf/wpX+gboHR5Ar+xnD7HeaIGG8abmoHRCIZVD8Z/+UfE69chHCZoseCuO46B3+FB41U+dONXqA5MALAkVvP7js/wbfkdSFIAm82ClLDwuuf/nPqv/m+ESJj/IQmsXGnkn9/4JUYb7lXft80GVVXJU7dak8MitZKbJAkIQoJ4fInaWitOZxnBIKyuCkSjxUubeL3wuc+ZcLsTLC+vYjC0UllZyQ9+kDxmeTl8+MOxrAlFgeJ6XF5enmKbsry8zPj4eEp1kdPppL+/n0AgwD333IPZbE5xfM5Ga8kWG6Vl0m1LtGaLHR0dACnmlBtZxN/OPpNCIpvSYIfDwWte8xpe85rX8KlPfaoo5yHLMh/96Ee57777OLHBvpIr7hgyMRqN2+4zydZfq1Aifzwep6urC5/Pl1JQkHUabWQEwxe+kJyjUVKSFMmvXEF697uRX5vqwptIJBgYGADg7rvvVifSZYJaXlpWhsloRIhEbs0MUToBEQgEJCQMmFYXQZYZ/PcBPr1oorxc5o/+KEY0bqDHeBfmlruSOkZydSApRMciEvYv/V/E7p8QdNWzgonE1CzHx5/nTf5/5rdGPoJRuhUVVEpL/N/VX2DF1oxgO0Y4vMSlvm/y9p5PIhtI2qzEZMoDM/zyt9/B7721nVVbkjCdzmRvYzi80WeRIB6PUFNTQl3drVyW9vU3S/s3/Petn68X2TfqSYlGYXk5xurqAtXVZqqqKm4OnZQ31GS2g3TbFG3pcTAYxGg0smfPnpT0VrqIr3wv8olastU00s0WtZb6WnPKdEv9YvexKO9hp9JcG2Gn5r9/6EMforOzk+eee65ga+4aMslm2mKuUcNm+kgmFEKADwaD3LhxA7PZzKVLl1KetLJdX/z+9xFmZ5GPH781i2NqCvF73yNx5oyaH4lEIrS1takEtVnXbMoMknvuSZbpDgwgNzQk62mDweTI2ZJSIlIJggCSwYwjskypGMJmk29ugAI+H0xNCRgMYDav/9zO2fqx9rYTqG7hB8/biUbBZK6lLjDOm9YeRpQSiNwiVRGII/Krvs/yrYpv4nC4+JnH/xkAyWhEkCUMRgEwYE/4+eOD/8Tqr3wYSBJJRQXMzq5/z36/n2jUjMVixuFYH4maTMmqrZUVYZ1GUl4uq30mFouM05msKMskticJLZVkfD4fi4sRGhvttLSsJ/hCazJwq/TY4XDg9XopLy+nuroaj8fD2NgYFotFjVoU25SNSo9zbZjczmafyVI/fbCVQizxeLzg5pTp2Kk01+0ejPWbv/mbfPvb3+aZZ57Zck/MBbuGTLZCrmSylT6y0THyEeDdbjft7e0bTmDMyvZkdRVhaCjZtKf9+4YGhP5+hLExZJcLn8/HjRs3cLlcHDt2jCeffHLDtddZx1utRP/szzB//OMIIyNJsT2RSO6ugohRiiKKYJbCxA0WRJOI3X5rA4zFIBYTMJvllGGIsVjyNWXiMrIYZC5QSjRWgt1uwmgUiMjl1K4uYGR9hGkkwZFoJ5DcZKrWJpAREQUDgnCrD0ZCJtD9PF7vgzdTOZXArZszeY4yKyureDxhTKZ6DIbM1ihOJ3zoQ/GMEYLZjKq9VFcn+0iybVpMNokOUlZ2noqKnTXsU1wVHA4HJ06cQBRFWltbSSQSaiFEb28vsVhMFfGzKT3eqmGyENVW6YOttON4/X4/FosFo9FIZWUlpaWlBdcfd0P5cTEjE1mW+c3f/E0effRRnn766YLPmb+jyCQej2dVMrfd+SPb1UxkWWZycpLBwcFNI6CsyMRgSJJI+nlIUrKiymBgYWGBzs5O9u3bx759+zQvydy4qEQk2jkK8tGjRL72NcTnn0fweBAGBjB885vEoxLizAqiLBO0uogZLLid+9atC0ltJL2SOxSCFdHJajiCJRpFwEw8nkwOlUdW8QgVuGTPOkKJY2BM3M/CQrLDfZADHJM7NRu9ADKYEDAcuRuDwcDg4CCRSOSmpXodpaVNrK0ZmJvzEY8nsFqrkWURUZQJhZK6CqSmsTbyuUxHkiy27iOZnp5mcHCQw4dP8uyzpVn9TaGgjC+oqanh8OHDKfeJwWCgurqa6urqTUuPFa0l14bJQqehtJb6+/fvV4dZ+Xw+JiYmUno4lB6NfLEbRP5gMJiz0Wu2+OAHP8g///M/8+///u84HA7m55OmpE6nc9Pqwmyxa8hkK4JQvixbMXs+80e2QyaKC+/i4uKWmkVW65eWIp88may2EkWE1dUkiYRCyI2NjIgiI52d64T29EFWyrltNBURAIsF6YEHmJ+HRN0k9U+9QHxljV7nEYxGKBEClCbWGGh9Q8ZTjcVuWZzc+rdEp7mRyPGTOLoHiC2VsSrbqWKZEGH+ll/mM/wPJJLpLQVGEnwm+ps8/h/Jz9kS/whf4pcwSRESoglBlhBlCZ/RxfCFX6bKVEl9/WFCoRDBoBtJmuV1rxsikTBiNBo5ePAgIPGXfynh9ye1DS2JOJ0yZnPhNnrF+XlycpIzZ84QjeaorOeJlZUV2traaGlpYd++fZveT5uVHnd0dCDLMpWVlVRXV6fMTd8oagmHw2rZcTFKj5VzdjqdtLS0IEmS2qMxOjpKT08PTqdTLZXerqX+bkhzFTMyefjhhwG4//77U37+5S9/mV/4hV/Ie/1dQyZbQettlenDyFUfyYRcNROtZnHp0qUt2T1bd1/pzW9G/MEPEB99FCGWFKolp5PJ/fuZdLszuguraycS0NYGbW2Iq6uwbx/CPffABlYq8/PwX/6LmbW1g9yz8ju8c/b/UBqYQUBm1VDG9ZPv4WrVm1I2YrM5Wfobj9/yxwIIh+MEgzEOHizF9N8+xsyn/gFHVyc1whIrQjlfFd7HvyTexSK1/C3/hRKSi0Yx8ft8gu/wNqxCkjv/hfdSJSzxp8b/iT3hRwaGxMP8F9Pf4/6jBg0RWHA4yvmbv6kiHL6GyWTEZrOxvNyBIAj8yq/U4XBUrRsIpW1kzBeSJNHf38/y8rI6f2VhIfm7TGL+RgL/drG8vExnZycHDx6k+eaEylyQXnrs8/lYWlratPRY2XQjkQg9PT2Ul5djs9mKVnqsjRq0PRrp5pSjo6OYTCaVWLK11FfSqLtBgC+WZpLvALatcMeQifJFisfj68oHFX3E4/HkNX8kF81kdXWVtrY2XC4XJ06cyOpLmC2ZCCMjEAwiHzyIbDYjGQz4FhcpefFFLv/8z2POYFMviiJLSxB++v/f3nWHN1mu7ztJ996Tlg5aCi3dLEVBkD1a1J/iRBwHEDwMFcTJ8aiAC0QFBXEi4oECosiGskG6S6GlLW3pTNLdNM1+f3/E9yNpkyZtMwrkvi6vc0iTfF/Wd7/v89zPfR+Ew+nDAAgUtnawyikG+fsKbF96VumJ1QkiEQttbSzY2REUh07EZz6JcCzIAFsqxhXWMHh7hcKxSXlfZVNaeREeN04BZ2fl5DghBI2NjeDxBLC19cPSpRx4DfHGqZkr8c4eLlzZbai3CUAjcYdCDPyieAr7kIopnGNwd5Jgb8sENHM8wSaAj4/SZ6umhoXNiiWomfAsElg5aJQ4YltGIuQKNoa6KJiGt0jEQnOzHJcu5SEx0RuRkZHMLo3OY/D518HnC+Hu7s5MkRtKty+Xy5Gfnw+hUIgRI0YwvQfac9HU3Fe+l+pGkr0FjTWOjo42SHlEtSk+aNAgiMVi1NfXM9JjVb8uR0dH5OXlwdHREcP+cWbWlNVCn7cv0uPuejLaLPWLi4shEom6mFNqeh7Gk+4uaMAbC/2GTPRxNNVUJlLtj1C7+t5C3zJXbW0trly5gvDwcISGhvYoEVEvNde5cwCbDRIbC4lYDB6PB9uICHjX1YHk50Pxj1W5KlpbbfHpK82YcvUsxGwvtNj80xEmCkRmFiB24Bk4/msOuFx199uaGuUUua2tsl3D8fFEi+tkCAQAWlhYuVKKoCDlhZsSCZ+vnBq3tgY4HAW4XC7EYjGCg/wxoDYf4b8ehpUTCx7syajESNSwWeCQWz0LAGiHE/7gpMLLkaBZoDwf1j/+jFZWBCyW8jMXWTmhxPtetLYq78DhUI8u5fNIJBIIhWIEBQVh8OBbuy/VmntERASEQiFzUaQWL9RUsbfJeRKJBDk5OWCxWBg+fLiaFZC7u3KORJv818ZG/6FFbbh58yZKSkoQHx/P+D8ZGra2tmrS4+bmZjXXYxsbG7i6uqKjowMODg5GlR7rc19Nflh011JaWgpbW1uNlvqqyjVjgRKsrjkTC5mYAJ0tVQyVz06hi0yoQWBlZSXi4uLg4+PTo+fXOxGxoQGws4OwvR38+nq4/bNSZPF4IAKBxofIZFaw41XBmbSi3S0Ijv/whVDIRo3MB6EZ13FtmAz/+dABAgELbLaSQDo6gNJSFqysWLCzAxITFbCzU3pcdXQAvr4EAQHq22MbG2XPoaFBjrIyPgAWfL0HYNaJFRhxbTtsrORgsYCJig1YR17AG4p1ULDY6G6XTf8mld5yc1EolMN/9vY0eZdF20cghAWRSITmZhGsrJwREGCL7prdDg4OCA4OVjMppBYv1Jywc4+gO3R0dCA7OxuOjo5ad6Z9JQttoPG7VVVVSExM1J6WaWDQ8pKNjQ3q6uoQEBAAZ2dn1NfXMxdqfaTHqrsXQL9dS2+b49os9Wn2CN210Au4sckEgNl6JsbGbUUm9GJviP6IJnS3c5BKpcjLy0N7eztGjRrVq9UDh8OBVKrbwoMMHQrR5cvgA8qSjKOjstPNYoFoqIkrFApYWVmhVdQBBZHBiiMB29oWUglQVcWGt1iGg8ec8W2ePUrLOUrZrw1BUpICNjYAm80CIcpBvtZWJZEIhcp/c7lASIj68VxdgSefbEFmZj5cXV0xePBgOB3aC8/vfoTCxgpyO+V7Q9pFeJFswWXrMTjuqDSLbG1lQfUtoKRByYTLVRIGnU6/coUNFkupxhIKlbuXq1fZIEQOhcIGLJY9ABZ4PDnCNIvOuqCzSWHnHoGrqyuza9FkB0691nx8fBAVFWVSu3BCCAoLC8Hn85GcnGzyVSyVpAcFBTGN/uDgYEZ6XF9fj2vXrkEikcDDw4N5H7VJjzvvWlTnWTpLjw2xWOycPdLQ0MAkJgJASUlJn3PetYG+xu4Cxtrb22/LlEWgH5GJPj9IejHOy8vrc39E2/Nrymlvb29HVlYW7O3tMXr06F5nHuizM5HL5bjm7w8vZ2cENTeDbW8PNDcrI2gTE0FGjGDuq5pBEhMTg+3uQjR1eMKRVwiu40AoiC04Uhbc0YgrnmPh5MqGlRXAZhMoFCxmcrytjcVczC9fVpINvcC/8ooN/vc/idruRLmyy8OQIUGMB5jNkd0AIWiWOID8o/BSKBzggBbM7NiFNGkK7O2Vz0mPpVCgSxlIoVCSm5WVkjgGDFBO3VdXszB/vg04HAJClBJxa2sryGTKx1AL+Lo6zSFWtrYEmloKnXsENBmRz+errba9vb3h7u6O5uZm5ObmYuDAgT0qcRoCCoUCV65cQVtbG4YPH24QOWdP0NTUhJycHISFhWHgwIFqf+ssPW5vbwefz++V9FjTwKShZbudLfXp5yqTyXD16lXI5XK1XoshLPVVM+a1QSgUWnYmpgCLxVLz4jdUPjsFh8NhLtD0A6dOrQMGDOii3dcbVVVgVVbCViqFoJuMd5FIhOzsbLBcXRG+dq1yEv7KFcDODor/+z8oUlJAvdLVJtqhtKmQOzoiP/Jx3Fe7G5GCCnSI5LCW2yGbk4A8x1iw2R1gsx3A4dwaYxGL1XcGqlJfFktJNKoN5KqqKhQVFWHIkCEICAi4dd/mZoYoWP88lsUB2DICL04TbGwAf38CheKWRby1NRASQuDjQ3DzppLcYmMJE/0ulRLIZCwmu8TKipZHWLCysgIhyl0Mmw3Y2SmJ5N//tkFra9fPyMWFYONGiUZCUYVqMqLqarugoABSqRQKhQIBAQEIDAw0KZHIZDLmYjd8+HC9SnGGBFWMRUZG6qwEqEbbdic9puSiS3osk8kgk8mY240lPbayssKQIUMYMqyvr2cs9R0cHBhi6a2lvj7SY2OquYyN24ZM6uvr0dbWBnd3dyQnJxultqkqP2axWEwpbejQoQjUoITSifZ2cNauBfvwYUAoxAA2G46xscDGjbdi/P5BS0sLsrKy4OnpyUwuy4cOxT+e7mrSH9Wac+eVTqFdAprDAxDQUgiZQITjLYEosR6MCDsZxCIxpFJrKBd9nH8sKpQ7EfoUzs4EbLZyxyKVstDWBuzfz4a3NwGXy4Vc3oQXXkjo4jSquOcesP++DBZRgMVmg8UG2EQONoeFm8H3IspdgQ8/lCIwkKC+XlnGsrMj8PEBqqtZWLXKGu7u6pbzAgHQ3EzPqwO+vlI4ORF4eTmAxVKuXIVCQCBgwcdHuSNpbVVa2asu2js6oDFKWBdUV9sODg4oKSmBn58f2tvbcebMGTg7OzNlHFUfKUNDIpEwIpOkpCSj24p0BpfLxZUrV3qtGNMkPa6vr8fNmzfVZkQ0SY+lUikKCgpga2sLZ2dno0mPVS/0qmQYEhKiZqlP3Z+ppb6np6fei1pdSi6ZTAaRSGQhE2NBtT/i7OwMHx8fozXJ6PNKJBJcu3YNDQ0NGD58eK8bnJzPPgNn1y4QV1fAzw9obob72bPg/Oc/kH/+OXM/alM/aNAghISEqF+U/rkq1tbeMieUyymRcGBnpxwhUfWPKoMvyqx8IbQB8tkc2NsQODlZw97eAdbWrH9WeAS1tS2QSKxAiPs/K3xlDohEArS3K8+By2XhrbcokQWDzQ5GXJwIkyerX5Rl8+aB/ctvcKisg4JYgaUA2AoZGh0G4FT4s7ATAsHBBAMHkn96Gyr58P84/mr7Tba1tSE/vwB2diPg5WXbJXukc6nM3r5rPklv/Tup+3N1dTWSk5OZsqpYLGbq7apuvd7e3vDw8DDYvIJIJEJWVhYjvzW2dLUzqqurUVRUhNjYWHjryiXWA6plxfDw8G6lx25ubkz6KSVR1XKYoaXH2j4zbZb6NTU1KCoqgpOTE5OK2Z2lvj6yYKB7j73+jH5DJtq036rzIxUVFb12DtYH9EuYk5MDNputPaVQH/D5YB84AOLiwuxCFC4ukEmlsDl7FoqSEijCw1FaWoqysrJu1WG1tcC//mXFZJATwmHeLxcXgq+/lsDfH1i3Tt0/qrKSheXL2bC3V0AmA8RiAjabBZmMA0IADscTCoUchLCgUBCwWIp/ppjZUPW7YrHohV/ZW6mv7+p1RQICUPnN77j2zKcY0/wXWCwgc8B07B26Cs3wBfQY1Os8j9HRoST2/Px8BAeHwN7eTuVcjA/qbtDU1IThw4er1bJtbW0REBCAgIAAxq2Xz+ejqKgIYrGYubh4e3v3+jtEe3Wenp4YMmSISctqgFJ6XFpaivj4eINlXnSGNulxSUmJmuuxRCKBlZVVl16LroRJfYlF3+n37iz18/LymBKeJkt9feznAVh2JoaAakCWpvkRQ6UtakPzPzUVe3t7xMXF9Wl1yaqrU14NVX6ELBYLcmVEIBRVVcgVCNDS0qIx9766+tZOpLJSqaqysSFwcCCws2MxCifV8k1n/ygOh8DLi6CtjcWUiwIDlX0LJyeCNWukKC9nY8ECZf/CykopuZXJ5FAnE2rXotxFNDZqfs3S4DCsHbQV9rZy2NmzmNqZLodcW1sCFxfyz2u5dbtIJAaL1YphwyLg6Oit8bmM4b4LKEsOeXl5kEgkGDFiRLelDOrW6+np2aXeTleulFj0DYOiQ7EDBgzoNujMGKDWMJWVlUhMTDSoyKU7UOmxquuxj48PE22rj/S4twOTvW3w98RSXyaT6ZQF29nZGX0K31joV2RCoW1+xBhJiBTV1dW4evUqOBwOQkND+/yBEn9/ZbNcIGAyQ1gAOB0dIHZ2yObzIf1HHda5mZqbCyxYYM1YlUilQG0ti5H0jh6tYHoC2nI8AGX5a8sWMUSirhciOzsCf3/lj8jaWhkiJZWyALDRWXCmUKg//vPPrTF7tgIDBqjvEm6RAgcdncpKLi630hc7w88P2Ljx1q6Klja5XC4SEoYgMtILtbXK/Hca1asKZ2fl6zFU2BW1ybG2tkZycnKPehSa6u20jJOVlQU2m81cED09PTU+Nw2OGjRoEIKDgw3ymvQFnaWqq6szi/RYIpEgMzMT9vb2iI2NBZvNZvJYNEmP6XtJlW29HZg0hGOwLkt9QgisrKzA5XLh4eHRRRUqEAg0StFvF/QrMiGEoKysTOv8iKHCq1ShUChw/fp1VFVVISEhAQUFBX2yoWfg5QXFrFlg//ST8t/OzmC3tsKqpQWVo0bBOioK8dHRXVZDXC6wapUVioqU5KGU6RKmLyCRKA0L9VWFKi251C/imZksNDezcfWqcigwLEyO9nbldLmTE1BXJ0NVlXa1UHu7DDk5xWhqcoOdnbrn1cqVUqa5rlpi1ybNpVD+jTBZ6VZWrZgxI4EpLfn7A1u3dk+MFcrgxj7tXmgejaurK6I1fD49hbW1dRdbdSo7zs/Ph7u7O9PEd3BwYJrdQ4cOhb8WPzVjgRCi1is0RlRsdxCLxcjMzISTkxMjQqHQJD1W3f31VXosk8kM3o/qbKlfXFyMpqYmVFRUdNm1ODk5GX1g8fTp0/j444+RmZmJ2tpa7N27F6mpqQZ7/n5DJoQQ5OXlddv0NvTORCqVIicnByKRCKNHj4ajo6NBjyFfuhSQy8H+80+Az4eczUb16NEgb7yBmJgYjSuQjg7l6pvuQqiMl+5ACOnqTt8TZGayMGWKHVRbT1TSy2YDr7xShevXRaiq6mrZQsHhcCAQ2GPRIke0tUlhZUVgbW0NKytrsNm0lwN8841Ym7+kRtDPgxDCxMyqQhMxqkJbuUx5Ptp3RhS0tBQQEICIiAiDrxBVLV4iIyMZixc+n4/r16/D2toaUqkU4eHh8PX1NeixdYHOsAgEAgwfPtwgcxU9gUgkQmZmJlxdXTF06NBuL+yadn99kR4rFApIJBJlGdpI0mM2mw0bGxu4uLhg6NChEIlEzK7lxo0bWLZsGUJDQ+Hg4IDW1lajlBbb29sRFxeHefPm4eGHHzb48/cbMqGpaoMHD9ZanzZkz4RmPzg6OmL06NHM6rqvAVlqsLeH/K23IHvhBVRdvIhykQhiLy88GBvb7cOUcy7Kiztd9NPrWl+NP5ubWZDJlM9Nd/UKhZKsZDJg61ZvsFjdD2WyWCx4eweBzbaGl5ccbLYYEonwnxo1B4TYoanJpsc7guzsbGZV2puSQ+dymSp07YzoHEV4eHiXgTxjgVq8BAUFobS0FBUVFfDw8EBFRQXKy8uZi6GXl1evB2X1gVwuR15eHsRiMZKTk00+w9LR0YGMjIxeCw20SY8rKyvVXI+9vb27SI8VCgXq6+tRXV2NqKgojeUw+v/7CtUGvJ2dHSPgkEql+OCDD7Bjxw5UVlbCy8sL9957L6ZNm4b58+cbjFimTp2KqVOnGuS5NKHfkAkABAUFdXshN9SuQZmEl4fg4OAuK1BD735kMhnya2vR6u6OqKgo5Ofna72vcmASUK6+Wej8m6I7CJFIaf3el+Yzh3OLqORyAkKUB3N1VV60+PzuH0vh5MSGo6M9AHsoFARisRjNzVI0NUlx+XIexGInZnpcG0G0tLQgJycHfn5+jOtvb0HLZT1BTU0Nrl27ZrbSUlFREbhcLkaMGAFnZ2cQQhjH4/LycjWLFzrzYqhdk0wmY3aDSUlJRiUtTWhvb0dmZqbGQK/eQJP0mEq4b9682aVn1dTUhPz8fGaGRtPAJH3evkiPAe1qLmtra8yYMQMNDQ0QCAT4/vvvcfDgQRw+fBiLFi3q0/thSvQrMtEFmrbYW9CeTGlpKWJiYjReOAxJJnRGgMPhYPTo0cwEtbZzo/Mf9GOhp0H/lz6UGjUC+pVvugeBQiEHi2UFQgBbWxbc3QkABcrLlQfx8yMM8dDAR01gs1mwt7eDQmEHuZz1D1HX4dq1a0xMLO0P0NUvNVs05Y6Agjb6y8vLjeq8qw0KhQIFBQVoaWnBiBEjmCYyi8WCm5sb3NzcMGjQIHR0dDBNfGrxQt9Hd3f3Xl/c6DCktbV1n9WLvYFAIEBmZiYCAgIwaNAgozSeO0u4qfS4tLQUeXnKmGh/f384OTkxaZHGkB4DSjLpbtcnFArh4OCAkJAQLFy4EAsXLuzlqzYP+hWZ6JO22NsLvb6Z8D0NyNIGmnzn7e3N1IBVVzyqr1W1OQiwwWYryUIqvWWKaGtLfauAd96RIi6O/HM76VFfQhWEUHuKrj8IFxflDuTWQKHyeGIx/lF96Ya7uztCQ90wePBgCAQC8Pl8NcmkjY0NGhoaDJbF0ROo7giSkpK6hI0ZG3K5HLm5uZBIJBg+fHi30mN7e3sEBQUxzrfU8bigoAAymUwjSeuCuYch29rakJmZqWYYaWyohmq5ubkhLy8P/v7+kEgkuHTpkk7pMYA+7VrMGYxlCvQrMtGF3u4aqF043SF098M1RM+kpqYGBQUFiIiIwMCBA5kfCv0iqZKJKpGw2Ww4OLDg46O8cCsUXXccrq4Eo0Yp0Nf+rEymgEym+MePrKs9PI2ipw7CqgTi4HBLTaap1Nb5NtWY2LCwMIhEIly9ehX19fUAlE6tLS0t8Pb2Nopba2dQxVhbW5vajsBUkEqlyM7OBpvN7nFpqbOqqa2trQtJ0/6Ak5OTFpFHBzIzM+Hu7o4hQ4aYnEiodVBISAhCQ0NNemxAWebOz89HbGwsMyisr/S4L7sWXfJjC5mYEL0pczU1NSE7Oxs+Pj46VSL0GL3dmRBCUFxcjJs3byI+Pr6L/QQ9Np227UwkLBYLvr7KJrImCSyglMH2hUhcXQnYbOVEPIvFASG3HIOV4VR0xwMMGqRAaysLn38uwcCBt9jG0VE5EOniohya1DTr4uKiPNfOkMvluH79OoRCIeMwQGvaNFuEXgw9PT0NXsOXSqXIzc2FXC43i2Ei3RE4ODhg2LBhfRuMVZnG1mRNQuNrvby8GIsXKjzx9fXtc3+qN6DOw+Hh4SafoQFu+YwNGzZMzXFCm/SYy+WiqKgIDg4OzHeSLni0ZbVo27XomrK/nVMWgX5GJrq+2Kp5Jvr8CCorK1FYWIjIyEgEBwfr9ZjelrlkMhny8/PR1tamNe9ElUzoDkiTWaOSLAxvGyKXy2FllY+PPmLDzy+KkX9WV7Pw3/9aw9ZW2RuhOwtClP6SoaEEERFdz+ebbzTPfQC3Zj9UoU36q5otQhvPZWVluHLlCjOH4e3t3ecdBHVltrOzQ0JCgsl7BNQexcPDwyg7AlVrEhoEVV9fj8LCQkgkEri4uKCtrQ2BgYFmIRI6jKmP87AxQIlEl8+YNqPHzmFquqTHnQcmpVJpt3b6QqEyWtpYEAgEKCkpYf5dVlaGnJwceHh4GITY+xWZ6AKV7+raLioUChQWFqK2thaJiYk9aqxyOBxItGWtakFHRweysrJgbW2NUaNGaV3t0h+vqq2CrnwDQ0EkEiEnJwdWVlZ45pm4f1b8ynJedTULX39N0N7OUsbjqsDRkUDb7JquuQ9V0PeI1ug1fX6dG8+d5zAcHR0ZYtHXloRCIBAgOzvbaBdyXaChUoGBgUZrNqtC1TBx8ODBqK2txbVr12Bra4vKyko0NTUxfZaevpe9AZVeR0VFqUUXmAp1dXUoKCjolWFlZ6NHVenx1atX4ezszLzXLi4uGnctlZWVEIlEsLe3Z3Yt9LdPdy/GLnNlZGTggQceYP69fPlyAMDcuXPxww8/9Pn5bysyUbWI10YmNJdbIpFg9OjRPZ7i7WnPpKdlNDabjYaGBvj6+ppMhtnW1sZcSDWdY2Agwa5dEgg1mDE6OCj/3hdQ6a+vr2+P5J+qUbt0MI3P5zO2JJRYdLn0Njc3M5k0pva5AoDGxkbk5uYiNDQUIZ1jK02AhoYGFBYWYvDgwRgwYAAkEolGuSx9Lw1tcU97FOYQWgBgiDQuLg5eXl59ei5t0mNqqa8qPaaWKdXV1bhx4wYSEhLg6uraZWCSPm9TU5NRJ+DHjRvHeB8aA/2KTHT9yOlFUCaTaVz90zhVFxcXJCYm9upH0ZOeibZGuybQmmpwcDDKyspw/fp1eHp6wsfHx6hDaXRrTpud2s6xr4Sh6/i0Rt7bC7nqYBqVeKq69NKyg7e3t5rAgsfj4cqVK4iIiECQhshjY4Me31wrck1ZJDY2Nmo2H1QuW1xcjI6ODjXH476WFumOoHOPwlSgRBIbG9tnItGE7qTH+fn5sLe3R0dHB4YMGQI3Nzc1xZdqr6WqqgoXLlzAoEHanSf6O1jEmFTVQygUCp0Z6ceOHcPIkSO7uOxyuVzk5eUhNDS0T6vPmzdvgsfjITk5Wet9qBleZWUl4uPjdX5JVZtz9LwEAgF4PB74fD4EAoFBewMUlZWVKC4uxtChQ82yIqTHj46ONpo9iGpELJ/PR2trK1xcXODt7Q25XI6KigrExMSY3J4EUKZSXr9+HTExMWa5kNIskmHDhuld2qGN5/r6ejQ1NTGNZ5ov0pPfVU1NDQoLC412Idf3+HFxcSafIQKAiooKFBcXw9nZGQKBADY2Nl0EEYCScCdPnowRI0Zg69atJvdEMxT61c5EH3TeORBCmEyQYcOG9fmiqWtnQq3JBQKB1ka76rlpS0WkUtnw8HB0dHSAz+eDx+Ph+vXrcHJSTo37+PholXd2B0p2tGfU23Cv3kI1UMrYx+8cESuRSJjAKqFQCBsbGzQ1NcHKyqpPA349geowZEJCglGbqtrQ2ywSmos+cOBAtcZzbm4uAKhNj3e3m6ZEaswslO5gbiKpq6tDaWkpEhIS4OnpqSY9LiwsRH19PTZt2oT77rsPv//+O0aMGIEff/zR5CmahkS/OnN9LpqqF3uqoGptbdWYCdIbdNcz0bfRDqhH6wLdN9rt7e3VegP19fXg8XioqKiAtbU1Qyz6zGDQ96SjowMjRoww+SqHGga2trZ2CZQyBaysrNDS0gK5XI4RI0Yw5HLlyhVGhUNX2sYoLXa2cDd1ap4hs0g6N56p4zFV2rm5uTHlMNXPmRKZORYywK0dmbmIjMvloqCgQI3IOkuPq6qqkJSUhN27d+PGjRtgs9l44403MH36dNx///23pQ19vypzEUJ0KqnOnz+P8PBwODs7M1YQ8fHxBpsXoHX4MWPGqN1OG+2+vr461UCqZS2g9yZxqtPOfD4fCoWC+UJqysKgii1ra2vExsaa3GeJSn8VCgUSEhJMPsNBDQs7OjqQmJio5nxLVTj0vWxvb4ebm5ua31VfQZMZm5ubkZiYaHIiVyWypKQkoyqDqMULn89HY2Mj7O3t4eXlBZlMxrgKmCpUSxXmJhLVgcjuSotNTU2YMWMGgoOD8e233+LUqVM4cOAArl27hgsXLljIpK/Qh0wuXboEd3d3VFZWdgnPMgQaGhpw5coVjB07lrmNBmdFRkbq9I9S1ZYbUvarujLk8XgQiUTw8PBgLobUZ4k6r5pa+kpdBgwxjNcbUBUfi8VCfHy8TiLtfDGkvQFvb2+4urr2+HOjRCYSiZCYmNity4IxoJpFkpSUZFIik8lkTBpiW1ubmiy5JxYvfQUtrZmrtMjn85GXl6dTbNDS0oKZM2fCx8cHe/fuNfl3xVjoV2QCoNvwK0IIzp49C6FQiKFDhxpFndPc3IysrCyMHz++1412TYOIhkZ7ezvTwG/5Jxze09MTkZGRJp+ibW1tZXZthnB+7Slo+bG39vX0Ysjn8xmLF1XZsa46Nt2RAdCLyAwN1SySzjsyU4A6P9AeHSGE2QEKBAJGEOHl5dWrHqA+MDeR0CwVXWKPtrY2pKamwsnJCfv37ze5lY8xcduQCS0h1NTUIDAwENHR0UY5fltbGy5duoRx48YhNzcX7e3tSEpK0ln7V/XpMdUgIqCsTxcXF8Pf35+JCbWzs4OPj0+vV9k9AZX+hoWF6ZRHGwNUDu7j44OoqKg+H181DZHP56vtAL28vLpcqMViMbKysmBnZ4fY2FiT78hUs0gSExNNXlqkhpk8Hk/j70QkEjHqsIaGBkbRpCuWoCeorKxESUkJEhISzNKjoZP9ulST7e3tePjhh8Fms3HgwAGT9xONjX5HJhKJpMtgjVgsRk5ODuRyOezs7ODi4mI0PbZQKMTp06fh5OQEW1tbnStNqtiiogA60Wps0B9xXV0d4uPjmR+R6iqbz+f3aLivp6CrQWNKf7sDHQYMCQlBSEiIUd53VdlxS0uLmtKOzWYjOzsbbm5ueg2sGhqqWSTm2BHR0lpjYyOSkpJ0rrKpxQvdAUokEq3zQfqCNvvNTSRDhgzpNguno6MDjzzyCKRSKQ4ePGhyYYYp0O/JhNpQuLm5YdiwYSgqKgKHw8HgwYONcnwul4vs7GwEBQWZrNHeU6gqthISErT+iOkQFS2H0UyRvg5Kqkp/4+LizFJWoMNwUVFRCAwMNMkxJRIJ02epr6+HQqGAk5MTIiIi4OHhYVIyMXcWCa0UtLS0ICkpqcelNUIIBAIB8362trYyRE0TEXUtDsxNJI2NjcjJydE5kCoSiTBnzhy0trbi8OHDZhEmmAL9mkxqa2tx5coVhIWFMZkHRUVFkMvlGDp0qMGPXVVVhatXr0KhUGDChAk6dyTGaLTrAjUrtLGx6ZFii/54+zooqRrolJCQYJat+s2bN1FSUtKjYTxDQtVCh8PhgM/n9zpXpDcwdxaJao8mKSnJIA1kStS0HEZ31HSmpTNZVlRU4MaNG32WP/cW9DswePDgbhczYrEYTz31FOrq6nDs2DGzLLxMhX5HJlKpFHK5nLFyV80cAJTZF0KhELE6ctR7Aloyqq6uRmxsLDIzMzFu3Ditqy1TNtpVQRvd3t7efVax0UFJPp+PpqYmvQYlVe3b4+PjzaJYojsi6nNkatAekarzrWquCCVq1ZhdQxKuubNIFAqFmmrNGKSpapfD5/MhFouZhY+Xlxd4PJ5ZiYSKdHS5H0ulUjzzzDMoLy/HiRMnzDI8aUr0q6FF4JYypr29XeOEeV/SFjVBJpMhNzcXQqEQo0aNgqOjI2MTrQnmarRTjydDNbp7OihJpb/29vZmsW+nZZWmpiazDEMCt7LiOyt2OueK0KYzn89HaWkp7Ozs1GTHvSUAc2eR0HRIqVRq1Lx41UTEwYMHM30rLpeLwsJCAEBAQADTrzTl+9DS0oLs7GxERER0SyQymQwvvPACSkpKcPLkyTueSIB+SCbZ2dkghGD06NEav6yGzGgXCoXIysqCra0tRo0axRxP0zE6N9pNRSSEEKY2bKxGt7W1NWP8pzooSbMbXF1d0dLSwgxsmvoiRi1sJBIJRowYYRZdfnl5OcrKypCQkKBzGM7Ozg4DBgzAgAEDIJPJNFqSaBs81QbaOzRlzK0qVJv9SUlJJrX9oBYvdAcYEhKC9vZ2Ro6t6nhsTBECTYgMDw/vdixBLpdj4cKFyM/PR3p6ull82cyBflfmam1thbW1tdbVW01NDW7evIlRo0b16TiNjY3Izs5GQEAABg8erHa8kydPqjX1OjfaqWKrpISFtrauz+3sDAwa1Pe3VaFQMLLL+Ph4k2/pCSGoqKhASUkJrKys1PoCvVXf9BRisVit0Wxq76LOPmN9yYpXHTzl8/kQCoVqsmNtfSuaTkh3paYGjRnmcDiIj483+a4UUAY5VVRUICkpiVFCURk33QXScCk6LGnI3WtraysyMzN1fgZyuRz//ve/cfbsWZw8edIsIWDmQr/bmTg4OHS78zBEmYsmMEZFRWlcYaimLWozaiwpYSEuTnvTOje3o0+EQlfjIpHILDnlgHLyv7S0FDExMfDz82PKDdREjw6j+fj4GKXsRHeOrq6uiI6ONkt/gEpfDVFaUw3/ioiIgFAoZMo3RUVFGsO/aKiUudIJpVIpsrKyGMGHOYjkxo0buHnzphqRAMrfqbu7O9zd3REREcH0Aamdvr29vZrjcW+/P3SWKTQ0tFsiUSgUeOWVV5Ceno709PS7ikiAfkgmutCXMhdd6dfU1HSbwEjNHrtrtGvakahC19+7g6pia/jw4WaZHygtLWXMAqkChZYbQkJCIBaLmRX2jRs3DD4oSWvTAQEBiIiIMEt/ID8/H0KhEMOHDzfKVLmDgwMGDhzIOPTSFXZWVhY4HA4cHR3R1NSEoUOHmiULRSKRIDMzE/b29oiNjTU5mQNgvoediUQTVPuAquVF1ahdWl7UVzggEAiQmZmJgQMHdhtsplAo8Prrr+PQoUM4efKkWXaQ5sZdQyZUidTR0aEzgZHD4TCqsttZsdUbqEp/R4wYoXU1bmtrq9YXoIOS2dnZfR6UpKvx8PBws/woaX9AoVCYjMxV+1YKhQLFxcWorKyEtbU1rl27Bh6Px6yyTVFepPJjalFj6u+hqvtxcnJyjy2CrKys4OPjAx8fH7Wo3YqKChQUFMDV1VXN8VjT75sSSVBQEEJDQ7UeS6FQ4O2338aePXuQnp6O8PDwHr/eOwH9jkx0XbQ5HA6Toawv2tvbkZWVBXt7e7VGuyYQQsBmsyEQCBh9++2q2OopVKW/w4cP1/uiZWVlxViVq8o6CwsLmUFJSi66LsxUMTV06NBuJ4qNBYlEwpR1zKFaA5TzTnT37O7uDoFAAD6fj+rqaly7do0pL3Z3IewLVOXHQ4cONfn3kO6Mq6ure0UkndE5aldVbXfjxg3Y2Ngw7yfNvGlvb0dmZiYCAwO7JQdCCN5//33s2LEDJ0+eRGRkZJ/O9XZGvyMTXaA9E30lgQ0NDcjJyUFAQIBO7yZa1vL19WW217QnYGgrks7HpUNY5krlM5T0V1XWGRkZyQxK3rx5E1evXoWbmxtTDlPtA6kGSsXHx5tFSkkvoubq0WjLIqFBamFhYV3Ki7a2tsyFsC99AQqhUIjMzEx4eXkZxOusp1AlEmPZ6Kuq7VRDqwoKCiCTyeDq6orW1lb4+fnpJJJ169Zh27ZtOHHihFEGqW8n9Ds1l1wu73bnIZFIcOLECUycOFHnBe/mzZsoKirCkCFDdDbDVPsjVK2lakVCfYSoFcmVKzYYM0Z7U/zs2Q4kJOh+axUKBQoLC8Hn882i2AJuldZ8fHy6KNsMic6Dko6Ojsz7WVNTAx6Ph8TERLP4FtGShrmcj3uTRaIp70ZVdtzT8hxdjZtrjoUq52pqapCcnGzyWSJCCOrr65Gfn8+Uup2dnZnyoqrFCyEEGzZswKeffopjx44hMTHRpOfaH9HvyERXDrxcLsfRo0cxfvx4rU00eoGura3Vay5A10S7qhUJj8dDe3s72tsD8MQT2nPi9VFzUcWWWCxGfHy8WRRbtD9h6tKa6qAkn88HAPj5+cHf399k8boUzc3NyM7ORnBwsFlmOAyRRaIp/Et1alzXc1IyDQgIwKBBg8zyHhQXFzNkao6h1I6ODmRkZMDHxweRkZFqooiGhgZYWVnh0KFDiImJAZfLxSeffILDhw9jxIgRJj/X/ojbjkwIIThy5Ajuu+8+jT8QOkFPLbl1/Yh6Y40iFArB4/GQldUGPl8EJycnRqJoZ2en15xJR0cHcnJyYGtri9jYWLNkP1dVVaGoqAjR0dHdWmcbC6o9muDgYMZRlq6wfXx8ejTY1xtQtU9ERIRR8nF0wVhZJJp2gXTX0lltRwcig4ODERoaelcTibe3t8adqUKhQGNjI958800cOXIE9fX1GDVqFJ5++mlMnz7dJN+dNWvW4I033sCSJUuwYcMGjfdJT0/HAw880OX2a9euISoqyqjnd9v1TFgsllZFF220Ozg4YNSoUd1ehPoy0e7g4PCP7TmYGjaPV4PGxisAHOHi4oO2Nu0eVy0tLcjJyTF6WUkbtEl/TQkqf7azs2N6NP7+/swKm8fjobS0FPn5+fDw8GD6LIZUMtXW1uLq1atmI1PVLJLk5GSD+lx1tsvprLajxMLhcJCXl4fQ0NBupa/GAi3vcblcJCcnmzzqGFB+F2mfSFuJk81mw9PTE6NHj8a+ffvw3Xffgcfj4ddff8XixYtx6tQp3HvvvUY7x8uXL2PLli16exIWFRWpDdiawhC135GJPhd0TWRCG+2BgYE6a96qg4j0mL1djalKZFVLN+Xl5bC1tWXkiXQ1yOVyUVBQgPDwcAQHB5t8FdjZ48rUqYyAsqSSnZ0NDw+PLmaFqsqbiIgIZlCytrZWbVCyr0om6jxsrmZ/Z3sSY8qPra2t4efnBz8/vy5qO7FYDEdHR1hZWUEsFpvUqoYarPL5fLMTiYeHR7eCA0IIfvnlF6xYsQL79+/H+PHjAQCvvfYaGhsbjdrnEwgEePLJJ7F161a8//77ej2G+uqZEv2OTPRBZ3lwbxrtxsgg6exxRVeDNJvczs4OAoHA7GUlmUxmNo8r2p8ICgpCeHi4TjLoPChJyZoOSlK1nb6DknRXVlVVhaSkJLMIHsyZRULVdoQQVFVVISwsDGw2m3E1oA1nb29vo0XsAl2JxBz9QrFYjMzMTLi5uXXrOUcIwa5du7B8+XLs3r2bIRIKXT3ZvmLRokWYPn06HnzwQb3JJCEhASKRCEOHDsVbb72lsfRlaNyWZELlwaqN9uTkZJ3lGlNax3M4HGZXQsOsGhsbYWVlhWvXroHP55ukJ0ChKv2Nj483S4+GztH0tj9ha2uLwMBABAYGMmTN4/EYsqZ9Fm0ybtrorq+vN8j8Qm9g7iwS4FafSHWWJzQ0VC38q6ysTOP8hSFACEFhYSHzOZiDSOh0v6urq85Zmt9//x2LFi3Czp07MWXKFBOeJbBz505kZWXh8uXLet3f398fW7ZsQVJSEsRiMX7++WdMmDAB6enpuP/++416rv2uAQ9oz4GnuHTpEvz9/cHlco3aaDcEpFIp43ibkJAAW1tbpifA5/PR0dEBDw8P+Pr6Gi1USTUn3Rw9GuBWxK8x5mg05V/QngB9TxUKBfLz85lGtzkuYObOIgGUSaJXrlzpYqPfGarzFzT8S/U97UtKZ2FhIaNcMyeR0On+7q4Ff/75J+bNm4eff/4ZDz30kAnPEsz0/5EjRxAXFwcAGDduHOLj47U24DVh5syZYLFY2L9/v5HOVInbkkwuXrwIoVAIV1dXnU6ytNFujgwSuhuws7PTqtii0808Hg9tbW3MUJ+Pj49BlD1U+ksbrOZQ6lCjvvj4eKM3+6mMm76nNKhKIpGAzWYjKSnJqCmI2mDuLBLgluAgNja2Rw1ZTeFfbm5uanYk+j5PTzLjjQGpVIqMjAw4OjrqtIk5fPgwnnrqKXz33Xd47LHHTHiWSuzbtw+zZ89W22XL5XKwWCyw2WyIxWK9SqQffPABtm/fjmvXrhnzdPsnmXTOgVdFfX09s7obPny4yRrtPQVVbNGLhz6rUJFIxMyyNDc3w9nZmVEx9aYkU11djcLCQrNZk9AyZH19PRITE81SVmptbWX6RHK5nHHm9fHx0Stn3FDnYM4sEkD5XSgqKkJsbCy8vLz69FwdHR3MjqWxsREODg5q7ryaXh8hhBF+JCcnG8U4UxekUiljXKmrxHjixAnMmTMHmzdvxlNPPWWWz6ytrQ0VFRVqt82bNw9RUVFYuXIlYmJi9HqeRx55BI2NjThx4oQxTpPBbdMzoSFR169fh5ubGzw8PPSyRjFGo10XqGJr0KBBCAoK0vuLaGdnx8g5JRIJs7q+ceMGY6ft4+PD2JNrg6r0V5+hTWOAyl6phb45Lh4dHR3Iz8+Hi4sLhg0bBrlczlwEMzIymERJQ/cEVGHuLBJAWS4pLi5GfHy8Qb4L9vb2CAoKQlBQkJrJp7bwL0okzc3NZiWSrKws2NnZ6SSS06dP4/HHH8fnn39uNiIBlDY6nQnD0dERnp6ezO2rVq1CdXU1fvrpJwDAhg0bEBISgujoaEgkEmzfvh1paWlIS0sz+vneFmRCcyWoFr22trZb52Bz9Ueov1RZWRmGDRvWJ223jY0N02ymP1jloGSWWnO/sx9Tf5D+SiQSpimenJxscgt94FZZibov09KAqjMvtSIpKCiAXC5X6wkYQqBg7iwSAIznW2JiolGkoqomn4QQpndVUlLCzAhJpVJIJBKjWfnrgkwmYyIddFnpX7hwAY8++ig++ugjPPfcc2YjEn1RW1uLmzdvMv+WSCR49dVXUV1dDXt7e0RHR+PAgQOYNm2a0c+lX5a5pFIpU5qiFyapVMo0TouKiiCTyRAdHd3lsYQQRjZsyrIWJbyGhgbEx8f3KZFP13EaGxuZBj4hRC1QqaCgAFKplGn2mxodHR1q1uXmcN2lWSgDBgzQS36sakXC4/GYBMS+DErSRre5ZODArVCphIQEs0igBQIBrly5gvb2digUCjXZsalKjDKZDFlZWbCystIpw758+TJSUlLw3nvv4eWXX+73RNLf0K/JhK4unZyc1BrYpaWlaG9v7zINSvsj5lJsSaVSxMfHm2z1RWNgeTweuFwuRCIRbGxsEB4eDl9fX5PvCKhqzFxmiYByeDU3NxeDBg1CcHBwr56DDkry+Xy0tLSo9a70GZSk/Ym+7k57i86zNOYwzqS5OG1tbUhKSmJSI1V9rlRLjMZYdMjlcmRlZYHNZuuMG87OzsaMGTPw5ptv4pVXXrEQSS/Qb8mEy+UiNzcXwcHBXZL2ysvL0dTUhISEBADmbbSrzm8MGzbMLPMb9CLu5uYGR0dHxujPWDYkmtDY2Ijc3Nx/bGZMrxoDgLq6OhQUFBhUcEB7V/QiqGtQ8ubNmygtLUVcXJxZelXU56q2ttZoFu660JlIOn/3FAoF48PG5/PVMm8MJY+Xy+XIzs4GAJ2RCvn5+Zg2bRpeeeUVrFq1ykIkvUS/JJOSkhIUFhYiOjpaY1xpZWUl6urqMHz4cLM22pubm5GTkwM/Pz+zrcS1SX+pGSVdXbu4uDB9FkPbVtCL+JAhQ8wSLwvcajIbQq2kDaquBtTpmBKLu7s7KioqGNGDOcpKqlPlvXUf7iuocWV7e7teMmxVKTefz0dbWxtcXV2ZXYuDg0OPf1dyuZxJykxMTOyWSK5evYqpU6fipZdewurVqy1E0gf0SzIpKyuDg4OD1oZhTU0Nbt68iZEjR0KhUEAul5u0rAUoL6BXr17tUzmlr9BX+nvLjJKHxsZGJkfEx0e7GaW+oB5X5izpqPYGTOVHpFAomBIjn8+HSCQCAISFhWHAgAEmn2VRld6aa4ajp0SiCaopiI2NjT0O/5LL5YwTdUJCQreVguvXr2Pq1Kl49tln8eGHH1qIpI/ol2RCZwK0gcfjobi4mCGT21Wx1ZdzoNLfnpZTVM0oGxoaGMsMqgzT932k5ZSamhqzr8RpqJY5Sjr0Il5fXw8/Pz80Nzczw6eqq2tjgpaVWltbkZSUZBbFFHUYEAqFBhsM7bwTJIR0G/6lUCiQm5vLiHW6I5LS0lJMnToVjz76KD755BOzuBHcaeiXZKIrbbG+vh45OTmIiIiAj4+PyVaBqoqthIQEszU26Qo0ISGhTxdQaplBV9csFkstpljbD4xevFpaWpCQkGCW/AnVczD3SrxzFolIJGIugHQnaKxBSdWLeGJiolkUfPQcOjo6kJiYaJTfIxWb0PdVKBQy4V+0J0jt/BMTE7sVn5SXl2Pq1KmYOXMmNm7caCESA+G2IhNqjSKTyVBRUcHYZdBGszGJRdVx15SKrc7nQFVjhpb+Un8rSizUi6mzGaVMJmNWf+aSH9NShkQiMdrFS59zUL14aTsHmiXC4/FQX18PKysrRhTR10FJfc/BmFAoFMxwqrGt9FUhFAoZYmlubgaHwwGbzUZMTEy3A83V1dWYNGkSJk2ahM2bN1uIxIC4bchEW6O9o6MDXC4XPB4Pra2tBve2ApRf3JycHLMqtmiYlCmSGTvPXVAzSg8PD1RXV8PW1lanJ5qxIJVKmYAnc7kfq2aRxMfH630BVR2U5PP5fRqUpE1m2hswx2AoLSuJxWKTEknnc8jLy0NraytcXV3R2NgINpvN7FhUHaRra2sxZcoUjBkzBt9++61ZZqDuZPRLMukc3Uulv7oa7Z29raiCydfXt9dlEKrY8vf3N5tBX1tbG7Kzs+Hl5YWoqCiTr6ba29tRVVWFyspKEELg5uYGX19feHt7m7S8pGrfbq6BSENlkWgblKQXwe4WQnSiG4DOJrOxQImE7g7NRSSdG/6dHaQrKirw008/Ydy4cdi9ezcSExPx448/muU9u9PR78mkt9YoEomEIZbGxkY4OTkxxKJvjZ8qtsyVDw4oh/Dy8vLMOr9BJ8oDAgIQFBSkli1O31eqDDMWaCSzpnRGU8GYWSS0bMPj8bodlKQ7Mw6Ho3MQz1igZUaZTGa2XREhhBEdaIs8JoSgrKwMmzZtwsGDB1FRUYERI0YgJSUFM2fORHR0tEl+T/pktwPAqVOnsHz5chQUFCAgIAArVqzAggULjH5+hkK/JhNDTbRLpVLmh9rQ0AB7e3uGWDRJY+mXsLy83KhzC7pgbtdf4NYcS3h4eBejQhqmRN9XOzs7hlh0mVH2BNR1NzAwEIMGDTILoZoyi6TzoCSNf3Z3d0dJSQkTaXA3E4mqcWR3fbvGxkbMmDEDISEh+OKLL3D06FHs378fGRkZKC8vN/oO5fLly3j00Ufh4uKCBx54QCuZlJWVISYmBi+++CLmz5+Pc+fO4aWXXsKvv/6Khx9+2KjnaCj0SzKRy+WQSCSMPNiQE+0ymYy5ANbX18PGxga+vr7MBZB+URsbG82m2FKdnTDXJDWgnOe5du2aXv5S1JGXvq/UjLKvjebGxkbGdTckJKRXz9FXmDOLhMpj6+rqwOVywWKx4OfnxwgjTEkoqn0aXdJbY4FmotB5mu7KgS0tLZg5cyZ8fX2xZ88eNdKRy+VGf++oym/Tpk14//33uw21WrlyJfbv36+WObJgwQLk5ubiwoULRj1PQ6FfksnHH38MPp+P1NRUDBs2zGgfumr0K5/PZ45DQ5TMJTelAUJ9lf72FnSWhu7MPD09e/R4apfB5XLV5gN6egGkZonmnKzvD1kkIpEImZmZcHZ2RmBgIDPUJxaL4enpCR8fH6OldFKoTpWbq0+jmtKoy8q+ra0NKSkpcHFxwf79+82ivpw7dy48PDywfv16nQmJ999/PxISEvD5558zt+3duxePPvoohEKhWXaAPUW/7EINHToU33//PSZNmgRfX1+kpKQgNTUViYmJBi0vqFq509Uni8WCTCbD33//zfzNWFkXnaEquzWXXTcdBKR2/73ZmbHZbHh6esLT01PNjPL69etMpC69AGr7kdCY356mAhoS/SGLRLW8RrPKPT09ERkZifb2dvB4PFRWVuLq1atwdXVldoOGHJTsL0RSVFTE5MZ399tob2/HI488Ant7e+zdu9csv6OeZrfX1dV1iVH29fVlKinmKnP3BP2STKZPn47p06ejvb0dBw8eRFpaGmbOnAk3NzfMmjULqampGDFihMF2LE1NTcjNzUVAQAAiIiKYXAYej4crV64wNu90ZW0MYlGV/iYnJ5vlByuXy5khvBEjRhhkZ8ZiseDm5gY3NzdERERAIBCAx+OhoqICBQUFjILJx8cHtra2aruihIQEo8f8akN/yCIRCoXIzMxkVHyquyIWiwUnJyc4OTkhLCxMbVCyuLiYGZSk0QS93VFRw0RCiFmJ5Pr16+Dz+UhOTu72e9nR0YFHH30UALB//36zDNRWVlZiyZIlOHLkSI+ITFPvVtPt/RX9ssylCR0dHThy5AjS0tLw559/wt7eHjNnzkRqairuueeeXn/JaS52ZGSkRsUWXVnTWRapVMpc/Ly8vAxCaOaW/gJKkYLq7IQpBuA6OjoYxR01o2SxWIzU0xz9KqB/ZJG0t7cjIyMDfn5+Pe7TqA5KNjQ0gMPhqBlS6vv9ohJkFoul03nXWKC2PXV1dUhOTu52xyUSiTBnzhy0trbi8OHDZrH4AXqX3X4nlLluGzJRhUQiwbFjx5CWlob9+/eDzWZjxowZmD17Nu677z693njVJvewYcP0UmwRQtDW1sYQi0gkYko23t7evSI0Kv0dOHAgQkNDzVaTz87ONqtKiEbstrW1gRBiUDPKnsDcWSSAcnGRmZmpd7hXd6D9K9oXlMvlTJ9Fk78VBSUSfbJAjAVCCEpKSlBbW6uTSMRiMZ566ilwuVwcPXrUbDtaoHfZ7StXrsQff/yBq1evMrctXLgQOTk5lga8qSCVSnHq1Cns3r0b+/btg1QqxYwZM5CSkoIHHnhAo2xQ1d8qPj6+VytgQgja29sZYmlvb2d+pN7e3nqt7KlaypzSX4FAgOzsbLPOb6jmxdO+GG0y19fXw9raWi2m2FjEYu4sEuBWwz84OBhhYWEGfW7VQUnVzJvOg5L9gUgAZRRFdXU1kpOTuy1XSaVSPPPMMygvL8eJEyd6LBgxBTo34Dtnt1Np8Pz58/Hiiy/iwoULWLBggUUabC7I5XKcPXuWIZa2tjZMmzYNqampmDBhAuzt7VFXV4cDBw4gJiYG8fHxBvOWos1QHo+HtrY2uLu7MxfAzsfoL9Lf5uZmZGdnIygoqM8r4N6CltcAaLQm6a0ZZU9APw9zZpEAtz4PUzX8NQ1Kenp6gs/nw8bGxqxEopoU2Z2iUSaT4bnnnkNhYSFOnDgBHx8fE56l/uhMJs8++yzKy8uRnp7O3OfUqVNYtmwZM7S4cuVKy9Bif4BCocDFixexe/du7N27F/X19Rg3bhwyMzMRGxuLXbt2Ge2H0rkX4OrqytiP2Nraml36C4ARF5hzul8sFiMrK0vv8lpnM0qpVKqmDOtNmZE2d+vq6syWTAjcmqcx1+chkUhQV1eH0tJSyGQy2Nvb9yqawBAoKytDRUUFkpOTu/085HI55s+fj+zsbJw8edJs/S0LlLhjyUQVCoUCW7duxdKlSzFgwADU1NRg4sSJSElJwdSpU+Hi4mK0Y4vFYoZYmpqawGazweFwEBsba7a6LpXdxsTEmG0lJxQKmajhoUOH9niHQftX9L2lZpQ9KTPSAbiGhgazJRMCt3LrBw8ejMDAQLOcg1QqRVZWFmxsbBAdHa3mbwWAKYUZe1CSKvl0CTDkcjlefvllnDt3Dunp6WZ73yy4hbuCTH799Ve88MIL2LBhA55//nnk5eUxO5bS0lJMmDABKSkpmD59utFWYdTXCQBsbW3R1NTENJmpX5ixV3+qoVrx8fFmIzOaWe/v74+IiAiDvO7OZUbqHq3NjFJbFompwefzkZeXZ9a+mSqRxMXFqRE7lcnTchgdlKTkYkjVX0VFBW7cuIGkpKRuF3gKhQLLli3D8ePHcfLkSbPNAFmgjruCTI4dOwZCCCZOnKh2O12Z7t69G3v27MHVq1cxbtw4pKamYsaMGfD09DTIhY5Kfz09PZkmt2riYX19Pezs7BhbF0MHKAH9Y7IeUJZzcnNzjWpcSWcu6G5Q1YzS0dGRsS03Zw4IcEuCHBMT02VgzVSQSqXIzMxkYgW62yFS0Ql9b1Xz2n18fPq0s6PiB32IhCqf0tPTDS5SsKD3uCvIRB9QGSIllpycHIwZMwYpKSmYNWsWfH19e3Xh00f6S32tuFwu4xdGL36urq59vuCqqqUSEhLMtgqnfRpTlnM6m3za2tpCoVDA2traYPGyvQGdbzLnhD8lEtqz6mmpsXOipIODA7Mb7MmgZGVlJUpKSpCYmNit+EGhUOCtt97Crl27cPLkSURGRvbofC0wLixkogF0CjstLQ179uzB5cuXMWrUKMyaNQspKSkIDAzU64dCpb898Zai6iXqa6Vq+eLu7t5jYpFIJMjJyQGLxepRkJOhQd+LYcOGma1PQ61JaKRBb4f5+go6yxIXF2c2GatEIkFWVhYT+NbX105tP6icm763NKBK2/NXVVWhuLgYCQkJcHNz0/r8hBC89957+PHHH3Hy5EkMGTKkT+drgeFhIRMdIISgqqoKe/bswZ49e3D+/HkkJSUhJSUFKSkpGDhwoFYL+4qKil4ZJVJoMkykxKKPLLajowNZWVlwcnIyW5gUoGyqlpWVmVUG3TmLBAAzzMfj8aBQKNQsc4z1XlVWVqK4uBjx8fFmey8kEgkyMzPh4OBg8FwW4Nb3lu4I6aAkTZSkCxpKqrpscwghWLduHTZv3owTJ04wn58F/QsWMukBCCGoq6vD3r17kZaWhtOnTyM2NpYhlkGDBkEikWDDhg0YNWoUEhMTDWYJQghRu/jJ5fJuL360ye3r64vBgwebZYaEWmHU1NQgMTHRqKq57qAri0TVjJI2mWmcrre3t8F2c5RUda3CjQlKJDSt0ti7MVXVHR2UdHd3h62tLbhcLhISErolVUII1q9fj88++wzHjx9HQkKCUc/Xgt7DQia9BCEE9fX1+P3335GWloYTJ04wwU1CoRDHjh0zmu6dTjLT6XuJRKI2b9Ha2mr0JrcuqDb8ExMTzWK4B/Q8i4Q2mSmxCAQCZgBVV5xud6BDquYkVVMTiSYIhUKUlpairq4OAODi4sKQdmfbHEIIvvzyS6xduxaHDx/GiBEjTH6+FugPC5kYADRQa+rUqZDL5RAKhfDz82Os8435wyWEQCAQMMQiFApBCEFgYCAiIiLM0iORy+XIz8+HUCg0q+zWEFkkmswoVeN0dYHKsaurqw26U+0pxGIxk4kSHR1tFiIBbkVhx8XFwdnZWa3PYmtrCzc3N9y8eRMTJkzADz/8gNWrV+PgwYO45557zHK+FugPC5kYANevX8eECRMwceJEfPPNN+jo6MCff/6JtLQ0HDp0CP7+/pg1axZmz56NhIQEo/2QKyoqUFJSAl9fXwgEAggEAmaQz8fHxyTKJalUitzcXCb7wlwNf2NkkUgkEqZc09DQwNi8a5Nz0+l6LpeLpKQks+3O+guRUCl0XFxcF2NVKjy5dOkS/vWvfzFx3atWrcKyZcuMPlC6efNmbN68GeXl5QCA6OhovPPOO5g6darG+6enp+OBBx7ocvu1a9cQFRVl8POj0eX9GRYyMQCam5vxyy+/4KWXXupyQREIBEwmy19//QUPDw/MnDkTs2fPxvDhww3S6FXtTah6SwmFQmZV3draCjc3N8bWxRi7BbFYjOzsbGb4zVwNf1NkkXSOf+5sRgkAhYWFqK+vN+t0vVgsRkZGBlxdXREdHW22bAwej4f8/HydUmhCCH7++Wd8+umnSExMRHZ2NiorKzFp0iT8+OOPRus1/fHHH+BwOBg0aBAA4Mcff8THH3+M7OxsREdHd7k/JZOioiK1sqW3t7fBvveUQOj/trW1oaCgADY2NkhMTDTIMQwJC5mYEEKhUC2TxdHRkclkGT16dK+8pRQKBQoKCtDS0oKEhAStq1+RSAQejwcul8uUa+iQpCFCsGiTm160zL36NWUWSWczSgCwtraGTCbT6XhrTNC4X3MTCZ3y1yULJ4Rg165dWLx4MXbv3o0pU6YwUb2HDx/GkiVLTPoaPDw88PHHH+P555/v8jdKJk1NTUYhuOzsbPzxxx9YuHAhvL290djYiOHDh8PKygrFxcWYP38+li1b1q9mbSxkYiaIRCIcP34ce/bswe+//w4Oh8PsWMaMGaNXeUg15jchIUFvB2SxWAw+nw8ul8tMiFNi6c2Frz8ox4D+kUVCP5PW1lZwOBzIZLI+m1H2BiKRCBkZGWpxv+ZAfX09cnNz9Zry37t3L/71r3/ht99+w4wZM0x0hl0hl8uxa9cuzJ07F9nZ2Rg6dGiX+1AyCQkJgUgkwtChQ/HWW29pLH31Br/88guefvpprFy5EkuXLsXy5cthbW2NVatWobS0FM888wweeOABvPPOO/1GKm0hk34AqVSK9PR0pKWlYd++fZDJZEwmy7hx4zSSBC0pWVtbIy4urtcXKTohzuVymT5AT0KpqG26OcO9gP6RRaJQKBjhQVJSEqytrdXMKIVCYY8zb3oDSiQ0o8Zcnwk1sBw6dKjOXeKff/6JefPmYfv27Zg9e7aJzlAd+fn5GD16NEQiEZycnLBjxw5MmzZN432Liopw+vRpJCUlQSwW4+eff8bXX3+N9PR03H///QY5nx07duCpp57Cm2++ifr6eixatIgJ1srIyMDMmTMxatQo/Oc//0FsbKxBjtkXWMikn0Emk6llsggEAkyfPh0pKSlMJkt+fj4uXryIUaNGGbSkRPsA1NbFzs6OIRZN9hh8Ph/5+flmtbHvL1kk1LKmO78vKjnm8/lobW2Fq6sr8/4aotQI3Co3mptIqKX+kCFDdBpYHjp0CE8//TS+++47PPbYYyY6w66QSCS4efMmmpubkZaWhm+//RanTp3SuDPRhJkzZ4LFYmH//v09PjYhhPmsRCIR09Pcu3cvE4515MgRPPjgg0wPJT8/HzNnzkRkZCTWrFmDpKSkHh/XkLCQST+GXC5nMln27duHhoYG3HfffTh79izmzJmDzz77zGgXC7lcjoaGBoZYrKys1BrMVOJpzpz0/pJFIpfLkZOTA7lcrreCTZcZZW9lzJmZmfD09ERUVJTZiSQqKkqnjdCJEycwZ84cfP3113jyySfNds6a8OCDDyI8PBzffPONXvf/4IMPsH37dly7dq3Xx+RyufD19UVHRwd+++03PPvsszh27BgmTZqEp556CmvXrkVAQABDPgUFBbj//vuxe/dug5XYegsLmdwmUCgU2LhxI1asWIFBgwahoqICEydORGpqKqZOnWrU+QWFQoGGhgZmVU1lm+Hh4Rg4cKBZmu39JYuERtxS77PelBs7m1HSHaG3t7feRp8dHR3IyMiAl5eXWYmkqakJ2dnZepl5nj59Gv/3f/+Hzz//HPPmzetXRAIAEyZMQFBQEH744Qe97v/II4+gsbERJ06c6NXxPv/8c3z33XdIS0vD5MmTkZiYiJ07d4LD4eDgwYOYOXMmnnvuOaxevVqNUNra2sw2v6QK03QDLegztm/fjjfffBM///wz/u///g+5ubnYvXs3PvroIyxcuFAtk8UQTsOqYLPZjK9SSUkJKisr4e3tjZs3b6KiokLN1sUUxKKaRTJ8+HCzDUVKpVJkZ2fDysqqT1Joa2trBAQEICAggNkR8ng8ZGdn62VGSYnE29vbrAII2j+LjIzUSSTnz5/Ho48+io8//rhfEMkbb7yBqVOnIigoCG1tbdi5cyfS09Nx6NAhAF0z2zds2ICQkBBER0dDIpFg+/btSEtLQ1paWo+Oq1rSCgkJgZ+fH4YPH47o6Gjs2rULgHLBMnXqVBw6dAhTp06FQqHA6tWrGdk73ZGrlsrMAcvO5DbBvn374OLigvHjx6vdTqfvqXX+tWvX8MADDyA1NRXTp083WCYL3QnQuQlHR0cmOIk2mKlyydfX12hmifr0JkwB6rpra2urV+Rwb0ANE7szoxQKhcjMzISPj49edjHGQktLC7KysjBo0CCd/bPLly9j1qxZ+O9//4uXX37Z7EQCAM8//zyOHz+O2tpauLq6IjY2FitXrmQykDpntn/00UfYsmULqqurYW9vj+joaKxatUprw14TRCIRRo8ejcceewyvv/46AGDixIk4c+YM4uPj8dNPPyEyMhIymQwsFgscDgdnzpzB2LFjMXPmTPzyyy9mK+1qgoVM7iDQ4UVKLLm5ubjvvvuYTBYfH59e/XCpSqm9vV2rPQr1C6OzLNQs0dfX12CSWJlMhpycHBBCzGqnTyfKqQOxKXZj1IySlsNEIhHc3NzQ2toKPz8/s5a2WltbkZmZifDwcAQHB3d73+zsbMyYMQNvvvkmXnnllX5BJOZCfX091q5di++//x6vv/46XnvtNZw8eRItLS3Ytm0buFwuvvnmGyQkJEAikTALp0uXLuHSpUv497//beZXoA4LmdyhoDb4NJMlIyMDo0ePZjJZAgIC9Poh07kJmUyGhIQEvbPVBQIBQyw0n51O3/eGBCQSiZoU2lzT9aqDgL3JrjcEqMlofn4+OBwOpFKpQcwoewNKJPrY1uTn52PatGl45ZVXsGrVqruaSCgaGxuxadMmfPbZZ3j33XexZMkSAMBff/2FzZs3o7a2Ft988w2SkpJQVFSE3bt3Y9WqVcz3ztylLVVYyOQuACEElZWVTCbLhQsXmEyW1NRUBAcHa/xC0gs47Qn0dndBJbFcLpdx4aXEos+gZecsEnNN1/cX2W17ezsyMzPh7++PQYMGMe4GqmaUtBxmzOn7trY2ZGZmMu7U3YEaoS5atAjvvvtuv7kAmgtSqZRZVO3Zswc7duzAnj17sG7dOrz22msAgMOHD2PLli3IysrC/PnzsWbNGrzwwgv49NNPzXnqWmEhk7sMhBDU1tZi79692LNnD5PJkpqaipSUFISHh4PFYqG4uBhXr15FaGioQV2PqQsvl8tlZi3o9L2mFbWuLBJTob29HVlZWWZvcre3tyMjIwMBAQFM5IEqJBKJmjKMRulqM6PsLQQCATIyMphh1e5QVFSEqVOnYt68efjwww/veiJRxUMPPYTm5mYEBQUhOzsbxcXFeOWVV/D+++8DUAoVdu7cib///hvTp0/H22+/DaB/7UgoLGRyF4OWSyixnDhxAlFRUXjggQfwyy+/YM6cOVi3bp3RvrR01oLL5aK5uZmxd/fx8YGDg0OPs0iMBYFAwOwEIiIizH4egYGBDOl3B21mlN7e3r2KgO58HgMGDEB4eHi39y0pKcHUqVMxZ84cfPzxx/3e+dbYUCWBb775Bu+99x7OnTuHkJAQVFdX48cff8S6deuwaNEifPjhh8zjBAIB02yXy+VmK/N2h9uKTNasWYM33ngDS5YswYYNGzTex9TW0HcKaJLj+vXrsW7dOgwcOBBWVlZITU1Famqq0c0bqb07j8dDY2Mj7O3tIRKJEBAQYNbmMi3l0Avn7UIknUFnheiuBQBTCvPw8ND74kR3RoGBgYzDrjaUl5djypQpmDVrFjZu3HhXE8n8+fO7DD++/fbbOHbsGC5cuMDcxuVy8f777+Orr77CO++8g9WrV6s9pj/uSChum0/38uXL2LJli94eNEVFRaitrWX+i4iIMPIZ3t5gsVi4fPkyNmzYgE8//RQZGRl46623UFRUhPHjxyMhIQFvv/02srKyoFAoDH58GxsbDBgwAImJiYiLi4NIJIK9vT1qampw4cIFlJSUoK2tDaZc+7S0tDClHE0lJVOBlpQGDBjQ6/Ogs0JDhw7F2LFjmR5YYWEhTp06hby8PNTV1UEmk2l9DipDDggI0LkjqaqqwvTp0zF16tS7nkhyc3MZAldFTEwM6urqkJ2dzdzm6+uLqVOnwtHREe+99x5+++03tcf0VyIBbpOdiUAgQGJiIjZt2oT3338f8fHxOncmxrKGvpPxwQcfIDQ0FE888YTa7QKBAH/99RfS0tJw8OBBeHp6qmWyGPJC0TmLpHOpxsbGBj4+PvD19dXoF2Yo0AE8Q4Zr9QZ0ZxQUFKTzAt4baMpo1xSoJhQKkZGRAT8/P52lvtraWkyePBn3338/tm7d2i9LMqaETCZjxCubNm3C888/D1tbW2RnZ+PFF1/E6NGjsWDBAiY35fTp09i6dSsWL16MkSNHmvPUe4Tbgkzmzp0LDw8PrF+/HuPGjdOLTIxlDX23QygU4vDhw0hLS8OBAwfg6OiIWbNmMZksfblw6MoiUZ0O5/P54HA4zEWvLz2AzqDeUuY0sARuEUlwcDDCwsJMcsz29namFEYFEu7u7qiuroafn5/O3hWXy8XUqVORnJyMH3/88a4nkl27duHAgQP44YcfUFdXh8GDB2Pw4ME4ffo07Ozs8NNPP+HDDz9ETEwM7r33XkRERGDFihWYNGkSc43rrz2Szuj3ZLJz50588MEHuHz5Muzs7HSSiSmsoS1QQiQS4dixY0wmi7W1NbNjuffee3s0T9LTLBKFQsEEUvF4PLBYLHh7e8PX11er7Yg+oDsjfUwKjQlKJPqopYwFkUiEmpoalJWVQaFQdMm96Uwq9fX1mDZtGoYOHYodO3aYLLulP+Onn37C888/j19++QWPPvoocnJy8OSTT8LGxgZnz56Fo6Mj9u7di/3792P//v0ICgpCVFQUdu7cCaB/90g6o1+TSWVlJZKTk3HkyBHExcUBgE4y0YS+WENboB+kUilOnjzJZLLI5XLMmDEDqampGDduXLfDjn3NIlEoFGq2LnK5nCGWnjSXabSsOZ2QAeUgYFZWllmJBLiVi+Lp6Ynw8HDU19eDz+ejvr4etra28PHxgVQqRVRUFFpaWjB9+nSEhYXht99+M7rNTU8z2wHg1KlTWL58OQoKChAQEIAVK1ZgwYIFRj3PmpoaLFmyBCKRCJs2bUJQUBDy8vLw5JNPAgDOnj0LV1dXiMViCAQCiEQixtfsdtmRUPRrMtm3bx9mz56t9obK5XKwWCyw2WyIxWK93mxDWENboD9oJsuuXbuwb98+CIVCTJ8+HbNmzcKDDz7IzJMoFArk5eWhubnZYFkk1HaEzrJIpVI1Wxdt3xdaYtMVLWts0Iny0NBQnYOAxgTNjqfzPaqrY9Vy41NPPYWGhgZ4eXnB1dUVx48fN4mDbU8z28vKyhATE4MXX3wR8+fPx7lz5/DSSy/h119/ZfJC+gqaM9LS0qL2Xf7jjz8YafRLL70EuVyOa9eu4emnn4ZEIsGZM2e6LKLoc91O6Ndk0tbWhoqKCrXb5s2bh6ioKKxcuZJJHdOFvlpDW9B7yOVyXLhwgclkaWpqwuTJk5GSkoK//voLV65cwdGjR41iWKfaXOZyuRCJRGoRurQMV1NTg8LCQrPG/QK3zBLN3fSn3mPUMqa7Mgufz8fcuXNRU1MDgUAAoVCImTNn4qWXXsLo0aNNeNbdZ7avXLkS+/fvV1tQLliwALm5uWrS3L7iyJEjePnll7Fq1So888wzDCGsWrUKX375Jc6dO4fY2Fgm2/7ZZ59FUVERKioqzBbsZij0a+pzdnZGTEyM2n+Ojo7w9PRkiIR+aBQbNmzAvn37UFxcjIKCAqxatQppaWlYvHixxmOsXr0aLBZL7T9dJY5Tp04hKSkJdnZ2CAsLw9dff224F32HgcPhYMyYMdiwYQNu3LiBI0eOICgoCIsWLcKff/6JsLAwHDx4EG1tbQY/NovFgouLCwYNGoR77rkHI0eOhJOTE8rLy3Hq1ClkZ2fjypUruHbtGuLi4ixEAuW8T2ZmJlxcXHQSiUAgYAKtcnJyUFVVhcOHDyMwMBD19fUmO2e5XI6dO3eivb1dK4FduHABkyZNUrtt8uTJyMjIgFQqNdi5XLp0CcXFxViwYAGWL1+OX375BXK5HKtWrUJSUhI++OADpsc3ZMgQfP/993j22WdveyIB7oA8k9raWty8eZP5t0QiwauvvqpmDX3gwIFuraGjo6Nx7Ngx5t/dlc7Kysowbdo0vPjii9i+fTuzXfb29jbYdvlOBZvNRkJCAtavXw9/f3989tlnOHPmDNauXYsFCxbgwQcfREpKCqZNm2bwTBYWiwUnJyc4OTkhPDwcQqGQmUVisVgoLy+HUCiEj4+PXn5hhgSVIevjumtMUCJxdnZGdHR0t++/UCjEo48+ChaLhT/++IMJJxs5cqTJ5KydM9v37t2rNWK3rq4Ovr6+arf5+voy0nNd0cLdgRZ3WCwWFi5ciKamJshkMshkMuzbtw87d+7ETz/9hEceeQSbNm3CpUuXMHPmTADA0KFDbzvVljb06zKXKbB69Wrs27cPOTk5et3fVNvlOxW//vorPvnkExw6dIjZCRBCUFBQwFjnFxUVqWWyeHh4GFzRUl5ejrKyMiYTRdUo0RjZ7NpAiUSfHBBjQiqVIjMzEw4ODjq92EQiER577DEIBAIcOnTIbKvqnmS2R0ZGYt68eVi1ahVz27lz5zBmzBjU1tb2SnChqa9BCMGGDRuQk5PD+Gi98sorKC0txauvvoq33noLgwYNwvHjx29r4tCEfl3mMhWKi4sREBCA0NBQzJkzBzdu3NB6X1Ntl+9UzJkzB2fPnlUrKbFYLMTExGD16tXIzc1FXl4eM/AWFhaGWbNm4dtvvwWXy+3zBDwhBKWlpSgvL0dSUhJcXV1hb2+PgQMHYvjw4bjvvvvg5+eH+vp6nDt3DpcuXUJZWRna29v7+tK7oLm5We9AKWOCEom9vb1OIhGLxXjqqafQ1NSEv/76y6zlGRsbGwwaNAjJyclYs2YN4uLi8Pnnn2u8r5+fH+rq6tRu4/F4sLKygqenZ6+Oz2azUVhYiDFjxiAnJwcCgQAsFgtLlizB9evXsXLlSgwaNAi///47XnjhBZw/fx6urq44ffq01vO8nXHXk8nIkSPx008/4fDhw9i6dSvq6upwzz33oKGhQeP9dW2XLegeLBar29U+i8XC4MGD8cYbbyAjIwOFhYWYPHkyduzYgcjISEydOhWbN29GdXV1j4mFEIKSkhJUVVUhOTkZLi4uXe5ja2uLoKAgJCUl4f7778eAAQPQ3NyMCxcu4MKFCygtLYVAIOgzqTU1NSErKwuRkZFmJ5KsrCzY2dnptPeXSCRMs/3w4cNwd3c34ZnqBiEEYrFY499Gjx6No0ePqt125MgRJCcn9ylk7ebNm+jo6MD48ePxn//8B6dOnQKbzcbvv/+OyspKrF+/HgCwdOlSLF++HCtXrsT48eOxcOHCXh+zv+KuL3N1Rnt7O8LDw7FixQosX768y9+NsV22QDdoJktaWhr27t2LCxcuIDk5GSkpKUhJSdGayaL6+KKiIvB4PCZ2uCeQSqVqti52dnaMrUtPrd2bmpqYrHSa420OyGQyZGVlMYFj3RGJTCbDc889h8LCQpw8edKsYgVAc2b72rVrcejQIUycOLFLZjuVBs+fPx8vvvgiLly4gAULFhhMGvzRRx9hz549qKurw4IFC/Cvf/0LP/74I65fv46lS5di8ODBzH3pIKKqzcqdgDvnlRgINICpuLhY49+NsV22QDdYLBaCg4OxbNkyLF26FDU1NYx1/ttvv424uDgmkyUsLEzt4k7z6xsaGpCcnMw0i3sCa2tr+Pv7w9/fH3K5nCGWjIwMxtrd19dXp3CAWrUMHjyYGU4zB2QyGRN8Fhsbq5NI5s+fj4KCgn5BJIByLujpp59Wy2ynRAJ0FeaEhobir7/+wrJly/DVV18hICAAGzdu7DORUGJYsWIFJkyYgAMHDmD16tW4dOkSoqKicOnSJZw5c0aNTOj3404iEsCyM+kCsViM8PBw/Otf/8I777zT5e8rV67EH3/8gatXrzK3LVy4EDk5OZYGvBlACAGfz2eI5eTJkxgyZIha2Nfzzz+PsWPH4sknnzR4pK1cLlezdVH1C3Nzc1O7SFMiMbdVi1wuR1ZWFthsNuLj47ttBMvlcixevBjnz59Henq6WQnQ3Kivr4enp2eXxULnRnxBQQH+/e9/w8rKCidPnoRMJsPly5eRlJRk6lM2Ke76nsmrr76KU6dOoaysDJcuXcIjjzyC1tZWzJ07F0DXOZYFCxagoqICy5cvx7Vr1/Ddd99h27ZtePXVV7Ueo6ezLOnp6V3uz2KxUFhYaLgXfoeAxWLBx8cH8+fPx6FDh1BXV4elS5ciMzMTo0ePRlxcHC5evIiQkBCjWHxwOBx4e3sjOjoaY8eORXR0NBQKBfLz83H69GkUFBQwNiT9hUiys7PBYrF0EolCocCyZctw5swZHDt27K4mkv/9738YNWoUMjIyuvTLVIlEoVAgOjoaaWlpeOaZZzBhwgS4uLiY1c3AVLjrdyZz5szB6dOnUV9fD29vb4waNQr//e9/GXnhs88+i/LycqSnpzOPOXXqFJYtW8Z4/KxcubJbj5/Vq1dj9+7dXWZZtJULqPNxUVGRWpPY29v7jpMTGgtisRgPP/ww8vPzER8fj6NHj2LAgAFM7r2uHkFfQQhBc3MzuFwu6urqIJVK4erqipCQEHh6eprlc5TL5cjJyYFCoUBiYqJOIlmxYgX+/PNPpKenm8y1uL+io6MDw4YNg7OzM7755hsMHz5cazlTdacil8shEAjg6up628+R6MJdTyamQE9nWSyZLH0DIQQPPfQQqqurcejQIXh4eKCtrU0tk8XLywuzZs3C7NmzkZycbDRiqa+vR25uLkJCQiCXy8Hj8SCRSNRsXUxRO5fL5cjNzYVcLkdCQkK3x1QoFHjrrbewa9cupKen3/XBcrRRLpVKkZycDIlEgu+++w4jR47U+3tzO7n/9hZ3fZnLVOjJLAtFQkIC/P39MWHCBJw8edIEZ3lngMVi4eWXX8bRo0cZAz1nZ2c89thj+N///gcul4tPP/0UDQ0NSE1NxZAhQ/Daa6/h3LlzkMvlBjsPamcfHR2N8PBwREZG4t5778Xw4cPh4OCAGzdu4NSpU8jJyUFNTY3R5pSooaZMJtNJJIQQ/Pe//8XOnTtx7Nixu55IAGWjXCaTwdraGjk5OXB0dMTcuXNx/vx5vVNH73QiASw7E5Pg4MGDEAqFiIyMZDKeCwsLUVBQoFEBZslkMR1EIhGOHj3KZLLY2tpi5syZSE1N7XEmiyr4fD5DJN31xwQCAdO8FwgEGlMO+wJKJGKxGImJid2+HkII1q5di6+//honTpzAsGHD+nz82x2qOwrVMtWoUaPA4/Hw/fffY8yYMXd0+UpfWMjEDNA1y6IJlkwW40MikahlshBCMH36dMyePRtjx47V++JOiSQmJqbLgGt3EAqFDLG0trbCzc2NIZbeqNCoEKCjowNJSUk6ieSzzz7Dhg0bcPz4ccTHx/f4eHcaKHm0tbVBLBbD2tpabeJ/zJgxuHnzJn744QeMHTv2ricUC5mYCRMnTsSgQYOwefNmve5vyWQxLWQyGc6cOcNksnR0dGD69OlISUnBhAkTtF7cacBWT4mkM0QiEUMszc3NcHFxYWZZ9PELUygUuHLlCtrb23VOeRNC8MUXX2DdunU4cuQIhg8f3uvzvlNAiaS4uBhz586FUChEbW0tVq9ejSlTpjChZQ888ADKysqwadMmTJo06Y6bHekJLGRiBuiaZdEESyaL+SCXy3H+/Hkmk6W5uRlTpkxBamoqJk6cyAxBHj9+HAqFAnFxcQYN2JJIJAyxNDY2donP7QxCCK5cuQKBQICkpKRud1SEEHzzzTd47733cPDgQZNnkPRnVFRU4J577sHMmTOxcOFCHDhwAB999BEWLlyIefPmITIyEgBwzz33oLKyEleuXLkjrOR7DWKB0fHKK6+Q9PR0cuPGDXLx4kUyY8YM4uzsTMrLywkhhLz++uvk6aefZu6/fv16snfvXnL9+nVy5coV8vrrrxMAJC0trdvjVFVVkSeffJJ4eHgQe3t7EhcXRzIyMrp9THp6OklMTCS2trYkNDSUbN68ue8v+A6GXC4nFy5cIK+88goJDw8njo6OZPbs2WTJkiXE1taW/Pnnn6S9vd1o/zU3N5Pi4mJy9uxZsn//fnL06FGSm5tL6urqiEAgIAKBgFy6dIkcPXqUNDU1dftcAoGAfPHFF8TZ2ZmcOnXK3G9tv4JIJCJPP/00eeaZZ5jbJk6cSMLCwoibmxtZvHgxKSwsZP5WUFBgjtPsV7h792QmRFVVFR5//HG1WZaLFy8yIUiGyGRpamrCvffeiwceeAAHDx6Ej48PSktLu5UWW7JZeg42m41Ro0Zh1KhR+Oijj5CdnY33338fX375JSIiIvD111+jrq4O06ZNg4uLi8FVPNbW1ggICEBAQABjLsrlcvH333/D1tYWHA4HUqkUI0aM0Lkj2b59O1atWoX9+/dbhB2dIJfLMWHCBCaEb+rUqZBKpSgtLcUnn3yC1atXQy6XY+HChRg2bBiGDh16V8h/u4OlzHWH4PXXX8e5c+dw5swZvR9jyWbpO3777Tc8//zz+O233xAcHMxksly/fh3jx49HSkqK0TJZVCGTyZCbm4uWlhYAYPzCqK1LZ6+y//3vf3j55ZeRlpaGyZMnG+28bhdoyiapqqpCYGAg0tLSsG7dOuzYsQMRERH43//+h2XLloHFYuHYsWOIiooy01n3L1jmTO4Q7N+/H8nJyfi///s/+Pj4ICEhAVu3bu32MZZslr5BoVBg69at2L17N6ZPn45hw4bhP//5D/Ly8pCbm4t7770XW7ZsQXh4OGbNmoVt27aBx+P12b6+MwghKC4uRkdHB+655x6MGzcOUVFRDMGcPn0aV65cwf79+yGRSLB3714sXrwYO3fuNBmRrFmzBsOHD4ezszN8fHyQmpqKoqKibh9jKlshmUwGNpsNsViMiooKlJSUAAAGDBgAFouFxsZGKBQKRvjQ3NyMN954AxcvXrQQiSrMWGKzwICwtbUltra2ZNWqVSQrK4t8/fXXxM7Ojvz4449aHxMREUE++OADtdvOnTtHAJCamhpjn/IdAYVCofPvxcXFZO3atWTEiBHEysqK3HfffeTTTz8lxcXFRCAQ9KmHIhAISGZmJjl06BCpr6/v8ve2tjZSWVlJ9u3bR9zd3YmTkxNxcXEhb7zxBhGJRCZ6lwiZPHky+f7778mVK1dITk4OmT59OgkODiYCgUDrY06ePEkAkKKiIlJbW8v8J5PJDHZe9PNra2sjI0eOJAMHDiQDBw4kkyZNIsXFxYQQQnbv3k2cnZ3JokWLyJIlS4iNjQ3Zs2ePwc7hToGFTO4QWFtbk9GjR6vd9vLLL5NRo0ZpfUxERAT58MMP1W47e/YsAUBqa2uNcp53MxQKBSkvLyeffvopGTNmDLGysiKjR48ma9euJdeuXesxsQgEApKVlaWVSDr/t2vXLjJo0CDmQu7i4kKeeOIJUlZWZvL3gsfjEQDdNv4pmTQ1NRnlHCgpKRQKMmvWLJKamkoOHjxI9u/fT+Lj40lYWBjJyckhhBCyZs0acs8995CxY8eSX375xSjnc7vDUua6Q+Dv798l+3rIkCFqjf3OsGSzmBYsFgsDBw7E8uXLcfr0aZSXl+Pxxx/HoUOHMGzYMIwbNw7r16/HjRs3dJbCyD+lLS6Xi6SkJJ2zJ8ePH8ezzz6Ld999F3/88QfKy8tx/PhxBAcHw8nJyZAvUy/Q3g61u+kOxrIV4nA4UCgU+Oqrr+Dh4YF169ZhypQpmDlzJrKzsxEYGIgnnngCgLInefjwYRw4cIC5TddndNfBzGR220Mul+ssdZgCjz/+OBkzZozabUuXLu2yW1HFihUryJAhQ9RuW7BgQbe7GQsMD4VCQWpra8nmzZvJxIkTibW1NYmLiyPvvPMOycrK6rJjEQgEJCcnhxw8eJDw+XydO5KDBw8SJycn8t133/WL76pCoSAzZ87s8n3tjMLCQrJlyxaSmZlJzp8/TxYuXEhYLJZBZcxbt24lXl5exMPDg1RVVRFCCBEKhYQQQm7evEn8/f3Jjh07DHa8OxkWMukh5HI5OXHiBDl9+rS5T0UNf//9N7GysiIffPABKS4uJr/88gtxcHAg27dvZ+7TeZ7lxo0bxMHBgSxbtoxcvXqVbNu2jVhbW5Pdu3eb4yVYQJQX2vr6erJt2zYybdo0YmNjQ6Kjo8mqVavI33//Tdra2shLL71EVq9eTXg8nk4iOXLkCHF2diabN2/uF0RCCCEvvfQSGThwIKmsrOzxY2fMmEFmzpzZ62N3fg9KSkrIO++8Q2xtbcm///1vtfvV1taS8PBw8uuvv/b6eHcTLGTSAxQWFpL777+fxMXFkcGDBxNHR0fy+OOPkwsXLpj71AghhPzxxx8kJiaG2NrakqioKLJlyxa1v8+dO5eMHTtW7bb09HSSkJBAbGxsSEhIiM6hxZ4ORtK6d+f/rl271uvXebdAoVCQpqYm8tNPP5GUlBRib29PwsLCiKurK/nxxx9JW1tbt0Ry8uRJ4uLiQjZu3NhviGTx4sVkwIAB5MaNG716/Pvvv0+ioqJ69djOjXsqQGhsbCTvvvsuCQkJIcuWLSOEENLR0UEuXbpEXFxcyIEDB3p1vLsNFjLpAZ566ikycuRIcuLECSIQCMj58+fJU089RR555BFG+XEno7GxkQwcOJA8++yz5NKlS6SsrIwcO3aMlJSUaH2MKRQ5dwveffdd4uzsTKZNm0acnJxISEgIWbJkCUlPT+9CLGfOnCFubm7kk08+6RdEolAoyKJFi0hAQAC5fv16r5/n4YcfJg888ECPH6f6fVuwYAFJSUkhgwcPJps2bSLl5eWkpaWFvPfee8TJyYkMGTKETJ8+ndxzzz1kxYoVvT7Xuw0WMukBEhISyPPPP692W1lZGfntt98YRQz94crlciKXy019ikbFypUrdda5O8PYipy7BevXryceHh6MukggEJDdu3eTJ554gri6DQDrWAAAFKZJREFUupKgoCCyaNEicvToUXLu3Dni7u5OPvzww35BJIQQsnDhQuLq6krS09PVFhW0P0GI4WyFusPEiRNJcnIy2bFjB/noo4+IlZUVWbVqFSGEED6fTz788EMSGRlJJk+eTEpLSwkhyt/0nfZbNgYsZNIDfPvtt8TR0ZGsX7+etLa26v24/vKD7iuGDBlCli5dSh555BHi7e1N4uPju5TSOoOSSUhICPHz8yPjx48nJ06cMNEZ3zk4f/48ycrK0vg3oVBIfv/9dzJ37lzi6upK2Gw2ee211/rV905TqRMA+f7775n7dC7Drlu3joSHhxM7Ozvi7u5OxowZ06uSE30fduzYQWJjY5nf7ieffEK8vb1JdnY2c9/Gxkaydu1aEh8fT5YuXdrlOSzQDguZ9BCfffYZiYqKIlOmTCGHDh3q8verV6+Sd999lyxevFir6uR2XeX0ZjDSFIocC25BLBb3qx6JqaH6uiUSidrftm7dSlJTUwkhhLz66qvE19eX6Xdev36dHD58mBBCSH19Pfnkk09IbGwsefzxx0105rc/LGTSC5w+fZo88cQTJCQkhPz111/M7V988QUJCQkhDz30EHniiSeIv78/mTdvnlbykMvlt1XvoDeDkZrQV0WOBRZogiqR7N27l2zbtk2tl7lx40YSHR1NPvjgA+Lr60vS09OZv3399dfkiSeeIHV1dYQQQpqamsh7771HlixZYrLzv91hIRM9UFtbS27evNnl9nvvvZc89NBDRKFQkFOnTpGBAweSL7/8kvn79evXydChQ8kPP/xACCGkrq6OfPvttyQ/P99k525IBAcHd+kZbdq0iQQEBPToefqiyLHAAl345JNPyMCBA8krr7xCioqKmNvr6+vJ6NGjCYvFIn/++Sdze0lJCRk8eDB577331J6nvb3dZOd8J8BiQa8HaD44tRkHAKlUisTERBw6dAgsFgtHjx5FU1MTVqxYge3bt2P27NlYsWIFBgwYwEyh5+fn46OPPkJISAiSk5ORkZGBV199FRMnTuxiX63JxdTcuPfee7uY812/fp2x0tcX2dnZ8Pf3N+SpWWABAGDz5s1YvXo1du3ahZEjR8Ld3Z35m7u7O5YtW4Y1a9Zg1apVqKmpQV1dHX7++WcMHz4cb7/9NoBbvz0aemaBfuhfV6t+CpqfPWXKFDz33HPYu3cvFixYgLS0NDz66KMAgIKCAtx///2oqqpCSkoK0tLS4OjoiKNHj6KjowMAUFRUhMrKSggEAkRFRSEsLAzPPvsszpw508WenBKJXC6HQqEw7QvWgmXLluHixYv48MMPUVJSgh07dmDLli1YtGgRc59Vq1bhmWeeYf69YcMG7Nu3D8XFxSgoKMCqVauQlpaGxYsXaz1OSEiIRrdY1eN0xqlTp5CUlAQ7OzuEhYXh66+/NsyLtuC2QXFxMbZs2YLvv/8eU6ZMYYhEKBTi/PnzKCgowP/93/9h27ZtGDJkCD7++GNkZWXh6aefxi+//AJA+Xvrb4u42wbm3hrdTjh+/DiZM2cOiY2NJbNmzSKbNm1iaqyTJk1SkzUSotw+//zzz+TSpUukvb2dPProo2To0KHM30UiERk3bhx55plniFQqJYQoLRy+/PJLsmvXLiIWi9Wej/ZXfv75ZxIVFcUc25To6WBkbxQ5PB5PTT569OhRAoCcPHlS4/3pJP+SJUvI1atXydatWy2T/Hchrl69SiIjI9WGiDdu3EhmzZpFWCwW8fHxIU888QTzt5aWFrXH3079y/4IC5noAU0N9M5fxJ9//pmEhYWRY8eOaXyOc+fOkdjYWLJ27VpCyC2lyZo1a5j+gVgsJn/99Rd54YUXSExMDHF1dSWLFy9mtPj0yz59+nQyceJE0tzcbJgX2M+xZMkSEh4erlWhtGLFii49mPnz51s8xu4yZGdnEycnJ/LJJ5+QjIwMkpqaSpKSkshLL71Ejh07Rr777jvi5+fHLIDoAo4Qi/TXELCQSQ/QWX2l+gWUyWRk0aJFxM/Pj6SmppIvvviCrF27lhl82rRpE7GxsWFsRKiVw6xZs8js2bOZ51H9gh88eJDExMSQnTt3qp2HjY0N2bp1613xAxCLxcTT07NL7ooq7rvvPjVfJUII2bNnD7GysuoiD7XgzsZnn31GWCwWCQwMJAkJCeT48ePMwGxFRQUJDQ0l33zzjXlP8g6FpTjYA7DZbHA4HObfqn0ODoeDL7/8En/++ScGDBiAffv2QSgUwtvbG83Nzbh8+TIAoL6+HgBga2uLtrY2HD16FFOmTAGgTHDbv38/vvrqK2RlZWHKlCmIjY3F77//zhzn999/h7W1NUaOHKk1BvbAgQMoLi42+Os3B/bt24fm5mY8++yzWu9TV1cHX19ftdt8fX2ZjHQL7h4sW7YMhYWFOHToELKysjB+/Hi4ubkBUP5+vby8LPEKRoJFzWUgkH/UWElJSUhKSgIAtLe3w9HREWfPnsWNGzcQHx+PnTt3IjY2FlevXsX69evh7u6OF154AU1NTZg2bRoEAgEGDBiA1atXw83NDXw+Hy+99BJEIhHs7Oywbds23HfffQgODtZ4/Orqanz44YcIDAzE//73P3O8FQbFtm3bMHXqVAQEBHR7v87ESv7JmjBm7roF/RORkZFdbqurq8MTTzwBDw8PPPzww2Y4qzsflp2JgUAvWgqFAnK5HADg6OgIQCkJ5nK5WLNmDQQCATw9PfHII4+goaEBmzdvBpvNxo8//oi6ujp8+eWX2LNnD8rKyrB48WJwOBxERkbCzs4OAHD06FHMnDkTzs7OasenF8/MzEyw2Wxmt0PP5XZERUUFjh07hhdeeKHb+1lCvizQBi6Xi/Xr1+OZZ54Bi8XCoUOHANzev4v+CsvOxMDoLCtsbGzE5cuXMWDAAIwfPx7jx4/HO++8g4KCAkyYMIHRskskEnA4HAQEBDCpeQ0NDXB1dUVUVBQAZfmKxWLhnnvu6XIc+u+LFy+CxWLhvvvu03g+FHK5HCwWq1/LIL///nv4+Phg+vTp3d5v9OjR+OOPP9RuO3LkCJKTk2FtbW3MU7Sgn6OoqAgHDx5ETEwMPvvsMwCATCaDlZXl0mdwmLdlc+fj+vXr5P777ycrV64khJAucl+KqqoqEhsbSxISEsjHH39MHnnkEWJnZ0fmzZtHuFwuIYSQ2bNnkwkTJnRx4KWN+Lq6OjJr1iwyd+5co70eU0Eul5Pg4GDmfVOFJeTLuPjwww9JcnIycXJyIt7e3iQlJYUUFhbqfFx6ejpJTEwktra2JDQ0VGc2jqnA5/OZ/2+R/xoP/XdZeocgIiICp06dwptvvgkAsLa21pgdHRgYiOPHj+ORRx7BjRs3MGnSJMhkMsTHx8PHxwcAcOLECcycORMuLi5qj6XPd/nyZfB4PIwePRoAugw7tre348iRI3j++efx2muvoaCgoMt5EKXCr+8vvI84duwYbt68ieeee67L32pra9Wy7UNDQ/HXX38hPT0d8fHx+O9//4uNGzd2Wxvv6WBkenq6xvsXFhb2/cX2M5w6dQqLFi3CxYsXcfToUchkMkyaNAnt7e1aH1NWVoZp06bhvvvuQ3Z2Nt544w38+9//RlpamgnPXDO8vLwAKL/bqgIaCwwMs1KZBQw6y3zFYjHZu3cvycvLI4QohwVZLFa3qYZvvfUWGTNmDLl69SrznHRGpry8nCxcuJAMHDiQLFu2jEyfPp0EBweTzz//XOvzyWSy29bhWBd6Ohh5N4d88Xg8AqBbp2fLrI8FlsJhPwGLxQIhBAqFAhwOBzY2NkhNTWX+7uLiguXLlyM0NFTj4+vr63HlyhWEhIRgyJAhzHNSbNmyBWfOnMFvv/2GkSNHAgC+/fZbbN68GRMmTEB0dDQuXryI7OxsPProo/D09LyjV3He3t5q/167di3Cw8MxduzYbh/n4+PDSE3vFrS0tAAAPDw8tN7nwoULmDRpktptkydPxrZt2yCVSi29q7sAljJXPwKLxVK7gBOVctP999+PTz75pMsPmpayMjIywOfzMWbMGLXbWSwW6uvrcfLkSXC5XEyaNAkTJ07ETz/9hCeffJIxuwOUDf7//Oc/eO+99/Dwww/jqaeewrVr1zSea3/xCzMEJBIJtm/fjueee06nlDghIQH+/v6YMGECTp48aaIzNB8IIVi+fDnGjBmDmJgYrfezzPpYYCGTfgzVC5s2KSNVY2VmZuL8+fM4cOAAzp8/j+bmZuY+N27cgEgkwurVq3Hp0iUkJiZi7dq1cHNzA5fLZS4CmZmZkEgkcHZ2xtNPP43GxkYsWLAAPB4PgDq50eNqI5XbiWz0GYz09/fHli1bkJaWhj179mDw4MGYMGECTp8+bboTNQMWL16MvLw8/Prrrzrva5n1ucth1iKbBQaDQqEgaWlpZPz48cTKyor4+PiQp59+mly8eJG0tLSQ4OBgsm3bNub+crmcZGVlkV9//ZWIRCJy+fJlEh0dTd555x3mPrm5uYTFYpFz584xt7W2tpLt27czqXSacLvlQEyaNInMmDGjx4+700O+Fi9eTAYMGEBu3Lih874WSxsLLDuTOwQsFgsPPfQQjh8/DqlUii+//BLNzc34/fff4eLigkmTJmHHjh0oLy8HoNxZJCQkYM6cObC1tcVff/0FT09PTJ48mXnOlpYWDBs2jClT5Obm4p577sHbb7+NuXPnIi4uDnV1dUy5h+5GYmNj8cUXX6gNcJaXl6OiosKE74h+0HcwUhNGjRp1x9jWqIIQgsWLF2PPnj04ceKE1j6dKkaPHo2jR4+q3WaZ9bnLYG42s8BwkMlkWtVF5eXlZNy4cSQgIIA8//zz5NNPPyVvvvkm8/fJkyeTefPmqbkh//e//yVjxowheXl5pKGhgdx7773kwQcfJAUFBYQQQj7//HMmuS4/P58IBAKyZs0awmazSUlJidrx33jjDRIdHU1yc3ON8Mp7j3fffZf4+fmpGWzqi4cffpg88MADRjgr82LhwoXE1dWVpKenqynXqHs1IZZZHwu6wkImdyhUZb2qsuPDhw+TefPmkZkzZ5Jvv/2WEKK0x09OTiYbN25k7ieRSEhKSgp54okniFQqJb/++isJCgoiWVlZzH0KCwuJt7c3efjhhwmfzyd5eXmExWIROzs7MnLkSPL9998TQghpa2sjTz75JHnkkUeYi3Z/cDzuyWDk+vXryd69e8n169fJlStXyOuvv04AkLS0NFOeskkAQON/9PMkpGtuDSHKocWEhARiY2NDQkJC+s3QogWmgUUafIeis7sxjSKdNGkSI+GUSqUAgL///hsikQjR0dHMY6g6bOzYsbCyskJWVha8vb2RkJDAmEoOGjQIIpEII0eOhIuLC7y8vBATE4OkpCQEBATgs88+g6enJ4KCglBZWYlp06YxNhZUCm3O5mxPBiMlEgleffVVVFdXw97eHtHR0Thw4ACmTZum8zgymQyrV6/GL7/8grq6Ovj7++PZZ5/FW2+91a2dzalTp7B8+XIUFBQgICAAK1aswIIFC3r3YnsAosfQ6g8//NDltrFjxyIrK8sIZ2TBbQEzk5kFJkbnTBaK8vJy0tHRwfx71apVJCkpiaSnpxNCCHnooYfIww8/TAi5Fex1/PhxYm1tzSTbXbt2jbDZbLWkO0KU5bDhw4eTDz74gLz11ltk48aNaiWTOx3vv/8+8fT0JH/++ScpKysju3btIk5OTmTDhg1aH2NJj7TgdoNlZ3KXQdtKeODAgWr/jo6OhlAoZOz0g4KCcPLkSdTV1cHPzw8tLS3YuHEjkpOTERYWBgDYs2cPgoKCEB0dzew6RCIRLly4gKtXryI6Ohrh4eHYvHkzjh07hh07djDOyncyLly4gJSUFMawMiQkBL/++isyMjK0Pubrr79GcHAwNmzYAAAYMmQIMjIy8Mknn1gs1C3onzA3m1lwe6C6upoMHjyYjB07lnz11VdkypQphMVikXXr1jHmlQkJCWTx4sWEkFtRxxcuXCCJiYlk4cKFzHMdO3aMODs7k0uXLpn+hZgBa9asIQMHDiRFRUWEEEJycnKIj48P2bFjh9bHWKS2FtxusEiDLdAIuVzO1M6bm5sREBCAgwcPIj4+HmVlZZgwYQKcnZ0xevRo2NjYoK2tDTk5OUhJSQFwq+5+/vx52NraYs6cOcxzczgcDB48uF9KhY2BlStX4vHHH0dUVBSsra2RkJCApUuX4vHHH9f6GMtEuQW3GyxlLgs0QrWBf/bsWTQ0NGDu3LnYsGEDmpub8cwzzyA5ORlxcXEAlK6xgYGBzOQ9h8OBWCxGdnY2vL29GT8wQBkWRgjRa37hTsBvv/2G7du3Y8eOHYiOjkZOTg6WLl2KgIAAzJ07V+vjLBPlFtxOsJCJBTphY2ODDz74AO+++y7Gjx+PzMxMVFVVYf/+/YwdflhYGObMmYM5c+Zg7Nix+Omnn1BbW4uamhqMGzcOtra2AACBQICcnBx4enoiOTnZnC/LZHjttdfw+uuvM7uzYcOGoaKiAmvWrNFKJpb0SAtuN1jKXBboxKRJk1BUVISPPvoI1tbWeOyxx3Du3Dnce++9zH2cnJzw8ccfo6KiAqmpqbC1tUVWVhZu3LiBUaNGMfe7evUqSktL1XYqdzqEQmEX4QOHw+nWv8wyUW7BbQfztmwsuJ2hmnWimp1C0d7eTg4fPkxEIhFz26effkri4uLU/L7udMydO5cEBgYy0uA9e/YQLy8vsmLFCuY+lolyC253WMpcFvQaqqttmjxIVAYRHRwc1DIu5HI52Gw2PDw81HYrdzq++OILvP3223jppZfA4/EQEBCA+fPn45133mHuoy09ctmyZfjqq68QEBCgMz3SAgvMCRYh/SCj1YK7CjKZjJmEt8ACC+4MWMjEApOB/JMv352FiAUWWHB7wkImFlhggQUW9BmWJaIFFlhggQV9hoVMLLDAAgss6DMsZGKBBRZYYEGfYSETCyywwAIL+gwLmVhggQUWWNBnWMjEAgsssMCCPsNCJhZYYIEFFvQZFjKxwAILLLCgz7CQiQUWWGCBBX3G/wMWUtnGV5EzZgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAGQCAYAAAB8qh0GAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOy9eZwcd3nn/67qu2e6Z6bn1Ny6NZKsW7ZG8slhsIEAIZAlkBA2wPILsARvloRf2E2ym2MTyMbJJhAI/tlrHAgJjgM+8AG2sGXJliXNfc9o7nv6mKPv7qrfH60qdc/09HTPdI9Goj6vl1+gnu6q6uqq76ee5/k8n0eQZVlGgwYNGjRoyAHEG30AGjRo0KDh1oVGMho0aNCgIWfQSEaDBg0aNOQMGslo0KBBg4acQSMZDRo0aNCQM2gko0GDBg0acgaNZDRo0KBBQ86gkYwGDRo0aMgZNJLRoEGDBg05g0YyGjRo0KAhZ9BIRoMGDRo05AwayWjQoEGDhpxBIxkNGjRo0JAzaCSjQYMGDRpyBo1kNGjQoEFDzqCRjAYNGjRoyBk0ktGgQYMGDTmDRjIaNGjQoCFn0EhGgwYNGjTkDBrJaNCgQYOGnEEjGQ0aNGjQkDNoJKNBgwYNGnIGjWQ0aNCgQUPOoJGMBg0aNGjIGTSS0aBBgwYNOYNGMho0aNCgIWfQSEaDBg0aNOQMGslo0KBBg4acQSMZDRo0aNCQM2gko0GDBg0acgaNZDRo0KBBQ86gkYwGDRo0aMgZNJLRoEGDBg05g0YyGjRo0KAhZ9BIRoMGDRo05AwayWjQoEGDhpxBf6MPQMMvHiRJIhwOI4oiOp0OUdSedTRouFWhkYyGTYMsy0SjUcLhMH6/H0EQEAQBnU6HwWBAp9NppKNBwy0GQZZl+UYfhIZbH7IsEw6HiUaj6v8XBAFZlpEkCeUyFEURvV6PXq/XSEeDhlsAGsloyDmi0Sjj4+MEAgFqa2uRZZlQKLSCPGRZTiAdJdLRSEeDhpsXWrpMQ84gyzKRSIRIJMLi4iJer5e6ujqVQJT/VaCQikIiCuGEw2HC4bD6nnjS0ev1CdvQoEHD1oJGMhpyAoUcJEkCSBq1rEUOSr0m/jPKdkOh0IpIRyEejXQ0aNg60EhGQ1YRTwSSJCGKokoGG83MrkU6fr8fv99PRUVFgpBAIx0NGm4cNJLRkDXEF/cBlWCArJDMciwnnbm5OSYmJiguLlYjnWRCAo10NGjYPGgkoyErUCKKaDSaQC7xyLXGJD59puxLkiRCoRDBYFAjHQ0abgA0ktGwISi9L5FIJCE9thyiKOacZJTjAdRjUCIdjXQ0aLgx0EhGw7qRKj222vsBXC4Xg4OD2Gw2ioqKKCgoyIosOdW+k5GO8l8wGCQUCqnfQSMdDRqyB41kNKwLSud+quglHoIgIEkSvb29DA8PU11djc/nY2JigkgkQkFBAUVFRRQVFWGz2dZNOulGS0pqDWLEs5x04iMdg8Ggkk4631WDBg3XoZGMhowQ3/sCa0cvCsLhMAsLC4RCIU6dOoXRaFT/5vP5cLvduN1uRkZGkGWZwsJClXTy8/NzvrCnIp1AIABAIBAgHA5TWlqqkY4GDWlCIxkNaWN570v8wpwK09PTDAwMoNfraWxsRBRFNT0lCAJ5eXnk5eVRXV2NLMssLS2ppDM4OIggCBQVFanEk5eXl3S/2Vzsk5HO/Pw8s7Oz2Gw29T1KQ6hGOho0JIdGMhrWRHw/Srzdy1qIRqP09PQwMTFBVVUVi4uL6oK9GgRBwGazYbPZqK2tRZIklXScTicDAwPodDo1yikqKsJisajHkytxgZI6EwQBg8GQYIGjRDqiKK6o6Wiko+EXHRrJaEiJ5cX9dAnG6/XS3NyMIAicPn2a+fl5FhYWMt6/KIrY7Xbsdjt1dXVIksTCwgJut5vp6Wl6e3sxGo0UFRWh0+nUKCsXiCew1dJrkiQRDAYJBAIa6WjQgEYyGlIgnd6XZBgfH6ezs5Oamhr27NmDKIosLCxkJcoQRZHCwkIKCwvZvn070WiU+fl53G43s7Oz+Hw+zp8/nxDpmEymDe93LSwnX4V0otEo0Wh0hZAg3ndNIx0NtzI0ktGwAun2vixHJBKhs7OT2dlZDh8+TFlZmfq3XHT8QyyKcDgcOBwO7HY7V69eZdeuXbjdbkZHR+ns7MRqtaqEU1hYmCA6yBWSmX3Gn9f4WTrLfdc00tFwK0EjGQ0JyLT3RcHCwgItLS0YjUbOnDmD2WxO+HuuSGb5PgRBoLi4mOLiYiCmavN4PLjdboaGhlhaWiIvLy+BdAwGQ06PK/7YlpNOJBJJmK0TCARwOBzaWAMNtww0ktGgQpIk3G43nZ2dnDx5Mi1ykWWZ0dFRenp62L59Ozt37lz1czdidJHBYKC0tJTS0lIAQqGQSjoDAwP4fD61KVRpDNXrc39bJCMdn8/HW2+9xV133aVNDdVwy0AjGQ0JvS+RSISFhYW0e1/a29vxeDwcP34ch8Ox6ns3KwW0FpEZjUbKysrUVF4wGFTl0j09PQSDQex2uyqXLigoSDDhzBVWU68pkQ5oU0M13JzQSOYXHJIkEYlE1PTYWhJjBW63m5aWFmw2G2fOnFmzzqF0/OcS6yEyk8lERUUFFRUVAPj9ftxuNx6Ph66uLkKhkOpGEIlEchqNJVOvLU+vrTZLRyMdDVsVGsn8gmK13hdRFFOSgSzLDA4O0t/fz+7du6mvr09rcV/+ns2o0awHFosFi8VCZWUlsiyrpKP06USjUZqbmyksLMThcJCfn5/VhX21c6lNDdVws0IjmV9ALLeGiVc0pVr8g8Egra2t+Hw+7rjjDgoKCtLe52aRSjb3IQgCVqsVq9VKVVUVY2NjTE9PU1xcjMfjyboFTibHrk0N1XCzQCOZXzDE977EPxkriH9Sjl+Q5ubmaGtro6ioiNOnT2esyNosdVmut6/X66mpqaGmpialBY7yn9Vqzei41vsd0iEdURRXCAk00tGQa2gk8wuCdHtflNckSVI76Pv7+xkeHmbfvn1UV1eva2HaqumxjSCZBc7i4qLaGNrf349er0+IdOItcJYj21FYMtLp6ekBYMeOHdpYAw2bAo1kfgGQSe9LfCTj9/tpaWkhEolw6tQp1RhyPbgZ02WZQhRFCgoKKCgooL6+PqUFjvJfsn6iXCCedJT/rw1w07AZ0EjmFoeykGQy9wVgZmaGrq4uysvLaWho2LCM91ZIl2WKVBY44+PjdHd3Yzab1UhnM5pCZVlWyUObGqphM6CRzC0KJT2mqMfS7dxXFpyOjg4OHjzItm3bsnI8t0Iks9Ftx1vgQMyGRyGd0dFRFhcXAejp6cmZBY4kSUmVfsrxgTY1VEN2oZHMLYj1WsMozskAJ06coKioKOvHpeE69Hp9ggWOy+Wira0NQRAYHBzE6/WSn5+fYIGzUTeC5YKOZEg1wC3Z1FBFLq05TGtIBo1kbjFkOhZZwcTEBB0dHdTU1LC4uLiiVrBRbMbic7MvcEqEsGfPHiBmgaM0hvb396+wwCksLFxXGjPT85SKdJRZOvGko4010BAPjWRuEax3LHIy5+Th4eGsd+cnS5el81SdKW7maGn5sRuNRsrLyykvLwdWt8BRSMdut69JOtk452uRztjYGHl5eeqYao10frGhkcwtgOVjkdPtQF/NOVkUxawv1r+Ihf/1INV3WM0Cx+12MzExQSQSWUE6y6+FXBD7ctKZn5/HYDAkTA1NNtZAI51fDGgkcxMjvuEuk/RYvHNyfX09u3btSvhcLgjhVuyTyTYyPT+pLHDGxsaIRqMJPTo2my0nJLMcitBEEQdoU0N/saGRzE2K9Rb3452Tjx07phad45ELM0vl2HJNNDczkW2EAJZb4MiyjNfrVUlneHgYuB7lFhYWbsgCJxUUkok/tvj9pJoaqpHOrQeNZG5CSJLE1NQUsixTXFyc9o2YrnNyrtJlkJt0zfJ95BKbYV2Tre3k5+eTn5+fYIHT0dGBz+fjypUrah/Pei1wVkMymfTyY0vmMK1NDb01oZHMTYT4G3FychKDwUBJSUlanxscHGRgYIBdu3at6Zycq3SZcizaQpEcuYzCFAscs9lMWVkZFRUVSS1w4pVrqSxwUmF5JJPOsSUjnfipoRrp3LzQSOYmwfL0mGILshbinZNvv/32tJyTc5EuU6AspMrike35JzdzugxyHykpJJ/MAkdpDJ2cnKSnp2dNC5y19rFepEM6fr9fjcS0WTpbGxrJ3ARI1vsiiqJKOKvB6XTS2tqasXNyLiIZZQGQJImJiQm6u7sRBEFN1zgcDvLy8ja8ON3MuJGOCKIoqmQCrGqBE086q6Vb10qXZYpkpDM1NUUkElGPQZsaunWhkcwWRqreF1EU1WFVy7FR5+Rc1GQUdHR04Ha7OXz4MDqdDo/Ho9rkKwudw+FQHYszhRbJpEa6UUYyCxzltxoeHqajo4O8vLyEmo7yEJNpuixTKMev1+sTRlUrYw1AI52tBI1ktiiW974szz+vNsEyG87JuUiXeb1eIJa+O3PmjBqJ2e121SZ/YWEBl8ulpmtMJpNKOKmenG8VbFYks57FVq/XU1JSotYAw+FwwgNCe3u7aoETf93mCpIkqRY7ySKdtaaGaqSzedBIZothtbHIy5GMZKanp2lvb9+wc3K2I5nx8XE6OzsBOHLkCCaTaUUUFu9YDMmfnNfy8dqMKCDX2CqRzFowGAyUlpZSWloKXLfAcbvdRCIRWlpasNvtaqSzXguc1aCkjpNhtVk62qjqGwONZLYQlhf3U6ln4usmkiTR3d3NxMQEBw4c2LBzcrZqMtFolM7OTmZmZjhy5AiXL19O+7PLn5zjF7He3t4ESxWHw4Hdbgdu7nTZZkUyuVhM4y1wZmZmOHDggPqbdXd3EwqFMrbASYVM6j6pSEebGpp7aCSzRRA/FjmdJjQlkvF6vbS0tABw+vRprFbrho8lG+mypaUlmpub0ev1CZY1611Il/t4Kd3tLpeL8fFxJEkiPz9fnU6Zq0bDXONWiMZkWcZsNlNcXMy2bdtUT7PlFjgFBQUJbgSZpK+Uya3rQbqko401yA40krnBSHcs8nKIoojf7+f8+fPU1NSwZ8+erOWYN5oum5ycpL29ndraWnbv3q0eV3yEtNEbdrmlitfrZWpqivn5ebXRUFnAHA7HukQEm42bOZJJtQ9BEFb8Xj6fTyWd0dFRJElaQTqpjjObCrZ40onPDgQCAZqamjh8+DBGo1EjnXVCI5kbiPVaw0QiEcbHx/F6vRw9epSysrKsHtd602XRaJTu7m6mpqZUR+fVtpvtefb5+fls27aNsbEx7rrrLnXs8dTUFL29vZhMpgTl2lYVEdwsNZlUSKfjPy8vj7y8PKqrq1e1wIlXri2Xt6eqyWwE8UafSqZA8V/TpoauDxrJ3CBkOhZZweLiojpYLD8/P+sEA+tLl/l8PpqbmxEEgcbGxqRpu1ybZMbLu5ePPfZ4PLhcrgT5rUI42RgGlg3cCpGM8h0y7fhfboGjuBE4nU6uXr26wgJHSSvnEso9EC8KiI90tKmh6eHG31m/YIjvfcl0LHK8c7LNZuPq1as5OcZM02XT09O0tbVRVVXF3r17U978N6Iwr9PpEiZQxosI+vr6CAQCCUXpgoKCGyJv3UyH5FwhXnK/XgiCgN1ux263U1dXp9bZXC6XaoEjy7Jai1tvT9VaiEajK8Q38ZFOsqmh8aSjTQ2NQSOZTYQkSUQikaw4J8/OzuasFyHdiEOSJHp6ehgfH+fgwYPqnJNU28010jnu1UQESne7YpGvRDrxIoKbfaHYLBfsbBJZvAUOxBb/N998E5PJlNBTFR/pZGOyazrigngS0qaGJodGMpuAdHtfksHj8dDc3LzCOXm1ZsxsIJ10md/vp7m5GUmSaGxsJC8vL63tbka6LFMkExEoyrV4JwKILXC5ijhuRFE+28hGJLMWlEW6srISh8OhpkOVh4Suri4sFktaFjipsJ66j0Y6K6GRTI6x3BomXYJZyzk51ySTigxmZ2dpbW3NuOkz1ySTDSyvD8SnaqampvD7/Vy4cCFBubZVRQTJsBVrMutBPAEsT4euZoET38ibjo9fNuo+6ZBONBrF7/dTUlJyS5KORjI5RHzvSyaOw8FgkLa2Nrxe76rOybn0F1tt25Ik0dfXx8jICAcOHKCysjKj7W4GyeTC2FNJ1ej1etxuN1VVVar0trOzU13AHA7HhkQEWiST2X5Wu5+SWeAo6dCrV6/i9XrXdI9Q9pFNlwJITjqLi4t0d3dz++2335JTQzWSyQGU3perV6+Sn5+Pw+FI+wJJ1zk5F/5iqbYdCARoaWkhHA7T2NhIfn7+urabrT6Z1bafawiCkPDUHL+AKSICm82m1nNulIhgNWxGJLMZ3zeTVJbBYKCsrExVYgaDQTXSif/N4oUfOp1uUxRsyr2mCARWmxr6p3/6p7z73e/m7W9/e06PJxfQSCbLiO99mZ2dVReltZCpc/Jmpsvm5uZobW2lpKSE48ePr/tJ/WZIl2WK5QtYIBDA5XIldLbHjzNI5URwK0QymzWUbiN9MiaTKUH4Ee9G0NXVpVrgGAwGJEnKWU+OgviIKZnZpyzLvPzyyxw/fjxnx5BLaCSTRSzvfUl3sNh6nJNzSTJKukyWZfr7+xkaGqKhoYGqqqoNS1M3o/Cfy8L8WjCbzVRWVq4QEbjdboaGhhAEYcU4g2QS2VxhM9Jlm/Edsrkfs9nMtm3bVAscRW04OTmJz+fj1Vdf3ZAFzlqIRqOrpuUU0vF6vWmJa7YiNJLJApTwVlGPxQ8WW4sI1uucHE8E2b6pBUEgFApx6dIl/H7/ukcGJNtuLtNlWw2pRATT09P09vZiNBpVwlltPlC2sBnnfjPSZcr3yHa9BGLnxmq1YrVakSQJg8HArl271AeFkZERZFlOcJdeywJnLaQiGUC14VlPinorQCOZDSKVNUyq6ZUbdU7O5VN7MBhkamqKsrIyjh49mrVu+M1Kl21WyiZTxIsI4p0IFBHB4uIiOp2O3t5e9ak5m04Em0EymxHJbKa4QK/Xr7DAWVpawu124/F4GBwcRBCElBY4ayGd2o8WyfyCItlY5Hisli7LhnNy/DjjbD05KrLpyclJ7HY7hw8fzvqNvBX7ZG4Ulktvh4aGmJ2dRZZlBgYG8Pv9akHa4XBsWERwq0Qyyj2V6/0kW/wFQcBms2Gz2dRhewrpOJ1OBgYG0Ol0CT06y1OiyfazVlTm9Xqzkk24EdBIZh1Y3vuymrwwWbpsYmKCjo4Oqqur17RgSYV4kskGQqEQbW1tLC0tUV1dTSQSyUkaToGS6tNwHTqdDrPZzN69e4HrBWmXy0VHR4dqj6+k1zJN09xqkcxm9uKsBlEUV1jgKOas09PT9PX1odfrV5BOPNYimUgkQjAY1NJlvyhYPhY51UUoiqLqZRSJROjq6mJmZiapQ3GmyCbJKK4Cdrud06dPMzY2xvz8/Ia3uxybmS67GbH8uJcXpH0+n6pcixcRxI8zSLXA30qRTCauGRvZT6Z1n2TmrPPz83g8ngQLnHjSiUajKZtDl5aWADSSudURbw2TrnOyEskozslGozFhgNdGoNxkGyEZWZYZHh6mr6+P3bt3U1dXp243Fwv1VrWV2UpY7TsIwnV7/HgRgdvtZmZmhr6+PoxGY4JyzWQyJWxjs0jmZjf5VBCNRjdcE9PpdDgcDhwOBxB72Jyfn09o5tXpdFitVrVBdLmDhNfrBTSSuaWx3rkvoiiytLTEG2+8QX19Pbt27crqDbgRklFMN+fn5zl58iSFhYVZ2W4q3Ap9MpthyZIO4kUE9fX16hNz/OJltVpVwikqKrql0mVbreEzXej1+hXNvC0tLYiimNQCx2w24/V6sVgsG1bT/fmf/zn/9m//Rnd3NxaLhdOnT/MXf/EXano2Gc6ePct999234vWuri727duX1n41klkDmY5FVhAOh5mcnMTr9XL8+PG0GjIzxXqtZebn52lubiY/P5/Tp0+veHK6WSMZBTcrkW0kCoh/Yt65cyfhcFidoaOICBR1ksfjUbvas43NIIDNJJlcnKN4KCaZZWVlVFZWEgqFEtwIPvKRj1BWVkZRURHPP/88d91117oFAD//+c/53Oc+x8mTJ4lEIvzBH/wB999/v2qNlAo9PT3Y7Xb136WlpWnvVyOZVaD0vmQ6Fhmu1zj0ej0FBQU5IRjIvCEzfibNzp072b59+6qChZuRZG6FdFm2YDAYKC0tVReDQCDAzMwM/f39dHZ2qiICJb220V4PBZs1eXOz0mWbvR+j0ZjgIPH666/z2GOP8dhjj/Gf//N/Znh4mJMnT/K1r32NM2fOZLSf559/PuHfjz76KGVlZVy+fJm777475WfLysoSsh2ZQCOZJFhvekyRAPf397N7926MRiNjY2M5O85MSCYSidDe3o7b7eb48eNqjjgZtHTZjUEuF2iz2UxJSQkDAwOcOXMGn8+nKtdGRkYAEorRVqt1XceyWYX/WyWSgdTqsrq6Om6//XZ+8pOf0NHRwcjICK+88krGfXXJoIh7Uq0FCo4ePUogEGD//v189atfTZpCWw0aySzDWr0vq2G5c3JhYSFTU1M5s36B9ElmcXGRpqYmNQ+7vCC8HLkkA1mOzUpvbW3F6/UmFKrTsV9Pdx8aViJ+ltHyBsPlkycNBkOCcm2ta0bBZtVkNiNq3cxIJhWZ+Xw+NZ1VW1vLJz7xiQ3vU5ZlHnroIe68804OHjy46vu2bdvGt7/9bY4fP04wGOS73/0ub3/72zl79uya0Y8CjWSuId3el2RYzTk5l/5isDYZKCNqu7q6MhIe5DJd5vP5OH/+PHa7nR07djA/P8/g4CDt7e3qCOT1Nh7e7OmyG2VeKQjXxx0vFxEo14/Vak0YZ7DaA8GtFslsBZJZWlrKerf/5z//eVpbWzl37lzK9+3duzdBGNDY2Mjo6Chf//rXNZLJBMt7X9LV4K/lnJzKViYbSEVikUiEzs5O5ubmOHr0qDpbIx3kIl2mDGlyOp3s2bOHmpoawuGw6oQbDAZxuVxq42H8CGSHw5FR+iaXUdjNjHRJbLnsVhERuN1uBgYG8Pl8K8YZKIvkraYuu9HpMsi+pcwXvvAFfvzjH/Pqq69SXV2d8edPnTrFE088kfb7f6FJZiNjkf1+P62trYTD4VUNJNN1YV4vViOZpaUlmpubMRgMnD59OuO+nGyny6LRKJ2dnXi9Xqqrq9m+ffuK4zaZTAmNh16vF5fLpVp16PV6deG72aZRpoutasO/XESgPBAo1vjhcFgVEQQCgVuGZDYjXaasQWuRTDZ6ZGRZ5gtf+AJPPfUUZ8+eZfv27evaTlNTU0Y1oV9YklEWMsVpNROCmZmZoa2tbU3n5Fyny1LZ1tTV1bFr16513STZTJf5fD6amppUn650nsji3Ytra2uT9oAojWtK+kan09306TLIfcovG9tf/kCgiAgU/y4l9axEOusVEayGWymSUeyV0q3JbASf+9zn+N73vsePfvQjbDYbU1NTABQUFKhWN1/5ylcYHx/n8ccfB+Dhhx+mvr6eAwcOEAqFeOKJJ3jyySd58skn097vLyTJKNFLV1cXFouF3bt3p/25np4exsfH2b9//5rjhzejJqNsPxqN0tXVxfT0NEeOHMlIx55quxvBzMwMra2tVFVVsXfvXlpbWxPIK92FZ3kPSCgUUpVR3d3d6pO0IrHM5TnPJTZjNHUu/OjiRQR9fX0Eg0FsNpsqIlCi0PgGw41gK07fXC/iFayrIVs1mW9+85sA3HvvvQmvP/roo/zmb/4mAJOTk6raEGKehr/7u7/L+Pg4FouFAwcO8Oyzz/Lggw+mvd9fKJJZ3vuSSTor3jm5sbExrR99M2oySkTW3NyMKIqcPn16hQHfere7XsQPOzt48KAaWmcrDWc0GtXJhsqQKSW1BnDx4kWVlJIZEm5l3Ozd+BCLdOrq6qirqyMajbKwsIDL5WJ8fFztNo+XS2eqKtzMNNZmkcxa6bJ0ZMZrIZ1777HHHkv495e//GW+/OUvb2i/vzAkk6z3Ra/Xp0UC63VO1ul0ORssBrHv4PF46OnpUaOFbNwUGyGDUChES0sLgUBgRa0qF9Lo+CFTVVVVvPLKKzQ0NLC0tKQaEprN5gTSyeaMlmziZoxklmN5iine9h5ighQltaaoCpePM1grRbWZ4wRynS5Lx0nkZh5YBr8gJLN8LHL8ULFU0wg36pwc75Sc7YtVmWMRCAQ4fPiwqtLKBtabLlOcDgoKCmhsbFyxmG9WM6bNZqO0tJTt27cnLGqKvUq8VNput2e0YN0MNZPVsBXMK/V6/QoRgZL6VEQEdrs9YZzB8u1tlTRWtvaTziwZjWS2KFYbi6xAp9OtGslkwzk5VyTj8/loaWkhHA5TU1OTVYKBzNNl8XY1u3btor6+ftV+jM22lVm+qAUCAVUqPT4+jiRJCQ2h2S5SZ4LNIIGt1ihpMpmoqKigoqIiIfUZP+o4vilUGYt8Kw1GS4dkbtapmHALk4wkSUQikZTWMMlIJn7BrK+vZ+fOnRseLLbWvIhMoCjbKioqyMvLy0nqJxMyiEajdHR0MDc3t6ZdDWxOr0mqfZjNZiorK6msrFRH6S7vdI9PrW22VPpWj2RSIT71Ge9EED91Uq/Xq9b4gUAgK2MzkiFTx4+N7OdWnooJtyDJZNL7spxkFPt7j8fDsWPHNmxsqVyk2VA7SZJEX18fIyMjajG9o6MjZ5356RxzvDw5nX6creZdFj9KVylSK02Hiu16fn6+Sjq5Vq3dKjWZbO0j3olAmTo5Pz9Pb28vS0tLXLhwQa23rVdEsBq2iqUMaJHMlsJya5i1el/i1V8ej4eWlhby8vI4c+ZM1p5gsyFjDgQCNDc3E4lEaGxsVPOzuZJIp0MGy+XJ6dyQm5GG2sg+lF4e5eEiFAqpqbWuri5CoRAGg4GRkRGKiorIz8/P6YjqbGOrRzJrQRRFioqKyMvLw2azUVVVpY4zWC4iKCoqUvun1oPN7PZPdb4U9ahWk9kCiJ/7IghCWhe6EslcvXqVgYGBlPWE9WKjRDA7O0traytlZWXs378/4cLPlURaOXfJFiVZlunr62N4eDhBnpwOcuXuvBzZigiMRmNCvaCvr09tCh0cHEQUxQQXgngTyXA0jEGX2VP1rRDJbKbVv16vp6SkRLVMUkQEbreb7u5uQqHQinEG6RKgFslkDzc9yWxk7oskSfh8PkZHR1dMh8wW1ksE8b0mDQ0NST2G1lLHrRfK+Vu+YMTLk+Mjqky2u5XSZZlAEASMRiNWq5UDBw4gSVLS/g+Hw4FH7+Gy+zIfPfBRCs2FGe8nV7jZ0mWp9pGMAJKJCBTl2ujoaIKIQImIVjvWrWKOCVpN5oZivXNfIOac3NnZiSzLCc7J2cZ6/MuCwSAtLS0Eg8FVfdEgtxMsIfFGW0uenO52cx3JbJYyTBRFCgsLKSwsZMeOHaqJ5Jxzjqdbn6bd047OpeOdu96Z9lP0zRjJSLLEgGeAHQU70Im6LePCvLx/Kl7kES8iiFeuxdcUt4o5ZigUIhwOa+myG4HVel/S+dzAwABDQ0PU1dUxMjKSM4KBzNNlTqeTlpYWiouLOXbsWMrFPFc1mfh0Wbry5HRwK49fVkwk3To3ockQO/J2MCgNMumeZGhkiPbFdg5VHGJnxU4cDgcWi2VVmXeukAuS6Zjr4Knep3j/7vdzuOzwlnVhXi7yUEQEbrc7oWlXIZxgMLglIhmv1wugkcxmYq3el1RQnJNDoRCnTp1CEASGh4dzerzppstkWebq1atcvXo16diA1badq8I/XG9GdTqdacmT09nuzZouSweSLHF+/DxROcqekj10znUSLA5SVVfFufZzXA1dJX86n76+PkwmU4JU2mAw3HSRTFSK8troa/Q6ezlnPseBkgObEslkYx+KiCDeiUBRFg4ODuL1etHpdPT3929YRJAKkiSlfJBcWloC0Goym4WNpMfinZOPHz+OXq/H7/cTjUZzmqtOJ12mTIr0+Xzccccd2O32tLada5K5dOkSRqORxsbGrPUj5HohvZFOzP3ufrrmuqjKr0IURIotxZwfO0+VrQpZL+PRe6jYU8ER65GkqiglCshVPSDb13mns5MeZw97i/fS4+qhY65jy0Yya2G5iGBwcJC5uTnC4TA9PT0Eg8F1iwhSIRqNppw6qjgwb0ZUlSvcNCSz3rHIqZyT4wct5Sr/uhYRuN1umpubKSwspLGxMaPUXa5qHLOzswAUFhZy4MCBrF3guZq4uRw3IlqKj2LyjbHURnleORfGLzC6OMpdNXcxujBKy3QLlTsrE6TSynyWwcFBZmZmmJ6eTqgV5OXl0evqJd+YT5Wtat3HmE2SUaIYAIfFgdPv5NzoOY5IR7ZETWajUGo6DQ0NCSICt9vN2NgYkiSpQ/XWEhGkQjpTMW+kC0U2sOVJZiNjkddyTlZ+3HQUHuvFaiQjyzJDQ0P09/ezZ88eamtrM76Qsr1ox0/6BNixY0fWb+ZbNZIZWxxjdH6USDTC5anLWPQx52en34lO1KETdFTkV9Dj6uFw+eEEslDmszidTvLz8ykpKVH7c65evUpICPGk80m22bfx5TNfxmqxrusYs0kyShRTY68BoNpWTY+rBxs29gv7s7KP1bBZtjLKmrCaiGC5E0G8ci1d5++1pNI3u3wZtjjJLB+LnMmFlY5zcry3WK6QrCYTDodpa2tjYWFhQ9LpbKbL4hVtjY2NXLhwISeOyTd7TWa1RbrcWs4v7/1lZnwzvDXxFvtL9yMgIAoilbZK9KIem87G1NJULJrJr0y6LVEUEwa2SZLEv3f8O9MT00xPTvNPP/snDpUeUus5mdQKskUyShTjj/gJRAMEogEAgtEgrUutvIf3bHgfqbBZBpmr7SNeRKD8RoqcfbmIQPlvtebudCKZ9UZJWwVbkmSU2ksgEECv12cUvWTinKw0beZy5svymsz8/DzNzc3k5+dz+vTpDTkLZItkFHlyYWGhqmjLlS3/ZuBGEJlJb2J/yX6mhqeIyBEC4QBhKYxJZyIUDdHn7gMgGAnS4+7hqPco2/ITG1mTHbc34uWi8yKVjkr8YT/uYje1dbXMe+YTagUK6dhstlXPc7bOiyfoYSm0RFleGYFIQH291FqKy+NiIbRAMRuzZEqFzar7pEve8XJ2SBQRxNsTxY8zUIr9a5FMtqZi3khsOZJRCGZiYoLBwUEaGxvTvqDW45ycyok5G1CIQJZlRkZG6O3tzZqzwEZJJl6evHv3burq6tRjykW9ZzMimRv5xDflnWLAPcD2gu2ML41TV1DH+3a/b8X7BAQ1nbbib8uO/8L4BcYXx9nr2EswGiOo6R3THNl3JKlrMaDWCRSptIJsRTLFlmK+cPwLROTIir+9cf4NSqwlG95HKmxWumy9D4DLRQTxk1yVBwNlnEEoFNqUqZg3EluKZOKtYZSBYuncFBtxTt4MkolEIjQ3N+PxeDhx4oQqm9woNkIEkUiEzs5OnE5n0mPKRZH+VkiXrQZZlmOS5WiQGnMNS6ElolKU4xXHMerSW6yWn5vF0CIvD7+MzWhDL+rRi3okSeJnQz/jttLb0Im6BNdiSZJYXFzE5XIxNTVFb29vwsC2SCSSNRK2GJKTpFEw5pQAlN6tG5kuyxTxk1yBhAeDUChEW1vbCqGH8jvd7L5lAFtCF6cU90OhkPrjpju1MhwO09zczMDAAMeOHWP37t0ZXRy5kgHHH9/4+DiRSIQzZ85kjWBg/UTg9Xp544038Pv9NDY2Jj2mXKXLbmZ1WartKlFMRV4FEFOWjS+NMzQ/lNE+4kngwvgFhjxDiILI1NIUU0tTGHQGOp2dtM22rfisKIoUFBSwfft2jh8/zl133cXu3bsRBIGBgQFGR0eZm5vj6tWreDyenFz3uU5lbdacl1wqTi0WC1VVVRw8eBCdTsf+/fspKirC7XZz+fJlzp07x+uvv87f/M3fMDs7u2GS+fM//3NOnjyJzWajrKyMD3zgA/T09Kz5uZ///OccP34cs9nMjh07+Id/+Id17f+GRzKr9b7o9XpVUbYasuGcnKtIRpZlxsbGmJqawmazceLEiazffOshyOnpadra2qiurmbPnj0pi5u5TJfJskwwGMRkMmV1wbgR6TIlilkILWA32ZkPzquvt820UV9Qn1Y0s5zE3AE3NQU1Ca8V6AsQENR9pMLytE13dzd+vx+/309bW1uCDFcZCLaR87cZUYZyjm6mSCYVJElSF/94EcFbb73Fv/zLv9DZ2UleXh6f+cxneMc73sF9992nDuBLFz//+c/53Oc+x8mTJ4lEIvzBH/wB999/v7rtZBgcHOTBBx/k05/+NE888QSvv/46v/3bv01paSkf+tCH0t73ww8/fGNJJlXvS6rFX5H/9vX1sXv37g3VN3JBMkoqam5uTh2OlYvFLxOSiZ9Hc9ttt1FRUZHy/bmMZMLhMC0tLczNzWE0GnE4HBQXF+NwOHJq8ZMrBKNBFkILlFpLCUaD6uuF5kIicoTF0CLFlvQK4fHXya82/Cq/2vCrWTtOnU5HXl4ee/bsSfDympubY2BgAIPBoKZsHA5Hxg9tyvVyq0Qym7EPWZZXOKsXFhbyzne+k3e+8538zu/8Dk6nE5vNxp/92Z/xyU9+EpfLlbKBczmef/75hH8/+uijlJWVcfnyZe6+++6kn/mHf/gHamtrefjhhwFoaGjg0qVLfP3rX8+IZObn528MyaTT+6LT6dQBZPE/djAYpK2tDa/Xy+23375h5+Rsk4wiPjCZTJw+fZrJyUnm59d+6lwP0iWZ5fLkdMLvXDVORiIRLly4gNVqpbGxEZ/Pl6DCsdlsKunY7fZ13eibXfcx6828d+d7icorryMBAZM+vQVhM21lkg1sm5+fx+VyMTIyQmdnZ8LAtoKCgjXTR5sRZSjX+1ZSl60X8dmb1RAIBDh48CB/8id/AsDCwkJGBJMMynqUyibqwoUL3H///Qmvvetd7+KRRx4hHA6n/TD4h3/4hzeGZOJTMasNFlN+4Egkoj5ROZ1OWltbKSoqyppzcjYlzOPj43R2diaID3IpkU4n2lAcBYqKitY03Mx025liYWGB+fl5du7cqToXm83mFZ3vLpdLTecUFRWpUU46DW43Sl1m0BkwsPHrMdcGmastaDqdTiUUSFREdXV1EQ6HE6TSyQa2bVYksxljkTcjXaasC5lImNO1nFoNsizz0EMPceedd3Lw4MFV3zc1NaUKFRSUl5cTiUSYm5vLaI7UDUuXrfWUEN+NH++cnK55ZCbHsdHaQzQapauri+npaY4cOZKQM83G9ldDvDw62Q2vSKaXy5PTQTZrMsrAr7GxMaxWK7t37066baXzfdu2bYQiIYL+IE6nk+np6QSlVHFxMYWFhesaN7BZcAfcBCNBKvJTpyXjsZUMMuMVUbIsqxGny+ViaGhINZhUSMdsNm9KlLFZc142s+Ez1fnK9iyZz3/+87S2tnLu3Lk135uth4gte5cKgoBOp8Pn8yU4J2d7eM9G02Ver5fm5mZ1zv3yp+1cqtdWm2AZiUTo6OjA5XKtWzKdrXSZUn/x+Xzs3r2bqampNT8z55vj4bce5iMNH+FQ/SHq6+uJRCLqItfX10cgEFCfrIuLixOerG+0TFqWZX7U9yPcATe/ffS3006Xwda0+hcEgby8PPLy8lSptNLhPjExQU9PDxaLhYKCAiC3qabNJJnNSJdt5lTML3zhC/z4xz/m1VdfTToEMR4VFRUr7tWZmRn0er2aeUgXW5ZkIHZxNzU1UVFRoTonZxsbIZnJyUna29upqalZVam1GSQTf+N5vV6ampowGAycPn163fnbbKTLFhcXuXLlCvn5+TQ2NuJ0OhO2vxpeGXmFpukmrAYrB0sPIgoxSXtpaakaJfr9fpxOJy6Xi+HhYXUUstJrdSPR7+mnfbadQCRAy0wLt1fenvL9S6ElnH7nlopkUmH5wDblAWBmZgaAc+fOqc2G2XQshtUl0pIs4Q17sRmz8xC6WemydEhmoxJmWZb5whe+wFNPPcXZs2fZvn37mp9pbGzk6aefTnjtxRdf5MSJExmXKW4YyaS62BXn5Egkwo4dO9izZ0/OjmM9NRNJkuju7mZiYoJDhw6tyF1udPvpYrn3Wrry5HSw0XSZQsDbt29n586dau0tfiFNdg3M+eb46eBPyTfm0zrbSstMC0fLj654n8Viobq6OuHJ2ul0EolEaGtrSyhaFxYWbppVuizLvD72OsFIELPezOtjr3O47HDKaOb7nd+nabqJXy38VSqFylXfl41jy0WkpDwA5OXlMTs7yx133KFGnYpjcXxqbbWBbelgtQijabqJ1plWPtLwEfIMG3/y34xIZq19KGnKjUYyn/vc5/je977Hj370I2w2mxqhFBQUqJmXr3zlK4yPj/P4448D8NnPfpa/+7u/46GHHuLTn/40Fy5c4JFHHuH73/9+RvuWZXnrRTLxzsl5eXkbVo+tBZ1Ot2Y/Tjx8Ph/Nzc0AnD59Gqs1tSNuLmsyyo0ajUbp6elhdHSUgwcPrilPTgfrTZcpUunR0dEV3nHpREevjLzCtG+a/cX76XX38pOBn3C47DCisDpJxD9ZT09PqzUfZcR2JBJJWOQ20g+y1ueUKKbaVo1Jb2LAPZAymhlZGOHnIz9nzj/HZfkye+v2ruu40kEu5ybB9SjDYrFgsVhU+b7iQjA7O6sObIv/PTJ5Mk4WyfjCPi5NXmJ4YZiuuS5ObDux4e+xVVwFlpaWNlwi+OY3vwnAvffem/D6o48+ym/+5m8CsYdCxZYIYPv27Tz33HN86Utf4u///u+prKzkb//2bzOSL0PsftlSJLPcOfnixYs5tXyBzNJlSqRQWVnJvn370roIc5kuU262lpYWIpEIp06dypoFxXoimVAoREtLC4FAIOmxrEUyShRTbC5GFESq86tTRjOrHbdiu64Urb1e74p+EEWxpkylzAbioxibKbYwrBXNvHD1BdwBN0XmIs47z/O+8PuoYOMPCasd32ar1wRBwG63Y7fbqa+vJxqNqgPbhoaGEmTrygTKVPdVsppMl7OLiaUJikxFXJq6RENJw4aimc3qxdmsmkw6D4uPPfbYitfuuecerly5su79RqNRHn300a2RLlvNOTnXvmLp7kOSJHp7exkbG8s4Ushluszj8QCx+fInTpzIas0q00hmYWGBpqYmbDYbjY2NSY9lLZKJj2IA8o35RJYiaUUz8Viekou3zo9f5AYHB9VFTiEdu92+7oVYiWLyjfm4A24A8gx59Lv7k0YzShRTnldOobmQS3OXeH3qdXbX7F6x7VA0xI96f8SdNXeucG9OF5tBMmttX6fTrRjYpqTWlKgz3oVguc39cpJRohi70U5FXgW97t4NRzMKydzowr+SLsu22GmzsLS0xF/91V/d+EgmlXNyOtYyG8VaJOD3+2lubkaSpKSDz9ZCLtJl8fJkURTZvXt31kURmRT+lQh0x44d7NixY9WFJtU2/WE/50bPIcsy3a5u9XVJlhjwDNDv7mePY+3aXKaLXCAQUHtzRkdHARJ6czIZOz2xOEGeIQ9JlhIs8B0WB6OLo9xOIskoUcz+kv0xBZcuj5fHXuYDt32AQnNhwnsvT13mmYFnWAov8clDn1RfH1sco8hclNaT+2alyzKByWSioqKCiooKNepUSGdwcBCdTpeQWltOMkoUs7toNzpRR4GxYMPRjLIebEYvTiqSCYVCRCKRm9Yg02q18q1vfevGkYyyUKZyTt6sSGY1EpidnaW1tZXy8nIaGhrW9WST7XTZcnnylStXcqJKSiddFj/aenl/0GpY7VhNehMfP/jxhMVZgSiIVNtSSy7T2UcymM1mKisr1frB8uFTFotFJZy1rsW7au7iWMWxpH9bbu2vRDF5xjwWQ4ux9+gsjHvHOTtylg/s+YD63lA0xAtXX2AxuMgb429wX9191BfUMx+c568v/jWHyw7zG7f9xprfdTNIZqNiEyXqrKmpQZIk1YVgfHycrq4ujMaYy/Pc3BymfBOXJi9h1VuRZIlQNITD4uCq5+qGohmlIH+jSWZpaQngpiUZg8HA3XfffeNIxuPxqM7Jq+mub1S6LH4M8f79+6mqWv9cdSXtlA19/9LSkhr1KfLkXNm/rBXJhEIhmpubCYVCaUd4qbYpCiInt51c9/FmA4IgUFBQoDoZh8PhhDkggUAAo9HIyMhI0lSOKIhpS2i7nd3oRB16Wc9CcAEAv+Sn0FJI22xbAslcnrpMn7uPhpIGBjwDvDL8Cp889EleH3udAc8Ai6FF3lb/tjWJeCukyzKB0vCp9HmFw2H6+vrweDz09vYyMD/AQHAAjOAL+FS5vllvptvZvSGS2Qw1Yjoko4x+vlkRiURuHMk4HA7uuuuulGmeTJVf68FykgkEArS0tKiL50afIpL1sqwHU1NTak9O/DiDXAkLUpHX/Pw8TU1NFBQUZGxVk2tkcx8Gg4GysjLKysqQZZne3l4WFxdxu91cvXoVvV6vpnEyNZR8R/07OFx2OOG1pqYm6urqqK2oVV9Tohi9qMesN1ORV8Eb429wYtsJXrz6Ig6zA1fAxctDL68ZzdyIwn82YTAYyMvLQ5ZlDhw4wMGlgxycOojb7cYz7wEfFBbEVIbbStdXt4LNdWBOde8o8uWbefSyTqe7sTWZtRYnvV6P3+/P6THEk4zT6aSlpYWSkpKsNX8qTyrrJYK1RAe5IpnV0mWKP9vOnTvZvn17xlY1N+s8GUEQ1EWuoaEBSZJUAYFiKKmopBRDyVQLlSiIlOcl9lc5jA5KLaUJtQQliqkvqCcUDTHrmyUYDfJo66OMLY2xz7EPS8DCubFza0YzW7Ems559KOfVnm/n6K6Y6jA+1el2u+ke62bQNKj+HpmoCDejRwZiZJbqwUSZinkzk8yWkzAvx2aky5TCf39/P4ODgzQ0NFBVVZW1H3Z5w2QmCAaDNDc3Ew6HV01J5ZJk4hdrpQF1cnIy7frLWtu8maE4DCiGkvHmnu3t7SsaEJWUhyRLPN7+OLdvu539JfsTtrnCHkiK8MLVF5gPzjPgGWDWN8vw/DCl1lIG3APsLd6LXtRTYimhw9mxZjRzs0cysHpGYHmqMxKJJKgI29vbsdvt6m+S6iFgsyKZtdJl2bSUuRFQ5gttCQnzatgMklFm2kxMTHDHHXds2OV0OZRO90yJQHFPdjgcKaOqXC3c8ekyhewikQiNjY3rzhFvBsncqKe+eHNPZVaL0+lkZmaGvr4+1dxzTB7jqZ6n6HP18T/u+h/oxNRPzA3FDTgsDs6Pn8cb8mLSmQhGgoiCiN1oZzEYEw3kG/LXjGZulUgmnX0sH9gW/xDQ0dFBNBpVpdJFRUUJEcNWqckoJHOzRjLK2ndDI5m1Fp1cS5hdLpfavd/Y2JizgVmZ9MrIsszw8DB9fX3s2bOH2tralBdZrtNlSv2lsLBwwynEZLYyuSCdGx0txc9qUcw9PR4Pc845/rn5nxn3jDO/MM8z+c9w3+77sNls6rmI/631op5f3f+rvDL8Cj8b+hmBSIA7a+7k/Nh5KvMriUgR3MFYP44oiIiCSJ+r74aRTK63D+tPZS1/CFAadJ1OJwMDAwn1tXA4vGnpsls5knnxxRfx+Xy/mOkyWZYZHBxkYGCAnTt3qv0muUK6vTKRSIT29nbcbnfa7sm5JJnFxUUuXrzIrl27NjR9NH6bN5oAcoVp7zTPX32eD+39EPnGRLGI8lQ9EhlhRj/DwcqDjM2P8dzQc9gWbRh0BhwOB5FIZIW5pzvg5tzYOWZ9s0TkCP6wn11FuygwFfAHp/9gxb5KraunMbe6hDndfWz0YTBZg+78/Dxut5vR0VEWFxfR6XT09fWp3ne5IJ101GU3M8m89dZbN24yZrrIBcmEQiHa2tpYWlri9ttvJy8vj97e3rQsHtaLdIggmTw5W9vOFJIkMTc3x9LSEseOHVNTDtnAzZ4uW237Lw6+yE8GfsK2/G3cv/3+FX+XZIkf9/2YUDREkbUIo8HIxNIE1l1W9uXvw+VyMTMzQ0dHB8PDw+pTddN8zPgxEAlQaCpk2jvNLscuBj2DdDm7eO+u96Z97LdKJJNtIosf2LZz504GBwdVs9Xu7m5CoRCFhYVqPUeJPDeKtaKybDgw30h85jOfwWazbe10WbYlzB6Ph+bmZux2uzpZU9l/rvzFYG0imJqaoq2tjdra2gR5cja2nSmCwSBNTU0Eg8GEnHY2cDOry1JhfHGc10Zfwx/x88LVFzhddXpFhNE83UzTdBNVtljPVZ4hj6gc5emBpzl510kKCwuZmppSzT1dLhcX2y7yw7EfMrA0QEAOoEPHfHAeGZmIFOHZ/me5t/beFftKhVu18J8uJpcm6Zjt4N66e9GLqy9/VquVhoYGZFnG7/er9RxlrIRCOEVFRWlNbE2GtR5ss+HAfKMgSRKlpaX86Ec/2tqRjF6vz0oko9Q5lCmR8akfQRBy6i8Gq9dk4uXJa40MWA3ZXLg9Hg9NTU04HA5KSkrUjuNs4VZNl708/DJOv5ODJQfpdfdyfvz8imjm6f6nmfXNEpaup8OCkSCtM61cmb6iNqIaDAYKCwspLy9nyjqFbklHGWX4gj4i4QhROUrQH+RA6QHKrGUEo0HySY9kcl2Y34zC/0aITJZl3pp4iw5nBzX2GvYWJ3e8jicypRnSarWqYyUUV2nFFUIRdSikk27dMp102c0aySjXwU9/+tOtTTJKumwjYXg4HKa9vZ35+XlOnjyZtM6RaxVbsppMOvLkdJCtSGZ0dJTu7m51VPPw8HDWCWEzJlduthJHiWLK88ox6AzkG/KTRjMNxQ3oBB3BaJASy/XoUCfoVJ+y+PMSlaJ0O7upsFVQYYv1RkmyRDAYRAyLnMw7iSPqYLBjkHnHPMXFxWsOB7sVJMwbkRePLo7S4+4hFA1xeeoyO4t2Jo1mUqWxRFFcIZV2u9243W4GBgbw+/0JUmm73b5uqbTX61Wnjd5sUK6zP/7jP976JKNorddzc8zPz9Pc3ExeXh6nT59etfFpMyKZeCJIV568nm1nCkmS6OzsZGZmJsHiZ6NDy5JhswggVySWbLvxUQxAZX5l0mjmQ3s/xPc6v4fL7+KThz6J3ZRcKq+cI52o4yMNH0nq5SYgUJFfQSgYUtM4Y2NjAOriVlxcvMLc81aQMK+XyGRZpnmqmXA0zM6inQzPD6u9RssRjUbTFhcsn9gab7g6Pj6e0C9VVFSkzjJSZtaslS7biKXVVoDD4bjxNZlUUH6ASCSSkWWHLMuMjo7S09OzpjOwsp/NqMlkKk/OZNvrQSAQoKmpCVmWaWxsTMgt5yK1JQgCfb4+jnmPUZqXeTNnuvvYCCYWJ3i6/2l+/eCvYzWk7gea9k7z2uhrhKKhBOdob8jLC1df4O6auzHrYwt9v7uffnc/wWiQtpk2ztScWbG95ee72JJ6lvpyc8/FxUWcTidTU1P09vZisVgS0ji3QiSz3pqMEsVsy9+GSWdCJ+pWjWY2UvdZ/pssLS2pA9v6+/sxGAxqhAOpxwl4vd6b2rdMwZaPZICMoox4l+JU5pvL95PrSEYZ6OV2uzl58mTWJn6uN+JQoqni4mIOHDiw4mLPhfHm2OIYF+cvUjZUxkcOfCSr247Hqsft9yO2tyNOTSGbzUj79iHX1CS85anep3i6/2mqbdW8Z9d7Uu7HIBq4p/YeotLKa8esNyNwbXKpFOXi5EUEQaDYXMxbU29xW9ltSaOZtUhgNaKIHw4Wn8ZxOp309vYSDAaBmMhEkfBmm3A221YmXcRHMUoKs9JWuWo0ky1bmfh+qbq6OnWWkSKVBrhy5UqCFVH8fm92dRlAa2vr1iYZQRAyIoDFxUWampowm80Zy4BzSTKyLDM0NER+fn5Gx5UOMiWD+CgvVTSVi3TZW5Nv4Q67aZlu4XDxYRZGFzCZTJSUlGRUMF0X5ucx/NM/IXZ2giSBJCEXFRF9//uJnjoFwPD8MC8Pv8xSaIkf9f2I++ruSxnNOCwOPnbgY2vuWoliqm3VmHQmup3dSaOZtX7HPlcfT/U+xWePfnbVdJuC+DSOopC6ePEiS0tLXLlyJUG2m6m552pYK/2TDayHyCaWJuj39BOIBOhx9qiv+8I+mqab2OPYk7DNXNnKxM8y2rZtGxcvXqS2thaXy0VXVxfhcJiCggIWFxcxGo1ZKfy/+uqrfO1rX+Py5ctMTk7y1FNP8YEPfGDV9589e5b77rtvxetdXV3s27cv7f0qD0Mf/ehHt3a6DNKPMsbGxujq6qK+vp5du3ZldCHmMpKZmprC6XRSWFjIiRMnsn7xiqK4ooFvNUSjUTo7O5mdneX48eOq71YyZDtdNjI/QutMKxXGCuYW5/jeq9/jw/s/jCAIasG0oKBArSes90l7tc/oXn8dsbUVafduuEbywugoumefJbp7NxQX80z/M3gCHvaX7GdwfpBXhl9ZM5pZC/FRjDJTpshctGo0s9rxy7LMswPP8sbEGxwsPciDOx9M+xgUhZQgCOzduxeLxaLOaVmPuedq2KqRTL4xnzPVK9OTACadacUxb4ZBpuLAHD+wzefz4XK5+NGPfsQ3vvEN9Ho93/nOd5BlmXe+853rqs94vV4OHz7MJz/5ST70oQ+l/bmenp4Ei61MvQqVc3r06NGtHcnA2tYy8Qvn0aNH19XXkYuaTPxAr43cuGsh3ZqMMuET4PTp02tOfMx2uuzN8TdZCi+Rr8/Ht+Bjvmge6zYrVflViKKo9iI4nU6Gh4fVpz5l4cuky3vFcUsSuqYm5KIilWAA5KoqxJ4exIEBBvVLvDz8MiXWEow6IyadKa1oZi30u/vpc/UhCiJji7HivCRLTHmnEqKZsyNnGfeNc4QjSbfT5eziytQV9IKeFwdf5M7qO9eMZpZDebqMn9Oyc+dOQqHQCl+vZOae6W4/l1gPyRSYCrij8o60378ZBpnL5cuCIJCXl0deXh5/9Ed/xO/93u9x9913U1NTw7e+9S0+9alP8YlPfIJHHnkko/088MADPPDAAxkfX1lZWVZS+k888cTWJ5lUUYbSJW8wGNJaONezj/VAmUmjyJNHRkZyJixIh2QUj7aysjL279+f1g2UzUhmZH6E5qlmdD4dMjL76vcxEhjh4sRFPrjngwBYLBaqqqqoqqpSPdOcTidDQ0N0dnZit9vVKCfjjmtZhkgEln9vQQBJQpAkNYrZVxxLCVTmVyaNZlLtdzG0uGJoWVSOsqNwBzKJ53Jb/jaicuyam1ic4LHWx2Ae3iOvjJxkWebFwRcJRALsceyh29XNubFzGUUzynaSHb/RaEx4oo4vVvf19WEymVTCT5bWDEQCTHunt3Thf6vtY60eGbPZjMfj4XOf+xxnzpzB4/EwOzub02OKx9GjRwkEAuzfv5+vfvWrSVNo6UCSpJs3XabMlV9Pl3y6+1gPXC4XLS0tFBcXc+LECXQ6XU5rPqnIIF7NtnfvXmpqatJenLNZkzk3dI724XYcRgfBUBBLyAIytM62cqLiBHWFdQnvj3/S3rVrlyoLdTqdjIyMqDb7ysIXX09I+v10OqTbbkP34ovIZWVw7eYWZmeRCwoYKTHycuvLMeXXbBtzvjm22bbhDXn5Ud+PeHv92zEHIuhcLqKr1C6appt4pv8ZPnPkMwlzYvaX7F9h6b8cLw29xIxvBr/fz5WZK7zN/raEvytRTLWtGr2ox260ZxzNpNsKkKxYrUwHje8DUc69zWbjry7+FS9cfYE/afgTbLb0JoOuF5tFMrlOl6UTLcUX/gsLC7MmFkqFbdu28e1vf5vjx48TDAb57ne/y9vf/nbOnj3L3XffnfH2RFG8OSKZ+HRZNBqlu7ubqakpDh8+TFlZWVb2sVESUIr7/f39Kxb0TOommWK1SCYajdLR0YHT6UzbbHP5drMRyXg8Hvr7+jlccpiKigp6unuoslWh1+uRJVl9mk+FeFmoJEksLCyohLM8ylEW0+WI3nknYn8/Ync3ssWCEAqBXk/k3e9Gv62KO113EoqGOD92njnfHFX5VRwoPYBdtCK++Sb6/iGKxsYIG42I0SjSoUNw7Yk+IkX42dDPaJ9t5/Wx1/nlvb+c9vmZWJzgleFXKM8rZ3B+kBeHX+TuHXerstr4KEYhlMr8SnpcPeuKZjJNZ+l0ugR7oXiLlZGREWZDs/zzwD+zEFngubHn+Hz55zPafqbY6g2fmewjFZEpNZrNtpXZu3cve/deV9s1NjYyOjrK17/+9YxJJhgM8vjjj299kom3lvF6vTQ3NyOKIqdPn163Z9BybLShUXFP9ng8SeXJuYxkkh273++nqakJURRpbGxcVxoxG+kyZYrmb5z4Derq6hAEgeedz3P3bXdjsVgIh8MZn3dRFNWnup07d6pzQpxOJ2NjY/jCPgLhAIIg4HA4VCWfXFZG6Ld+C92VKwhXr0J+PtKhQ0gHDlAuinzp9i/R4+zh0tQlagpq0It6PnPkM5Rc6kC8fBm5pIRwcTEsLKB7/XUApGPHAGidaaXH1UOZtYxzY+e4s/pOyvLSe/h5aeglnH4n+0v2s2RaosvVxaXJS5yqiineBjwDdMx1EJbCdDm71M8Fo0FeHn6Zt9e9HYPOwND8ENsLVp9UqvyWG62ZLE9r/tHZP8Ib8WLEyHPjz3GX/S78fj/FxcUrJLkbhdJrdqtEMqn24ff7kSQp55FhOjh16hRPPPFExp9zOp389//+32+edJky476qqoq9e/dm9ULT6XRqH0GmWFpaSpBNJ5OC5rLZcznJOJ1OmpubqaiooKGhYd3naSPpsnhPtuVijGwXhuPnhESiEf7PS/8Hu9eOcdxIV1cX+fn5qmzUXlQE73xn0u3Isswz/c+wFFpir2Mvva5eftb5NP+hL4pUVgaFhcjBINGiImSjEbG7G+nAASIGHS8Pv4yAQK29lva5ds6NnUsrmlGimBJrCYIgYBJNhAjx7MCznNh2Ar2op8xaxof3fThpxGfVW9GLen4+8nMeaX2E/3rHf+Vw2eFVvx9k9/yPL43z3PBz2C127EY7o55R3gi8wY7IDlWSqwwGKy4uVhVu64XyHX4RajJerxdgS/TJNDU1sW3btow/V1hYyPe+972tH8mIosj09DQ+ny/pjPtsYL3pssnJSdrb26mrq2P37t2r3kC5mvkSv+34dN2+ffuoWdZkuJ7trieSCYfDNDc3EwgEknqyxQ/nyjbhjCyOMBmcJGgKUttQS6GhEJfLxfmr5/EN+Kg111JUVKTWE+IjvF5XL29MvEFFfkWs7mGy85OrP+Hd3uMUlCb2B8j5+YguF3i9tIaG6HH1UGuL9RuVW8vTjmZeGnqJscUxSiwlDM0PMReYw2q00jHXweWpy9xReQd2kz3p6AAF4WiYp3qeotvZzVM9T3Go9FDS85oLknm8/XHmQ/NU5lciCiIG0cDzk8/zxXu+yL59+/D5fDidTlwuF1evXlW73YuLiykqKsp4LoxyD+WSAGRZ3hLpMq/Xi06nW7eYScHS0hL9/f3qvwcHB1VLq9raWr7yla8wPj7O448/DsDDDz9MfX09Bw4cIBQK8cQTT/Dkk0/y5JNPZrxvq9XKfffdt7VJxu/3Mzs7iyRJnD59OmcWC5mms+LlyenUhXJJMkrE0draisvlypqbwHrSZUqjX15eHo2NjUmbK3PlxByVorTNtiEKIr6Ij465Du6ruw9LkYUZ8wx6q549NXvwL/iZnJzkcsdlLFYL9eX1FBUV8XTf0yyFltSpkhV5FfTOdPKSfohfWaqGOKNCYWkJ2WolYjbyct/LABh1RqJSFIfFQaezM61oxqgzcrrqtPpv06KJqpKqjBbfc2Pn6HZ1U2ev462pt2idbU0azWSbZEYXRnmq5ymMOiMRKVYztegsOANOvt/5fb5w4guqJDd+MJjT6WRwcJD29vYVisG1FvbNIhlIbfeSDaQzS8ZqtW74u166dClBGfbQQw8B8IlPfILHHnuMyclJRkZG1L+HQiF+93d/l/HxcSwWCwcOHODZZ5/lwQczq/0pkGV566bLZmZmaGtrw2KxkJ+fn1MPn0wimUAgQHNzM9FoNG335FzWZMLhMEtLS+h0uqy6CWSaLpuZmaG1tZWamhr27Nmz6m+bjGQkSWLAM8Bux+51H+/wwjBDniFKjCWYzCZ6nD0cKDnAgGcAV8AFwER4gmPbj1FXX0dvRy8TixNUhCp4uellXhh+gSV5idZQKwaDAVEUCQpRfmIe5T1To1gkCUIh9AsLCGYz0TvvZDA4yYxvhqgU5bWx1ygyF2HWmTGJJlpmWnj3jnen7LH56P6PJvz75ZdfzkiKr0QxAOV55fQ4e1aNZrJNMq+OvkpEjiDLMu5AbAR0KBLCqDPywuALfOHEFxLeH+8wACTU0sbHx5FlOaE3J1m9VWn2zGUvjnKfbkYkk8rhIls2//fee2/Kh7rHHnss4d9f/vKX+fKXv7zh/SoQBGHrRTKSJNHX18fIyAgHDhwgGAwyPz+f032mWzNR+k1KSkqS+n1tdPuZYm5uju7ubkRR5OTJk1m9MdJNl8WPsh7OH2bEP8JeIfmcDkhOMo+1P8b3O77PX7/jrzlYejDjY1WiGEEQMOlM2Aw2nBEnFycvMrYwRrGlmIgUoWm6ib3Fe5lammJoYQgZGWOFkXdsfwehkhCeRQ/eJS8+nw+j0Uh+fj6leYUI/hqEwWEMs7NEzGaip08jHTrEdhE+dfhTdM118fLwyxwuO8zp6lhkYtabM2riXE90p0QxavSVX7FqNJNtkvmVvb+yQprd3t5OdXU1eyr3rPn5+FqaYu7pcrlWNfdU7qHNqMfA5pBMKisfr9d70w4sW44bTjLxi87yJsb8/HxGRkayOh0zGdaKZFLJk9NBttNl8Qt7bW0tU1NTWb8p0olkotEo7e3tuN1u6g/W89ibjxGKhrin7h52Fu1cdbvxC+pCcIEnu59kcH6QJ9qf4H/d978yPtbhhWEGPYOUWEqYlqcJRoMUWYp4beQ19KKeoxVHkWWZPncfXXNdDM4PIiMjIHBl6gof3vdhPnbkugdZvLmk0+nkdTlMyZ49UFODrqCAkhMngNjNU2ev49zYOfKN+XiCHhyGAsrdIQiHkfVeyHChSPe6UqKYcDSMJEv4wj4MooHF4GLSaCbbKUqT3sTR8qMJrwUHg+wp3YPDtrpdUTLEm3vW19er59/lcqnmnoWFheqim0tnASVautE1GZ/Pt2GhxFbBDScZBXNzc7S0tKhd6coPkK3pmKmQKp0ViURoa2tTh56tp96RTZKJl0vffvvtSJLExMREVrYdj7VqJ8tl0j/o+QHT3mkEBJ7rf44vnPxC0s/Fb1cQBH7Y80NGFkZwmB2cHT1L+2x7xtHM+OI4Rp2R+eA8rpCLgC6AWW9mwjtBXUEdoiCCELMX+enQTwlGg9TbY9NRr3quMuAZYI/j+tP3cnNJpYA9NjZGYGEBz4ULqnhgLDLG6MIo+xz7GBi5Qseb/4uacQtEIkjFxUTf9jakO9a2NMmUBAbnB5n1z2I1Wpn1X+8Et5lsDHgGmPPPUWq97jeVLbFFqgU+W95ly2e0KJ5e09PTRKNRXn/99aybeyrYjKK/sp9bdSomwL/+67/yyCOPUF9ff+NJRpZl+vr6GBoaoqGhgerq6oS/59qGP9U+FhcXaW5uTilPTgfZqsn4fD6uXLmCwWCgsbERk8nE/Px8Tgrpyo2WbFFxu900NTWpDwSz/lleuPoCxZZijKKRV0de5cFdD64azSjHO++f56nep9CLenXY13qimRMVJ2gobgCgQ+6gsLCQGXGGae80Zp2ZXlcvEEurtc+1U2otVd+vRDM7C3eiE1fe9PGeUqFQiHA4TGlpKU6nk/budp6ffJ6gPshiYJay9mFavR4Ol7+NcmMRwvQ0hn//d8I2G9L+1F3/8ftLB7uLdvOX9/1lwjhnBWadOYFgIDtP/wvBBX735d/lN2/7TTUtuHwfuViglfHHeXl5dHZ2sn//flwuF6Ojo3R2dqoy9Wx4BG5Gjwykpy67mUkGYnW3kZGRG0sysixz+fJlfD4fp06dStp4tLzjPxdIVjNJV5683u1nitnZWVpbW6msrEzoE8qVci1+VHL8d1fcruPHBLww8AIz3hn2l+xHQKBjrmPVaCZ+W//a868MLwyr8l+HaX3RjMVgwWKIFYodJgeFpsKYNYop8XoaXxxnYmmCEksJ88FYnS/fmM+AZ2BFNLMa4jvgA4UB9GE923TbWOjtYmF6EH9BPmd9A7xHvg1rZSX6vj7EK1fWJJlMHxQEQaDGnr5MPRsk83T/07wx8QYRKcIdlXesIOVcuzArBLCWuWdhYaFKOhaLJaNj2iqRzM1ek/nwhz/Mhz/8YeAGp8sEQaC+vp6CgoJVlRabHclkKk9OBxshAlmWuXr1KlevXqVhfwP2ksSZ4bkmGaXYKkkS3d3dTE5OJgyDm/ZOq1GMKMSOqyKvYmU0EwrB2Bh54+OwfTsLFh1Pdj+pDvYKRoLYTDaG5ofWXZtRIMsyOwp3sKNwR8LrF8YusK94HzIy/ohffd1utDPjnWF30W5eGnyJhpKGNRfwiBTh8vRlgnIQj+BhQBhirHSenaZ8ukKz7HaOkj+ho8DrxdjbC0tL5OXlpeUdlgtslGQWggv8c9c/IyDQOtvKz0d/ztvqEj3Wct2Nn2z7y809vV4vTqczwdwzvjdnrZlFmyEuUPZzK5NMPG54uqy0tDTlIrmZNRm/309LSwvRaDSrfTnrJYL4etAdd9zBoH+QF7te5FcafkX1scoVycSny0KhEM3NzYRCIRobGxPOy8+Hf8700jSiKOIJeGKfQSYYDfLK0CsxkhkaQnzpJYTxcSqHhrAMDnL2aD4LwQV0go4J7/Wakklv4sr0FTwBD4Xmwqx+pzuq7uBQ+aGkf7PoLfS5+3hx8EXGl8b5zdt+c81F+UDJAXYX7cYT8DA83IxNNlAm5vM2+352F1VjjugINzcza7fTe+lSymbEXKQ845GMZDIhnqf7n2ZsYYy6gjrGFsd4ov0J7qm5JyGaybXV/1oEoEz8zM/PT2numcrNW0uXZRc3vE8G1n5yU6KMXF7Ayo994cKFjOXJ6UCRA2fylOT1emlqasJkMnH69GkkUeLS1Uv0u/ppn2nndE0sJ75aWmujULa1uLhIW1sbdrudY8eOrXgSvK3sNj57/LNJt7GzaCcsLCA+8wzC3BxyXR2+SARHNMq7rsxjvfvzhKsqEcTE4y40Fa6bYFY7B4uhRWa8MynrRK+NvoYn6KFtto1+d3/Kvh29qFdnlDzb/yxWRzkHnDqm3SOUmqPYDGHEqVGMlZUYP/hBKrZvx+PxqM2IHR0dqptxcXGx2huTy0hmOf61+1+xm+y8e8e7U35WiWKMOqNqdZMsmtmMdFkmUUYqc09l/LEiHiguLsZkMm1quizVfpaWltIaHb/VsSX7ZJZDp9OlbVO+HijjiAG2b99OfX191vejEFa6N4nS2FhdXc2ePXsQRZGW6ZaYBYm1hCtTVzhYdhC76XrqLNtPYMo5uHz5csppow0lDTSUNKy+naYmhKkp5L17QRQRdDqk8nL0U1PcOW0idMeZhHMiOJ0Ig4MweBG5pgZ5HZ5JyRbU5/qfo222jS/d/iWKLddv3l5XLyadiWA0SPtsOzuLdjK+OM65sXPsKlp7wuqsb5Y3J96k1LaNvGO7CLcHeN11lX1LZqT6eqL33Ye8cyciJDQjBgIB1XJleHhYPQczMzOUlpZmbLmyFpbfP8Pzw7wx/gaXpy/T6+rlP5/4z6t+VolilBSi1WBF8kkJ0Yxyj+Zygd5oKmu5uefi4iJOp5OJiQl6enqwWq0YjUai0eiakcZGkY6Euba2Nmf73wwo33HLk4zy5ByJRLIqVVS2qaSjIDZLIRdEFk8EqSDLMgMDAwwODnLw4EHVlC4QCXBp8hJWvZVKWyU9zh41mskFySh1IIhZf2/oYvdfq31cO05FwixbLAgeT8JbxTffRP/SSwjuWAe5bLMRvesuovfdt3LgWAYYXxznzck3mfHN8Mb4G+oQMm/Yy9+89TfYjDZ2FO4gLIWxGW1U2arSimYALk5cxBVw0VDcgCyIVB29l1bXEB27HmTvzlPq7JrlMJvNCQuey+WitbWVsbExuru7E2a22O32rEuPXxt9jeGFYZqmm2ifbedjBz6WQL4KIlKEH3b/kGA0yPjieMLr3c5uLk5epLGqMSfeaMuRzXqJKIoUFBRQUFDAjh07CIfDuN1uRkZG8Pl8vPbaa6q5p8PhSKueli4Ur0GtJrNFoFxU2a7LLC4u0tTUhMVi4fTp05w9ezandvyQmmQikQitra0sLi5yxx13JMzX7nH2MLY4xq6iXYiCiMPsUKOZfEP+mtvOBArxLiwsAKxrnHUCiopiEyhDIbj2kCDLMsLiIhy8riATRkfRP/88CALSvn2xqGd2Fv3PfoZcWYnUsHq0FI9k/T3nRs/hCXioyKvgtdHXOFV1imJLMa+OvMpVz1VCUojhhWEOlcXqNTajLa1oZs43x5sTbyLLsjpaGcAt+zjn72a32Eg6S6Ioiurvffz4cSKRiGq5Mjo6qo4tUNI663nYiieZ4flhrkxd4arnKgICESnCX7zxF/zlfX+54nM6QcfHD35cVeMlHDeiKq7YDIfkXBblDQYDZWVl+P1+TCYTO3bsUFNr8eae6xkHvhzKOnOr12SU6+2Gk0w6aptsK8yUqZrxaaBcm1im6pWJHxfQ2NiYsIgoUYxRjBkwRqUodrOdq+6rtM+001jdCGSHZHw+H01NTej1ehobGzl79uyGtyvv2IG8Zw9CVxdyWRn6xUX0i4vI1dWxwV/XIPb1wcICcpzUVy4tRXC5ELu60iaZ5VCimIq8ChwWBx1zHbwx/gb31t3LM/3PYDFYmJ2fxelzJijkQtFQLJrx9LO7KHk0E4qGqLZVU5Gf6AxeY6/BrDcjyZK6vXQhCEKC5Up8WkeRj9tsNpVw7HZ7WgtvfCrrtdHXGFkYYcAzoP79kZZH+L1Tv7cimhEEgQ/t/dCa21euk1xHMrnugFcyAkpvVE1NjWruqaQ2lXqaQjjp/gYKflFIRjknN5xk0oFer89Kr4wiw52YmFghT861VHo1EpuenqatrW1VY8nxxXF8YR8CAhOL11VYVr2VPncfjdWNWXE2drlc6tyIffv2IYpidqZjmkxIv/RLCCUlCN3d6EIhQvv3Y3nggVi9Rfldg8HY91j2cdlggKWl9PYlSRjm5tB7vVBbC6KoRjEHSg4QlsI4zA5eG32NpfASVz1X2Vm0k4gUYWppinJreUKvjCAIWPXXlXTLz0WlrZLPHP3MisMQxscRh4eR37qMtGdPLJq7vhEIh8FgiEV4q2xbwfK0jtIX4nQ6aWtrQ5blFcXrZFC2r0QxA54BREQkYtdkqmgmHdzskYyCZLWS1cw9XS4XbW1tSJKUEOWsNUxRKfqnIswbMRUzV7gpSCYbBBDvnpxMnrzZJCPLMv39/QwNDXHbbbetOidne+F2Ptzw4aR/M+vNWYnCRkZG6OnpWTGHJmu2/AUFyA88gHzPPUxeuIC4bx8FVVUIcedbrqiIEUwwCMpCGY0i+HzI27evuQthYAD9009TfeUKOr0ew8GDjLz9JG/OvYlZb1YL9A6LA4POQKezE4vBglFnZFfRrliPRcTLh/Z9SB19nDEkCf3TT6M7exYWF2OvlZYS/uAHkU6eRPfCCxj+9V8RR0aQysuJ/PIvE/nABxLqTWs9qS/vC1levM7Ly1MJJ777XUmXXZ66zND8UEIUAzHZ+WrRTDq42WoyG9nHauae09PT9Pb2Yjab1XpaYWHhCkXmWvVTpd9nK0zFzAZuOMlkMh1zvXA6nbS0tFBaWprgi5bNfayF+HRZOBymtbUVr9e7qtOB+jlBXJGOSbbt9ZCMJEl0dXUxPT3N8ePH1Sc1BRuZjpkUVitSfn7CE7x6LA0NSPv3o2trQy4sjNVknE6k3buJHkre26Ie5+wsU//3/1AxOU/Ebge9Hl1XF7OeDix35KOz5DG6MMqMb4bF0CJVtiqG5ofYX7KfWV/M98ukN9E+287FiYtJLVPSgXj5Mrrnn4eiIuTa2hhJjo5i+OEPkdraMDzySCxys9kQ+/sx/sVfIDidhD/96XWRebyx5Pbt2wmHw2qUo3S/K0PalPcfLT/Kd1q+k3R7YSnMN658g/925r9lfCybYcO/WSSTSc0rlblnX18fgUCAgoIClfjz8/PTkknfCukyiM2nueEkkw7Way0T71a8b98+qqurU06vzCXJKNYyiuBAGeyVDanqekgmFArR1NREJBKhsbExaYiflXTZMijRUTQapbOzk0AgQElJSSzN8OEPI9fVIba2giQRPXmS6MmTCQPDksH91qv8jvgT3r73MB+aK0fU65Fqari9o4N9x+/Cd/od/Nn5PyMoBQlFQ5Rby7Gb7KrbAMTSj3mGPCa9k+v+brqmJgRAumbsiE6nfh/DhQsAyPX1sb8VFyNMT6P/l38h8su/jHwtst7IIm0wGCgvL6e8vBxZlllaWsLpdDI9PY3H40EURSKzEU6WnFQHjS3HcpeEdJHrRkxlH7lulNxon8xq5p7xUvX8/Hy1yXk1QvP5fLcEyRiNxpuDZNbT9R8Oh1WV1O23307BGgtVrma+KBBFEafTycjISMq+k/Ug04hjYWGBK1euUFhYyPHjx1e12sh6JHNtm6FQiDfffBNRFCkqKlLTDFarleK6OoqPHo2letJcUJ4Z+yl9piWWxG7uEsupogAEAdlsxjrr4c3ZNgbnB9lRsANnwAkC/MW9f0GJdaVyziCun/SFhQXk5TURQUD2+RA8HqS6uoQ/yQ5HrH7T2wtHjmS9mdZms2Gz2aivr2diYoLh4WEikQj3m+7nvor71CinuLh4zTrCWtiMovxaM1iygWz3mynmntXV1UiSxPz8PGNjY0QiEc6dO0d+fr5ayyksLFQfGG92CfPCwgJ/+Id/uDVIJhfpMiVasFqtabsn5zJdpjy1DA8Pc/jwYcrLy7O6/UwijqmpKdra2tixYwc7duxIef5zMSo5Go0yMDBAeXk5e/fuJRqNsn379gTZbkdnJ9FoVE0xpCpoO/1O/k1uoyiiZ87o4znTAJ8OHgFZRggECBUX8vzV5zGIBkx6ExV5FXTOdfL6+Ot8eF/yWlcqpDpf0u7d6Ds7kSXpep0lEEAwGJDN5piMO74WGAqBXo+cl5dzWxlRFDEajTQ0NKg5f5fLpXp8KXWE4uJiCgsLM15oc92IuVn7yGVKTnmoCgQChMNhDh48qEY5ndeu+Z/85Cfk5+dTVFS04Ujm1Vdf5Wtf+xqXL19mcnKSp556ig984AMpP/Pzn/+chx56iI6ODiorK/nyl7/MZz+b3NEjFbxeLy+99BKRSOTGk0w6yIQAksmTs72PTBAOh9VBbDt37sw6wUB66bJ4oUG6xp/ZTpdNTU3h8XgoKyvj4MGDRKNRwuFwrCtYr6esrIyysjI11TM3N6cWtBU7d0W2q/yuz/Y/y4Q+wB5dIXMLbn4aauZjr45TFjYhbd/O+SqJnvEe6u31SHLsHBWZi/jp0E+5r/a+pNHMehE9dQqxqQmhowNKSyESQXC7iZ46hVxWhu7s2RjZXCMcYXIS6dAhpAMHVHVdrhCfzor3+KqtrU2oI/T09BAKhVQn4/5QP08OPMlfve2vMOlXH+29WfLizVCXbZaCLZm556VLl3j66afVeVHvete7eNe73sXb3/72NbMxy+H1ejl8+DCf/OQn+dCH1pahDw4O8uCDD/LpT3+aJ554gtdff53f/u3fprS0NK3Px6OiooKWlhY6OjpuHpJZqyYT7xJ85MgRNSeaLnJRk1lcXOTKlSvk5+dTWFiYs1B/LZJRGj2XlpZSCw1mZhDGx8FkQt65M2uRTLyTQUFBgVqIVjzpQqGQKpkWBQHR5cIeDmOrqmL79u3XZbvT03S1txO1Winctg0xX+TJniexWYsQC8OU9c7Sk+fnRxUuvjhYiuDx8Ebz0wilAkMLQwnHZNVbuTJ9hfu337/h76d+z/Jywp/+NLqzZxE7O8FsJnrvvUTuuQdhcRHTtZ4fJCnWdLprF8E/+INY7UaW8ft1LDNBAECvh/U+1Dp9Ti5NXeKg6eCqJJBsSJvL5WJ2bpb/cfl/cNV/ld363Xzs0MeSqqXg5o8y4veR67pPsn0oxP9f/st/4YMf/CDHjx/nG9/4Bi+99BJf/epX8fl8fPzjH89oPw888AAPPPBA2u//h3/4B2pra3n44YcBaGho4NKlS3z961/PmGSU/sZDhw7deJJJ5+lnrZpMIBCgqakJWZZXuASni2xHMso8mu3bt7Nz506amppyVvNJRTLKoDOTycSpU6eSE100ivjsswg//3nM0uVawdq6Zw/SzuSGkukiGo3S1taGx+Ph1KlT9Pf3q4V/SPSmi05NIZw7h254GEGSkEtKkE+fxrhvH9VvvUX997+PMDVFxGhk7vRp/uZAiP7pfrZZKlhyzqC3mjCZ9Dx1SM+H9p6hctrHb1z28s7f+X+QlynnYP1F7lSQq6qIfOxjMRWZIKi2MrLdTuBb30J3/jzC+DhyaSnRu+5C8PkQ33yTsA/efK2E4eGVNSG7XeaXfimSMdFcGL/A77/y+4SlMH9w+A+oEqrW/Ez8kLauSBdj0TEkQeJfh/+VI6YjSCEpYV6LYreiRTLpIxKJpDUV84EHHuDBBx8Ecu/SDTGD4PvvT3zoete73sUjjzxCOBxet0jphpNMOtDpdIRCoaR/S0eenO4+skECkiTR19fH6OhoQloq144CyS5Cp9NJc3PzikFnKz7/5puITz+NXFSE3NAA4TDC1atsGxqCo0dhlR6etaCQvyAICU4GPp8vlqvV69WnYsnnQ/fTn8LgINFt25D1esSZGXj6acRXX8Xwf/8vgiwjFRai9/vZ9uMfM24QKKkpIRBYYjY6j2QSkAWREPAaY/xKyR5qe11ULNmQGo7EDkqWEfr70fX0gDSCtH17bKBYtp9ek4kpzGaib7vmWixJ6H/wA/TPPIPgcuGQRN7vLmfkg/8v/u374s6hwMKCQKbiyogU4ZHmR+hydmHUGfnh1R/yxe1fTPvzkizx7aZvI8kSFfkVTAWmmHJM8b6696m1s3i7FaPRuCkkcysQ2VrR0tK12UPxyPX3hlg6e3k6v7y8nEgkwtzcnOqlmA6U9Gx/f//WIJm10jLJ0mXL5cnxTYTrgU6nIxgMbmgboVCIlpYWAoEAp06dSijc5VIinazRc3h4mL6+vqQjrZdDuHAhtigqF5jJhLx7N+bXXyfa2wu7U5tEJoOiYHM4HBw8eFB92nU4HFy9epXx8XGKi4tVK3bz0BC6sTHkvXsRrkU3ktWK0N2N4bnnYpJmxajTbgeTia+/ssjEn38FubgY41/GOtVdokhUkqgP5DHhG8bu8+F2u8lfWiLPasXw9NPoXngBYWkppvwyGIjeeWcs+siy83Eq6F5+GcP3voecl4e0Zw9ht4/yoQEqzv4VLTu+Ttiq5N9lAoHMF5jzY+c5P3EeidjCfGXuCp0lnRzhSFqf/+nQT2mdbaXQVIhRZ0RA4Dst3+EDuz9AdXU11dXVqt2K0+lkamqKYDCo/uZKT0g2F8etQADZQDoOzDdKvpxs5lCy19eCQjKvvPLK1iCZtbA8lZWpPDkdbJQEFhYWaGpqwm6309jYuCJvnUuJdDzJSJJER0cHs7OznDhxgqJ4S5NVIDidap+GCuX407V0iYOiYNu5cyfbr3XrK/WXyspKqqqqWFpaYnZmBufly0yOjVHsdFI8PY2puhqL1RqrzYgigsGAbm6OaE3NdQdnWUa22ym7Ood9Kkr06HFMR96J/plncJWUIFssFJXkIXd3s7R7NzPFxXRfukTR1BR7f/xjjMXFGA8cQBQEWFhA/+qryPv2ET11KuPvul7oX3oJQB1lIJvNzBTUsXNuhJL+i0weemfC+0WXE92VFpBlpEOHkFPUHCNShMfbHsfld2HRWWJkEJrnmfFn+Kj80TUXDCWKicpRzPrYnJtCcyGDnkH+ve/f+dWGXwUS7VYKCgoYHBykvLwcp9PJ8PCw+ncltbbRnrBbJV22lhTb6/VitVo3JXqJR0VFBVNTUwmvzczMoNfrM55to5DTzMzMzUEy8TWZ9ciT08FGSEBRtKWSBecyXaZsOxgMqrWf06dPq4Ow1oK8cyfi668jV1Zef9HnA72eSAYXV/yo6EOHDqlNgZIkqQuEcm5sRiOFTU2ITU1IS0sEp6eRJieZFASCxcXYzWbyCgooCAaR8/IQg0FkUUSQ5Zj9jN8PBgNRu51IJEL0/e/HMjWF8Y03EKJRxLw8pF27MP32b3No1y6i0Sih738fXSDAjCwTHRrCbDZjtVqxSRJiU9PmkYwsI8zMIC9LiUiiHiQZgzfR8XhX+49xPP8IhnknyDJycTGRX/91Iu9/f1L3BCWKgZiTQSgaIhwN0+pp5a3Jt7i98vaUh/fq6Ku0zrYSioaYWLrulxeIBPhOy3f48L4PrzD+lCQJvV6fML5AiXKGh4fp7OzEZrOpCsFkUynXwi9KJKPUZDYbjY2NPP300wmvvfjii5w4cSLjBwTlt62rq9saJJNOuiwaja5bnpwO1lP4lySJnp4exsfH11S05Tpd5vP5uHDhAkVFRRw8eDCjG0W+6y7kzs6YU3J5eUxeOzODb9cuImn4hkHsxmlvb8ftdqujCpQC/3KCgdjsGPH115FrahC2b8e8axfi88+zu7WV4LZtRNxuwj4fUw4H4m23UX7xIhgM6IqKEIJBxPHxmA3NqdjMFqm4GO9DDzH89NMUeL0Ie/YgHT6M6HZjeOIJjL29WMfHEcNhrDU1RCIRfD4fPp+PgNuNv78fb1+f2ieSbDHLWle7ICDt3Inu9devRzKyjD4SJCrrcFkq8Xpjb80faOPY699EKI0gXxNhCJOTGP7xH5Hq6pCOHk3YdHwUIwgCYSkMxCxj3CE3j7c/zsltJ1N+j8r8Sn5l36+oku94lFhKEpwSFCw/N0pPiBJJB4NBnE7nivEFSpSTzsNirkkm0+m160U6o5ez0Yi5tLREf3+/+u/BwUGam5txOBzU1tbyla98hfHxcR5//HEAPvvZz/J3f/d3PPTQQ3z605/mwoULPPLII3z/+9/PeN/KOXz3u9+9NUhmLQiCgM/no6ura13y5HSQKcksn3u/1kUhiuKq4oWNwu/3MzU1xZ49e9Y12VPeswfpP/5HhJdeQhgZAYMB6T3vYbaigoI0yErJxUPsachkMqnRSzKCQZIQL1+O1VaUuTkWS4w0nnkGs88XU4PZ7dgsFlxHjuCKRLBeuYJudBTRbEbetQv5v/03RLMZkWt+cL29RHftovrAASIGA0JfH8a//VvEiYmYX9jMDOLoKLr8fISjRwkV6GnPm+e+uXwWTp9mIRqlq6uLSCRCfkE+/ZF+3rP/PeRbs/9UGXnwQcSmJsS33gK9Hks4Sq0nxPTut9FfdIKoJ3a+dnT+HEt4Aalmr6pNkKurEXp60L366gqSaRm7RO9cF7IsEZEkJFlCQOBa/MfE4gShaChlz8sexx7+7J4/y+j7rCVhNplMVFZWUllZiSRJLCwsqISzPMpZbUhbrglAyTTcKiRz6dIl7rvvPvXfDz30EACf+MQneOyxx5icnGRkZET9+/bt23nuuef40pe+xN///d9TWVnJ3/7t32YsX45HSUnJ1icZv99Pb28v0WiUu+66a13y5HSQSaQxPz9PU1MThYWFSefeJ0MuajKyLNPb28v8/Dzl5eVq/WNd2zpwIDbLZX4+NlzMakVubl5TOqkU+JUISkndKecyqaV5OAx+f6IFiyzH1GR5eUinT8ee8PPyEKanKX39daSDB2HbNgJ+P3M7d9J/5Ai+sTEcPh92u53JyUlsNhvHjh2L3cCyjO4nP0GcmiK6fz+yKBKtrkYfCCC2tCDp9Zwrm+OCYYryQ29n3zvfyb5r0ZfX6+Wfmv+J73R+h6GxIe7ddi/FxcUEg8FVnQcyhVRfj1xYiK6jA4JBjLKMLc+O5T8co/z9IhCLQOzjc1jGdQjLLzG9HmFu7vq/fT70//7vHHztJf67zsoPKmsYLNbx4JFfZWfRLqanpzHIBj5+5uMYdNkXOGSi/BJFkcLCQgoLC9m5cyehUEgdRd3S0gKQEOUo53wzxjtD6jkv2drPZsySuffee1Pev4899tiK1+655x71gXG9UKLayclJLl68uDVIZrWLU5HgFhUV4ff7c0YwkD4JjI+P09nZqRa1M7mxskky8U7OFRUVaddfUkIQoLAw7p+p05jT09O0traqtSi4XuCHFE+EJlPMOPLKFeSSEhgfh6WlWBRlt8dSdnY7RCKIk5MIY2OwZw9yQwPmmRkqBYHynTtZcjgYHR1laGhIXYD6+/spKSmhSKfD2N2NXFGBzmCIiQUEIVZ3uXiR0RIjb1VIOIuqOdtQzQ6rGeO176wz6Xhh5gWcUSdNNPFrNb/Goidmqa80jypP3aumea5JpYX5eeSamth3ioP+2WcRp6eJvOtdIEkEIxECg4MUn30O87vvVt+vP7AL3asvIUWj12XW0SiEQkh796r7MnzrW+h/9jMKCgqoKrITXRyk2C8TLrrKe0/+DkO6oVivQw4IJnYI6ycAo9GYYJ2/sLCAy+VifHycrq4u1e0hlwa2QMKDUa73cyuPXhYEgdHRUX7nd36Hn/70p1uDZJYjvoDc0NCAw+Hgtddey6nT61rpsnhHgaNHj2Y8ljibJOP1erly5QoWi4XGxkYGBgZyIipYzSAzXj6uzMJZrcC/GqTGRoQLF9A//HBs9kokApEIcl3ddddllwtmZmL/Li6G/Hzk/HyE3l6E9nYWjhxhfHycvXv3sm3bNlwuF3Nzc3R0dCAtLXFifh6TwYDR4UCv08WOSa9HKCnhzXfsw23MZ2/BdvoWRumY6eBAyQEEQeCZ/me46rnKjsId9E210/3YV3lfn8iczcbcXXch19erC2CyNI8wOYnh7/8+FqUEAsh2O9F3vIPwb/xGbFZONIr+/HnkoiJQjCmDQQKlpQhOJ2JbG9FrJBN929uQnn8esbf3uqJsdha5vp7oO94R+536+9G98QZSZSVSYQEv5DURMeSxZzZC98B5mqeaKJALb4qplYIgqEPalPEFSpQjyzKXL19OGNKWlYera0j32t0o0iGZXJQENgPKGv23f/u39Pb28kd/9Edbj2QUefLi4qIqTw6FQuoilqtQNhXJBINBmpubVVv8G+koMDs7S0tLS8IkTUEQcvKUl8y7TJIk2tvbcTqd6u+TqsC/GmSHI7YIz8/H/Lzy8hC8XoTRUcTXXkO6+26EpSUElwtp166ESEDOz2eurY0uk4nDhw+rhL/c+yzS3Iz+hReYkGX0VitWs5n86Wmmqwu4YnZTnV+NzWzD4DVwceoi+0v3E4gE+KeOf0Iv6LEthvBMT/G4+wUe6K6laH6JgjfegD/6I7Y/+KCa5lFGIwuCQHFBAXv+8R8xdXUh1daC1YrgcqH/4Q+R7XYivxqT/xKNJgwrA64rxeJ6wuSyMoJ/9EcYnngC3bU0RvQd7yD8sY8hX2uSFcfGYsPdamtp1Ttp0c9RE83Hao4g+T281PsMH6j8DxmPgs4EuXoANBgMVFRUUFZWxtTUFLfddhsLCwtMTU1dd+6OGxC2kShkM4r+sLZM2uv1bij1fSOhXAcvvfQSn//85/lP/+k/bQ2SUS7OhYUFmpubsVqtCR3iCrGs9QSwEaxGAh6Ph6amJhwOBwcOHEir/pIMG41kZFlmaGiI/v5+Dhw4QGWc3DgXc19gZSSjSKQV+x6z2bxqgV8YHUU8exaxqwu5oACpsRHp9GmmnXqCQQH7j56jZGyS8I4GMBgRdQIGvYTY0YHY0hJLo83Pxz574oTaKCnLMnNDQ8yWlXHy5MmkPmyKzT2/9VsYFhexdXYSWlggHAzislr53m4Lg65xDhtLiBqjVBdUMzA/wMDCAN1z3QwvDFNprUC82k1FUKSzROYn+428bbQa0/Q0+m9+k+i992K0WtU0j1LM9p0/j9zaypTdji4YxCQImAsLMQaD6F98MSY7NpuJHjuG/plnkMvKYjY+gHF+HtlmQ9q3L+H7yPX1hL76VVhYiI1vXtYXJufnxxR2oSAv5I8QFWTyZQOEA9To7HTO97PX2sXBooNZvkKuY7OK8na7HYfDQX19PeFwGLfbjdPpVF2Mi4qK1Cgn0/EFm9Ejo+xnrWbMmzldBrFU+qFrwwa3BMnA9VqH4vW1XA4J5DQnqxT+45/IxsbG6OrqYteuXetSbS3f/npJJhqN0tHRkRA9ZGvbqRBfk1lcXOTy5csUFhZy2223pSzwC0ND6B9+GGF0FAoKEIaHEZubcbeN8MVXP8nCosivDU/zS04Y98UWAp0e9u4BY1kZyDLRX/91ZFGMuRrPzsaiAEFgsqUFIhFK7vsAXq9NlfoqMJlA7T8tLyf8//6/iG+9hX58HL3NhnN/DRM9j2MYGmCg998ICwJyWTneAiMv9L7Aa5OvEZbC+L0eAlEvmPUExTCPlU1wx1AtupISDJOTSC0tSCdPXjf2vFbMLrZaMZrNWOrqCAaDBAIBlhYXMQQCWCYmmB8aomDXLoT3vhexszNGwnl5GLxeDH4/kWuD25JCUeItg3ToENKOHXRONNFa4sRHlHZpCsEQRKqsYT7o5vzMeW5z3LaxCyIFcj20TLm+4/dhMBgSolev14vT6WRmZoa+vj4sFotKOOmML9isPpy1hq/dqD6ZbMLn83Hu3LmYNP9GHwzA8PAw3d3dq8qTFUfP9UzHTBfKj650lHd1dTE1NcWxY8cy7nZNhvX2ySj+X4AaPSTbdi5JZmZmhpaWFvUBABIL/MvH7up+8hOEsTHkAweup4Tm5jC++jLWmbcRKqrHX1SBOC1g1gUJySaikZg5MV4v8r59SHfcEdtPTQ26s2cJX73K1NgYYkkJll/6OF9/9hDziT2LQOwh/6GHwteJxmZDUvzCAP1IF6cvjSOPL4IgEPBHCfTNMrHzEF3OBYZCs+h0+XjlMFYREGRKwkZcBFnMFygPGRFEEfR6NYKDOJItLwerFZ3Ph9Vux3ptVozU34/X4aB/bg7f5CSFhYWU/+ZvUt7ejqW/n4DRyFB5Oft+7deSNlimhMlE6HOfo+gfvsb7R1sgKiGbTEh7dhO9821gNLI4u5jzUQI3Ul4cP76grq5OHV/gdDrp7u4mHA6viHKWn49cZkri9wGpFWw+ny/lSPatDOWcbt++nSeeeIJ/+7d/2xokU1lZSVFRUcpax3qmY2YC5Uf3+Xy0t7cjSdK66y+rbT9TIlBSdSUlJSnNP3NJMh6Ph7GxsfQL/OEwQnt7LN0VvyAUFyMOTlMVGCBoqadj9/uZm3icioUBnMYKgpIR/eQsiCLRX/7l65+rqGDune+k++xZKs6cof7kSaYXrMw/GRvLYrFcTxP6/QKL7ghySxuixRebRLnM3LPi5Tf5SHMYed89eLwGnntZR93SCFVClFdKHyKq/29EpRCS0cejlo9SNjeEt6QEmzUfh9GKMD2MtHMnhmPH0F27JhcWJCIRkGUJindS1HAUy4XXkEtL0dmtiC4XOiDv136NO+68E7/fj9PpZNbppLe2FuOuXeTl5bG0tERUllnPMifv3EnJn/wNH2xvR1hcRKqpQd6xQyWsLrnrpij8r7X9dPeRbHyB0+lkbm6O/v5+TCaTKtgoKipS78/NiGQgNckotjI3I5Tf5y//8i9xu90Eg8GtQTLpOLjmcnIlXH9CunjxIiUlJRw4cCCrTzWZEoGSPty9ezd1dXWbPsFSkiTm5ubw+Xzccccd6Rf4RRGMRoTFReTEDYIAESFWW/EbC3ni2F/xodY/ptzTQ14kgpRfSPRjHyb667+ufmxycpLOzk723nHHdaPPhdj/WCwy8Q98xZ5+jl/8Bo6rPegJQWEh0Xe/m+iv/VrMiy0cRnzzzZhSzWAgHIZQEGZN1eyJdHLE2MFl+334/fkEgw7m3vtZ6v+/r1LUO4QMhE0mIpWVeD/9afL0+mtOCyLPPqtjYUFGlmPnzVD4JQ6XOtgx/jqVvlnEsmIi73sf0rXZHhaLJcFk0u12Mz4+TigU4rXXXlv/WGSTCen48aR/ynU6a60U0EaxEQKIH19QW1urnnOXy0VfXx+BQCBh3lMuz5VS91lt+0ra72aNZBTEjwzYEiSTDnJNMuPj4wBUV1eze/furF9k6ZKMYlUzMTGRtlQ625FMKBSiqamJcDhMWVkZBQUFqTv446HTIZ0+je6f/zm2mFsssZ6R4WEiJRVcDd6mXnSjxUd4+J4nqRi/TNS9yGe/2UD1kVhqUpGxj4yMcPjwYQShhMlJgYgUoX/chddbicEQax2xWkEf9nP3Ww9T5Oolsr8Og8OMMDuL7l/+BYqLib7nPbGiuSyvUHXpDAJ6GfIt0jXSkhE90zh+/jTG0lJ0Fgv4fERNJjx33km71Urk7NlrdihlOJ3bsNn019TIOmS5iM6SL9Ex9+v8yv1OrNuLkS2W2KRMxfjz2v/qdDr1N/b7/dx2222xKOfaWGSLxZIwFnm9C+1m1Exyvf1sRRnKOVfOuxLlTExM4PV6OX/+fEKUs16xTzKkk5LLVjPmVsFNRTK5qMlIkkRnZyczMzOIosi2bdtycrOkU5MJh8M0NzerowLSVZhkk2SUaZ4FBQU4HA5CExPw5JOIg4MIRUUId94Ju3al3Eb0Xe9CGBhAbG6OSXVlGbmkhIUHPs7itxyY/fExjkib+SSBIgHJEQJk9Tdxu92cPHmSYDCfP/1TA/PzAtPWc0wYXiXY8kUsFGGzwdvuCXF88GkqZtsZse2n0mIAMTapUvD5EF98kegDD4DRiHTsGLrnnouNR76WmCoJT7JoKGI4rwFZBp/Py9uXnqXONY54/DDStSdcYW6OiqtXcZSVsVhdzdzcHIODUwwO+ikuFigvz8dms5OXF3PQddtKEXc6MNqlBJKOvw4U0YDy/+OfuJW6wtzcXIJ6qqSkhOLi4ozdBzZrvHOutp+rVJbValXTU263m6qqKlwuFwMDA/j9fnWaa3FxsTqkbb1IR8F2K6jL4rElSCYb0zHXg+UTNd94442cRUtKzne1m3FpaYkrV66Ql5eXdFRAKmSLZJQenLq6Onbt2sXYa69R/td/jc7lilkiyjLy008T+cIXkO69d/UN2e1EvvQlxNZWhNFRZKsV+fBhJHEb9u/LLCwIBALLPyJjMsnqTB5Jkrj99tsxmUwsLgrMzwvorYssOF4hIPQRrjiPcfw9NIy9xCcf+x9UzXdhjPiwWcYR3XdA3rWIKD8/Nu0zFAKzGem970Xs6UHo7MQcsrA7GEEymXm19D8wa9jG4sICkiTSGL2IbCuIWewoKC5GmJ5G196OraEBm81GUdF2ui772TX4DOVvnCcUDjNafYjJne8gpCsnHDYiigZ1YVGuAYVwZFkmEokQDofVdKQS5SyvKywtLeF0OpmcnKSnp4e8vLyERtBUi9dmRDK5LvxvVie+ck53796N3+9Xh7QNDQ0l/L2oqChjd+K1IploNHpD58nkAluCZNJBttNlbrebpqamhImauUzJKTdIspt9ZmaG1tZWamtr15Wqy0YPjjLk7ODBg6q9R+FTT6EbHMS9axfmvDxMRiO6oSEM/+t/EX39dQSvF2nvXqR7710puzWZkE6ehJMn1ZfKgb/5mxDBYOL3E72LWGeHscxIXOxxYysoSOokvWB7i6B5FFuggsHCV9nebuVP5j5DvrCETzChkwIU+CYRf/4S8gfeF2vwdLmQDx2KaZsBubKS8O//Prrz5/H8rIezXSV02m6nT3cY37gXnc6AKFqQBRFWK3PF/T6C38fdP/tT6sfOI+gFBFmgYaqN/qle/nXvp7n6zWeIzg1jLSzE8K53YW5sRLx2rUFs8VxcXGRwcBCHw5E0ylHSazabDZvNpvaIKItfW1sbsiyryqlkdjebEWnczCS22j4sFkvC+AKPx4PL5WJwcJCOjg61byfd8QXpdPsDN31NJh43FclkI10myzKjo6P09PSwd+9eaq4Nw4Lc2/FD4oUcb5+jLO7rwUYK/0pqanZ2lpMnT1JYWBh7op6dpWBggHBdHVGDgYWFBSKRCAULC+QPD8dUZLW16Lq7ES9dIvLFLyKvkUYDZfjmtWOVZcSf/ATds88SmZzEvbTEoYYG7F/60opxyGFhiXHDWYwUYBaqMBR2cKb6a+Q7l/DnlyCKAhF/GGPIi867gPTKK1BailxcTPTBBxNlwaWlLLzj/fxgSMejFj3+RZmwK4IoWtHrdYgitBee4b3Bf4RgUCUo5uaQ8/NjZp3XYHztLNXD55l31KiD3/RBLzuG3+I/T/dT4++FaJioJBH+wQ8YefBBvJ/6FCWlpTgcDrxeL01NTWzbto1d186fIq5IJpFWohyDwUB5ebk6s2dxcZG5uTm1t8tutyfMbrmZ01mwNWbJiKKoWtrs2rWLQCCgEv3IyAiiKKruA6uNL0inERPQIplsY7PSZdFoVF1Qjx8/jsPhSPh7LqdXxrsWKN+lra0Nj8ejzl9ZL9YbySgF/mg0qvbgKCkbORJBkGWMZjPGggIoKCCysIDQ04MkCEybTER1OiyVlRSMjCA+8wzRL34xox4P8cIF9E88gT8aZfqapNQ+Oor0jW8Q/sM/hLi8tMvyFou6UbbJDUQQMYZLmN72M+Z6Rex6AZ0IUYMZIeyNjRKYnY31PNXWIl3r7YlHJBITDPyH/+BmenqakpISCgoKCYUiBIPwvre9A9Ozl2LuA6KIEI2C2Uzkgx9EVowpAXPrJcI6gaDOCtemd4fIw7E4R9FsH/LeXehsVkRZxjA3x66XX2bgnnvo83jw+/3IskxJSQnV1dXqNRIf5SiEo6TYgBXiAUEQsNvt2O12duzYkWB3o8xuUYgpHA5veEJlMtxMhf/VkGmfjNlsTjq+YGRkhM7OzoQoR/G1SyeSMRqNOfmNbhS2BMmkg42msvx+P01NTQiCsOrUyFymy5QbUJIk9Vh0Op06f2UjWEEygQDC8HCshlJXd92EMQ5KDchms3HbbbepBKt8f6G0FLmhAfGNN5ALC0EU0ft8iD4fFBdTun8/fsDv8zEjigivvML0mTM4amooLi5O6yYRX34Z78ICs8XFlJWWYrFYkIqLEfr6EJubkc6cAcAbXmLa+gqETfgiIcKREATsjNgMXK5e5L5ZEOQoptASUVGPTgS5oQHp9ttj5pHPPEP0U59asX+3240sj3PnndUUXnOf9nrB44Haw0WEj34V3WuvIXR1xebdnDwZkwjHd52bRRwOGfvOxEjS1O9BMInItmv9DoIAJSXohofZMTiI/vbb6ezsZNu2bYRCIS5cuIDZbKa0tDTmIl1UlCAKUIhGIZ1U4oF4V2NlQmVXVxcul4tz586pUU5JScmGC9kKNiOSybVxpSRJ617cl48vCAaDapQzNjYGoD7UpvoeS0tLWftNtgq2DMmkMx0zGAyua9sul4vm5mbKysrYv3//qjdDrklGFEXcbjfd3d1rHksmiCcZoaUF8d/+DWFiIkYy27Yhvf/9yCdOqO9XCvxKDQhISM0oaZnIxz+OYXgYobs7RlTX5pdIDQ2EdXkIEljz8tDlhQkJIksBO+NtowhCO4WFheqCmaCU8ftj81zm51l8800CgsC2iorrqQWDISZ5drvVj0wGBjCYg4RCetxMEYmCXy8wHd1PW/lrPDDiQdIbEKJhBFkGkwHpwIFYt2ZxMeLFi0Q//vHYv4ktiH19/czO6jh4cAeFhas0vtlssVTbgw+ueu6l22/H8LOfIYaWQElxLCwgIqvpMxXXFg739DQ9PT0cPXpUdZOIRqNqs2BHRweRSASHw6FKbZWHotWiHCWVvDzKUSZUWiwWKioqKCoqUqOc4eFhdX77RuW6mxHJ3Og5L5nAZDKtGF+giDaCwSBvvfVWQpSjrAMKydxK2DIksxbWQwDxBe19+/ZRU1OT8v25rMkoaG9vZ+/evdTW1mbtplRJZmwM8bvfRVhYQK6vB0GIGVX+0z8RdTiQt29nZGSE3t5e1WRTTY8lmQEjHzhA+M//HN1LLyH09iLb7YgdHYQiAlcuCwRDAvpIgHKPi6Zdv8LFZw9SUCDzyU8uEQ7PJvR6lJSUsM3lovj//l8YHCSwsIDZ5aIgLw85rsZBMBgzjIyzFzqz+yD/+KlSQqHYMY4Mw//8n0aiURmj67t4o/+IPehClML4dflIJ8+Qd23xlkUx1gh67fspY6JdrgD19SexWjeWlpDuuYfom2+ie+WVmFwbQK8nevQoYn9/7DVl4fL5CEsSIyUlHD9+PCFFqtPpVrhIz83NMTk5SXd3N3l5eZSUlFBaWkpBQcGKKGctibRSkzGbzSsK2U6nU5XrFhYWqqRjtVrTvkZvhUgmVwaZ8eMLJEkiFArhcDgSRBtFRUW89dZbWCyWrEUy3/jGN/ja177G5OQkBw4c4OGHH+auu+5K+t6zZ88mTNFU0NXVxb5lhq2Z4qYhmUxrMvGmkidOnFBnjadCrmoyyiwaSZJoaGigbjUDxHVCiQLF5ubYrJGDB9WnZnnnToS2NrhyhU6/n+np6ViB325HeOopxH/7N/STk8gNDUQ/+lGkO+5gbg5CoWsXuaEeHvw0PAhGo0zZ4FtEv/H/UdzZgSAK6AwCU/XHGTv+XsyyjGceHun8FnfuOsI9x+6hqUlibGyBka5Jbn/8rwm5xnE5qtDZSqm3FiL2dSCfP4907FhMTDA+jnTgANI1B1cAURA5UHNdFGGTIV82EZXhmZKvcL7gP3LG8ywPzH4Xt7GMI3XXnAGiUcTZWaL33w8WizoyWxAEjh49yvCwYYWUevm/14TJROT3fg/prrsQr3nMScePI9fUIH76/0EcHEQ2WyAaIRIIMN1wiPz3f4ZodPUaXLySLH6mytzcHM3NzQBquktRkqWSSIfDYcLhMNFoNEEiHV/I3r17t9qU6HQ6uXr1aoL1yloGk7dCTWYzoqVoNIrRaKSiokK1aVpcXGRiYoLvfOc7dHZ2kp+fz5e//GUeeOABzpw5s/pgvBT4wQ9+wO/8zu/wjW98gzNnzvCtb32LBx54gM7OTmpra1f9XE9PT8LDTzbm2mwZkkknXZYuySg1D1EUVzWV3Og+0oWysIVCIUwmU06kiWok43YjGAzIy43/jEZGL1/Gs20bjY2NWCwWxL/7O3Tf/jZyNApWK+JPf4ru4kXm/uv/5I9ffjcez8oFo7BQ5g//8CTCf6nm9f/ZhsPkJVxexXT1MfQGM9YlGAk3cX7k3+leuoTFeQcf+qVCgsE8zkT72B1a5BJHiIwYEQRozyvg/tI58pxOGB9Hl5eHdNddRD760aR1JAUFBXD0qITdLl/TBpQQ5RN0NNk51Pp98oY6EWwm8PuR6uuJvu99+Hw+rly5gt1u58CBAwQCOmy22Ly05cRis8VcaOKRQLzLYDQaKbn33oTeIbcbHjv0GAdC32XP5Kt4oyJvbXsH7XWfIvL3dgoL4YtfjDPyTAFlpoqyKM3PzzM3N8fw8LAqo1XSajabTXUSAIhEInR3d8eUgQUFq0qk4XpTYk1NjWq9stxgcjW7m1tBXbYZVv8KySiIF22cP3+eb33rWzz66KPMzs7ya7/2aywtLdHd3U1VVVVG+/nf//t/81u/9Vt86lot8uGHH+aFF17gm9/8Jn/+53++6ufKysrU+mS2sGVIZi2kSwDKyOaKigoaGhoyumiyTTJK97zdbufYsWNcuHAhJ5GSkg6RKyogHE5I0QSDQRbGxogcPMgdd9wRi9bGxtA/8QSyyYRcVoYAyKWlCMPDWB/7RxZs92OxisSXFHw+8HgEQiEBfdk2OrbX4nDIxCstZWR6TN/DF16i393PWX5KMPhh9HoZh+BBF5aQdUZEBGQJJNmEt6iOkODlwqn3oysppObkDkoMBorWUOGYzbESSPz+h45+gFlLHbX7zqEPu5H37CF61114zGaaL16ksrJS7UPKy4P3vS9KMlW8Xp8gbGNuDv7wDw1JiRdi5PvHfxwm3gEoFIJhqYaJO77MD+d/HYNBT3l5GQWCiM8nXzuXqX7V5BAEQS0wKzJaJcoZGhpCr9erhFNQUEB3dzder5fbb78do9GYkB5NVcuJt17Zs2ePGuUoNvrKsLDi4mLV1+5WiGS2goKtrq6ORx99FFmWaWtrS5gdlQ5CoRCXL1/m93//9xNev//++zl//nzKzx49epRAIMD+/fv56le/mjSFliluGpLR6/Up+2Tih3o1NDRcN1PMABsRFyzH9PQ0ra2tCfNxcpWOU26MyKFDiLW1CF1dyFVV+Px+Fjo7MdbVUf+Rj8R8xSQJoaUFYX4eqb4edVkQBOTiYgxjQzjqx5FLalgu1ff7Ux/HJE1M6d9kr7WaJcnJK3P/gqx/L6JoYELYRkhnwRpdxCdZuE1uY9dCP9bLAUZt+3nK9i6CDgef2TXA7FwX4XAYh8OhigfSUuAJAqPlx1n65CEs16L8mZkZ2q9cYdeuXSvSBOnWV0MhAY9HwGKRWV7Ljyff5d2bkUgYr3eC4mITZWVlCYvwWucyXSyvsSg2NH19ffh8PvR6PXV1dQlpsuXiASW9lqoRdLndjaKc6ujoULfhcrnIy8vbsFoyGXIdKcHmpcvWkjArhX9BENTBX5lgbm6OaDRKedw0WYDy8nKmpqaSfmbbtm18+9vf5vjx4wSDQb773e/y9re/nbNnz3L33XdnfAzx2DIksxEX5uvFXJfaULgeZKPwL8syAwMDDA4Oqvb469r+9DTipUtw9SrY7cjHjiXUWpYfNxBrPPyt30J85hmWrlzB4/FgP32a/GujepXFQGexxGbdRyKJtimRCLJOR0g0k045/FrfWGzfyFwJf5+IEMBmrKHAYKZ9vI9I3fPoh97DQN5RLoVPc2foBRropOCalbIgwc75Jv73udN8+sgbzM/vpaxsD36/D7fbRSg0TldXFzabTS18y7Jtxf4BnM7YazMzsT6Y8fFxhoaGuO22Q9TWrm00uhasVlYQLyQnjIWFBWZmglRV2SgvTyMnlgUozYA2mw23260q/JQOdZPJpEYnir39ahLpVI2ger1+hUjhypUral+OYndTUlKi9odsFLdSumyzBpYtP++pos29e/eyN67/q7GxkdHRUb7+9a/fOiSzFlYjGZ/PR1NTE3q9ntOnT2/oKWqjkUYkEqGtrY2FhQVOnTq1ov6SdtPk2Bi6Rx6B4WHIz0cIBpHfeismRX7nO1e8PUFlVFtL53334aqq4uDBg+Tv2oXM9S5yURSRb78duaoq5itWWxtzJQ6FEFwuAne9mwVvGcWreqqA2TXJfYM/o3i4iZAhj/6qu3l1u4Nx25sU6CoQRdAJOiLBKJGG76Ofuh9EK/+Q9xDb/Fc5Jb1B/KUuIlEUnOLtl77O5z//v6/VQ0zk5RXygx9UceRIiLm5ObUOEQiYCQYb8Hrz8fnMiKKI3w8XL4rIMszPGwgEFvD7LTgcZ3jzTQO///thsjB7Li3EbIJ6sdtP4nBsrhzV7/er/U8HDx5EFEXq6+uJRqO4XC5mZ2fp7OwkHA6rRJCORDpVI6hSB2poaMBsNqtRTmtrK7Isq2m11brg04EkSVl1Q15tHzdaJh0fyawXJSUl6HS6FVHLzMzMiugmFU6dOsUTTzyxoWOBm4xkIpFIAhvPzc3R0tLCtm3b2Ldv34afQjZSk1HIzmAw0NjYmPRmSpdkxJdfRhgZiUUuohhb7icnY27CR45ccxC+DuV8BINBWlpaCAaDHH/3u7FYLAk5eNWi32Ih/NWvYvjqV2NNm0qj6P79eD71JfiblVGC8m/d1DjF3/kTHpztJ1pqh3CYM87LdB6JYiyYxmT00T8fxe/3E0YERx/B6pcxjL+XoJRPgexBRkBYRmJ6ovxG9DEG5Nt50/pOZiIOvF6BhQUBo9FIZWUlglCF1RprLrznngVmZ0euLZZ5FBc7mJ2tJC9PIBqdxWgMU1NTxuKigclJgdHRxBqI0SjnhHTGxsbo7e1l797beO21fFY3QMs+lIiirKyMvXv3Jk4r1elWmG0mk0grtZxMG0GV62u5ckrpDxkdHVUj0ni7m3SjnFupJpNqHz6fLyH7sR4YjUaOHz/OSy+9xAc/+EH19Zdeeon3v//9aW9HsTvaKLYMyax1sSlPMcqFMDg4yMDAAPv3789YebEa1ksyithgLbJLa/uBAEJ3N3J5eeLck/JyhK4uhKGhhB4SuH7urly5Qn5+PqdOnUIfNx442QwYqbHx/2/vuuObLNf2laR7DzqhpbS0FFq6GUUQAdlCi3oUJyB6GOJBUEGcfB4V3LhAUVQQEQ8UEGWvAgIinZQuuls6k+40+83z/RGel6RN27TNKJDr9+P7jsmbvG/SN8/93Pd93dcF6S+/gHfqFDgCAciQIWAmTwavVoF72o5DXiVGtX0QKpzC2CDk4kLgdO6wav4jYgS4dEfW1IQJBbnwTZiDNmdnVFVVwcXHBWKxB348bQHSGACJhAOlEpARSxB07F8AAAEHMyX7EWxRjG0OK8GX3drRVVdz8MwzVmhtBQAbqOQ2VfeDtbUM//53HpqbeSBEBgcHwM9vAAALZGZy0drKwQcfWGoQ1pydCd54o2fZTfvAq/4YLZOWl5cjOjoaMplrt6/RJ6iLqr+/PwIDA7s1ueuMIp2ZmclmHx4eHhpim51lORKJhKVHq/d+1OdDAgMDIZVKWYp0eXm5hqKxm5tbl5lKfwgA+jqHoTMZAFi9ejWeeuopxMXFIT4+Hlu3bkV5eTmWLl0KAFi3bh0qKyuxY8cOACr2WUBAAMLCwiCTybBz504kJSUhKSmpz9fSb4JMd6B/GKlUiuvXr6OpqQmjR4+Gs7Oz3s7Rm55MeXk58vPzdR727DaT4XJV/+RyzccpvVvLDVpfXw9AJVsRfnOwkS4C9LxaFx1vbzBPPHHr1FeuwOfzz/G8oBpKJQHh2EI88l40PfsiiLUNrKwInN5IUcnM8HhoaqSX6YKpJc7wvxKCi4MiMH6EHyZM8AI/uw6ujVngcq6i0bsamVajcKToQTwk2tfhUpTgIJMXjWLrUASJsjHcIh18jGefF4tVdGMbGwJ1RrpEwoFEYgNf30DweCJYWlrBxoaLuro6iMUWaGnxBmABZ2cCBwcu+17Nzdqb9dpgZUXg4qJihGnrvzg7K1FWdh0KRQ1GjRoFBwcHNDYCLi7o9DUuLprtsL5AIBDg6tWrCA4O7vYe1Ib2FOmWlhbw+fwuKdJ0MZZKpcjOzoaLiwtsbW27pEhbW1traH01Nzejvr6eVTTuyrfF0EGGsu36Q+NfHz2ZRx99FPX19XjnnXdQXV2N8PBwHD58mJ3Rq66uRnl5OXu8TCbDyy+/jMrKStja2iIsLAyHDh3CrC7ULnTFbRNk6A2WkpICGxsbjBs3rtf13c7Qk56MUqlEbm4uamtrdR721CnIWFmBREWBc/iwysa4qUk19yKRgHh5dVA6pkGOx+MhICAAADqd4O8SDQ2w+OwzlZtkaJDqh9DcDOuLR+EQEQDm0UdVx9nYAAoFmhqBI0d5KgowIQiRAd8VuOOY/Tg4OnLwx4Y0BB3/Dq8NqILSTbVYNHsG4b9ey3EpYzbiBYegVCub1cELf1vcA2tiDaJUwk1aDUZLXLSx6cgKa2tjkJOTCxubYfDxcYSjo2rmSiCQ3hQllKGxsRaAJezs7GBpaQ+xWPdJ/wEDgP/7P7nWORmGYVBUlAOgBaNHj2Z7G66uqjmYzmjKVlbQaUamO1B76rCwsD6XWQDN6fShQ4dCKpWy/TDqp0IDjr29Pa5evQp7e3uMHDkSALR65dD31SZ34+rqiqFDh2r4tpSUlMDS0pIdNnV1dTX4sCcNjqbOZPTZ+F++fDmWL1+u9bmffvpJ47/XrFmDNWvW6OW87dFvgkx3N5Dgpm6Wi4sL29DUN3Qtl0mlUmRkZLDqxbr6sOuaKSknTQLvyBFw9+5VNf05HMDJCcySJcDNaVx1m+a4uDikp6ejtlaJxkYChiEa2YuVFWnfxul4bVeugFNTAxIcfCtbcnEBmpvBO3ECzCOPABwOlPfeC4utW8Fw26BQOIHLIfBRlqHFwhllA0bBhnAgb5PDfv+v4LTVwSp6hCozUyhgm5eHDXF7UP/5LtQcSYLL95+BqeEjuSUOaZwYMEorSJqBNgaoFNujiQcUFnLg7Q3U1nIgl3MgkxGNICOTySESSeHr6wtnZ2dwOKqgRSVUrK15IERF0eRyRRCLRWhoaIZIZIuiogZYW7voZGusmoHRzHrooK29PQdRUaM6iCvqI4h0hfLychQWFiIqKorVQNM3rK2tO8jQUIq0WCyGlZUVnJ2dIRaLYWdn1yuKNKDp28IwDCt3U1BQAIlEAh6PB0tLS7i6urIulvqEetZvKNDA253U/50k8w/0oyDTGdQ9VywsLODv72+wG0GXINPS0oK0tDS4uLiw6sW6QtfGP6esDJzWVpCQEBBra9V0oFQK7uXLUE6dCrm/PzIzMyGRSBAfHw87Ozu0tFhj/Xp7iEQW4HA0FztnZ+D992VdBhrOTbOk9uU4YmOjqlMxjEqTa/p0cHJyYH3iMobJysHlMhBauOCI5yK0uAyBjQjwEhXBproMZORgtEm4YBQAYAGu80Dw8nLBbWqA9OFHQcaGoGn9FlQku0BBLMElDPyVZaiHO1LlEZAxHGzdaoGkJJU6ckUFBwIBFyNHKmFtrWJSCYUS2No6wctLxSpU73e0tamozDyeqiTk4KDaodvZKVFdLYdCUYesrCwolUqtPYiuQFUl7O3ttRqsGRK0/3Pjxg3ExMTofUK7M1AZGisrK9TU1MDX1xeOjo4QCAQoKirSmSKtnu0A2gdBadkMuEWqEQqFuHz5MmxsbHSWu9EVxgoyAIzSk+lP6NdBhlKCm5ubMWbMGDZ7MBS6yzSqq6tx7do1BAYGdttc1QYejwd5+16LFnAuX1bNrNwsQQBQKRNfuwb5xYv4u6oKdnZ2Gg1+wBqVlW1wdJTBzc0KlpaqhVIk4qC5Gd32H5RDhqhqOK2tKl0Ves6GBjCTJt3SWbG3h2LNGlz3+Bu7civAtbVCmWc8bljf8myxgAIcpRJtMh5On+NBcnO+1U5uCUc58MPrXDBeVvhgYxQsH0qET+4JDCdZ4PE4aLbxQLLnI5AWDkIA04TnRD8ipvwEJMQa2+WJ2MeZD6HQAi0tYkilElhZOUGhsIClpQzOzgTNzbd6ICKRKjZaWRGIxbfaWhIJFzY2Nhg2bBi8vYM79CCcnZ01FKTb/52pkoOnpydCQ0ONKstOCEFeXh74fD7i4uKMvuulmyw/Pz/2N+Dv789SpAUCAXJzc1kRSPo9dkaRbp/lqM/jqMvdWFpaYsiQIXB1de0gd6PuCKqrhFR70Ka/MUpynQUZQgja2truKFdMoB8FmfZ/3La2NqSlpWn0XwwpxQ/c6sm0H1pSScMXoLy8HJGRkfD09OzV++ucyTQ23nJjZB/kQMYwKLl6FR6jRrGDU/QHGh4eDjs7C1hZtaGtrQkcDkelUca1AyG3fnh1dehgfwwAcIuCV0Q8HE//AUilKpMuACQgAEw72mN9aysuceyx134BXF15sGt3qeW8IZC5ecGqsgoSaRAsLQBLCyV8ZZWodB+OFlsvtNYAJaU8cIY/gF8Cx2KYRRFsHXho9g5BvcIVHiUN+LlxFkKbcsAhSgAcjGROYrrsT7xa+z2UPA7s7FygUHDh6Aj4+gKvvqrZA2lsBD7/3BJCIQciEUcjy3F2JrCyIh16EBKJBAKBAHw+X2N37uHhAVdXVzQ1NSEzMxODBw/GkCFDjBpglEolrl27htbWVowaNUrnMq2+0NjYiIyMDAQGBnYQeW1PkW5rawOfz+8VRVrbICht/FtYWHQ4T319PWpra3H9+vUOcje6ZibGmvbvLpCJRCJzJmMMUM97Pz8/BAcHszdKd9IyfQWPx2NZJvRGUCgUuHr1KoRCIcaOHdunnaOuPRkSEqLKZtQ0yFobGiBraoJ7bCzcQ0M7SPRbWVnBxsYSrq5WcHBwgVQqhVgsRlNTMxob23DtWgUEAjd8+OEgCIWaf/bWVlVZKVEYihWiP2Hb0qp6wtoSPAsLlcfLTdy4cQP5+fkYPDjyJuW0Y3Yk4jqg/r5E2J/cjsGt2SB2dnCQtEJo54G0gHnIyrJAWxsHb71lCQ4HyM3zxXmeL2xsCEbbKkEI8GTLFgTLcyC1tGWzKK5chmnMEVhN3AqfF5bC0lJ1L9jaAj4+Ha/Dxwd4/33tDfvO5mRsbGwwaNAgDBo0SGN3np2dDblcDqVSCV9fXwwcONCoAUahUCAzMxMKhQKjRo3SO+mlO1AGW0hISLeSTRwOBw4ODnBwcOiSIk2DTncUaYVCAYVCwT6uXl6j5xk8eDDkcjkrq0Plbqgnj5ubW5eD2v2FIq0vdll/Qr8KMuqSLNo8742RyQC3bgaq3GttbY2xY8f2+YetayajnDABnAsXwMnJAfH0RHNjI5iaGtiPGQPLWbM0atrad0YcWFvbwNraBhYWAIejhLNzCyoq+Cgrc4CdHQfOzlawsbGGUGiBzEwLuCnqMIL5Exd4QajhTAAXSnCkXDzYkA2b//0P8rfeQuHNPkB0dDRKS1UrdHsFY9oHyfeZCPFcd/yT+zd8udVodQ9A6eB7UcINgkzGAZeryiY4HIDH46CpCSCEi+Rk1WdZ03YQhBBIFRaw4gEcEMg5FrAAwbiWbFgN0x7gamu1Z2rW1gQ9GHYGoLk7t7OzQ2FhIby9vdHW1obz58/D0dGRLQf1ZLCwp6A22RYWFoiNjTX45Ht71NbW4tq1a71msGmjSAsEApSXl7Plyc4o0nK5HNnZ2ayCeVfkAUtLSw25m9bWVtTX16OysrLDIGh7uZv+MCOjUCggkUjMQcZQkMvlSE9PR2trq1ZJFsDwQYbeZAzDoLm5GZmZmfD19cWwYcP0cgPqTJH28gKzahXw++9oPHMGjFIJlyefhMWDD4Lc/KF1FmBEIs3ei0ikEub08/PDwIF+cHW1gJ2dGDyeGG1tzWhpsQHDeGA4cuCOepRYDAOPx4WS4YJRAmInb9jm5iLnwgU0cjgYPXo07O3tIRAQ2NkRiEQcdqSHYVRzIVwuwYcfWYHHi8H1hlhYWgK2MmDCUAZg6HehoiJbWBDY2hI0NnLZngmAm9Rm1f8lhEBJlCoTNnC0ybcBUAWYF1+0QktLxwOcnAg2bZL1ONAQQlBYWIjKykrExcWxc1l0sJDP52uoH3t4eMDNzU1vpReJRIK0tDSWJmzohbA9KisrkZ+fj4iICL14i6iXJ6lNcWcUaRcXF1y7dg0Mw7DBVb2s1hVFWl1Cf8iQIZDJZCxFOjMzExwOR6OX0x/EMYVCIQCYezKGAperasaGh4d3mjFQaRlDXgMAVFRUoLS0tNdqzl29v65zOKIBA5AWHQ3bmBhEREZCIHSASEigbFFCqSTgcnk3aboE3t6q8o+zM9DcrBpQpJBIVIs53eFLpVzY2trC1tYWbm4ESqUqQjAclXwNj6MAuByAcKFUAkpGjsbmZkgVCoxWM1AKCQH27JFCKLx1rpoaDjZssISdHcGAAapzW1qqnheLgZYWlXiAulmkjQ0QEaFEa6sqeMTFKcHhAKfPzUWoMA9cogDDcMHjcsFTykHAQUXYdPDKbp3X2lr1HUilKhkaGxuiMd2vOjfnZoaju8yLUqlETk4OGhsbMWrUKI1aefvBwsbGRvD5fOTn50MqlbJlGg8Pj143o2lf0t3dHcOHDze673t5eTmKiooQFRXF+tPrG51RpAsLCzVUpGUyGSwsLDr0crpzBKXHqsvdKJVKtLa2stlUbm4ubGxs2McdHBwM8l13F2REN5uG5kzGQLCwsEBYWFiXxmU9dcfsKejAV3l5uc4Dlj2Brj2ZhoYGpKens1lUbS0HS5daoKWFfje3fgBOTgSbN8vg7a2iKav3H/h84I03rFBTA7z6qhWkUtXcCYfDg7U1EBvLQC63AsDBVUSjmjMQ/soylDBDwDAEPKUCihslkMwYh6iJEzvsokNCAPVFu7xcRUxzdVWpFVtYqPolYrFKGYD6sSiVquDC46lea22tCjqEqJSOeTzgN/dluF98ECGKPHAZJcBwoFRycNpqKt7980FwT9zqEzk5EXz99a2Ov61tR7Xknjpe0l6cTCbD6NGju6znU/Vjd3d3thktEAhQU1OD/Px8ODg4sAFHV1Xi5uZmpKenY9CgQaxVhLFAxwYqKioQExOjV1WNrkAp0uoq0p6enuyApi4UaV0HQdtnU8XFxRAIBEhLS+uR3E1PoMu0v42NjVHp8MZAvwkyusCQ5TKpVIr0m/a5I0eO1HuAAXQrl924cQO5ubkYNmwY/P39by5aSrS0qORUVDt01eJMd+iqzIUOXN5a9BUKlSikg4PKB0UsVv3gWls5aGkBzp7lQalUBYBGjiO+wRKs4XyOEco8KJQAo+SgwScItaNHo+jvv9mF0sXFBRwOB9XVmv2PqioVfdjKSjV/aWenykyEQlWAefttBTgc4K23LGFrS8AwgFCoohorlbdcDGxsCDyH8/Cy2y7M4R/A014noLS0wubqeTjj+zjcXS06fAdSaedltJ6C3guWlpaIi4vr0SKj3owOCAiAXC5ny0FpaWngcrnsQunu7q71vWlJR5sHjqFBCMH169dRU1NjEoo0NdyytbVFREQEuFwu64ejjSJNv0vKtOvNIKi1tTWcnJwgk8kwcuRI1nm0uLhYo2fk7u4OOzu7Xgf87kpyQqFQK2X+dsdtF2T0ZSqmjubmZrYsQaeLDYGuymWEEOTn56OyshIxMTHsrlhVf1b1J2xtSQdJFV126HZ2qpIZj6fKGlpa6MzIrYWZEOA8ZwLaHPwwhvkL1pJGVFgE4PF/j8GUB23Z/kNmZubNd/XBxo3DIJFYqqlAAyUlHPB4HNjYAPHxStjZqYgA1tYqBpiNDYGXF0FzM9DQcKuUpppnAcRigsbGZigUcth4+OMPl5cw4+sXQAhw7FkruLr27jvQFZTs4ezsjLCwsD73QCwtLeHj4wMfHx9Wr4vSo7OysuDq6sqSB+zs7Ngm+4gRI/SigNsTEEKQm5uL+vp6jBo1yiCT9V1BKpUiNTUVDg4OHVQ9tFGk1bPFvlKkFQpFB7mb4OBgiMViVtSzuLgYVlZWbJZDsyldYSxxzP6GfhVkOBxOl+UyQ2QyVVVVyM7OxtChQxEQEIALFy7o9xxtbSrp/txcOHC5sPX0BCZM0DiE0lNFIhHGjh0Le3t7ZGYSNDYSKJVAXZ0FGxjkcpXaS29gYwOEhSlx6RIPHA4QHKyETAbk5XHBMKr3z2gcjEyOPwhR/UBLvlIiYoIUPj5e8PLyYj3mMzKaIRDIwOO1wcFBJfFuZWV9szmrWvhbW1UZirpApKcn8NFHMo0MSCAA3nnHEm1tQE1NKwAuHB0HQC7nwsmJwNqaaPSZukN7QUpdXShpiUrdplmfUF/AqKUxncm5fv06LC0tIZfLERQU1CPfD32AzuAIhUKMGjWq132k3kIikSA1NRXOzs4YMWJEl8FdW7bYF4q0UqmETCa7qXN3iyINqORu1CntVO7m+vXrkMlkcHFxYc/V3dxSdww2GmTMmYwJoc+eDC0LVFRUICoqimXO6NUiWSCAxcsvg5ORATAM7JVKhPF44NrYQPnIIwDQgSZtaWmJjAwlHnjAml1YCaGlLlU2MnEi0+tAY219y0HAykr1flyuKpMAVEGBx+OCw1E9LpGALccBtzzmAwJc4ehoBUdHBSwspJBKJRCL28DlukAu50Gp5KkpDYANFoAq0KiX9fz8gM8+a0ZaWg7s7OwwbNgw8HiKm9eroh6Xleny2QicnMjNEqLmc+rn1wY6BxIUFNRh0NBQsLOzg7+/P/z8/FBUVISysjK4ubmhrKwMpaWl7CI5YMCADrpo+gTDMLh69SqkUini4uKMPoMjFouRkpLSa4JDZxTpiooKDRVpDw+PDhRppVIJgUCAyspKhIaGai2r0f+t3qsJDg6GSCRig1thYSFsbW015G7aBxRzJnMbQF+ZjFwuR2ZmJsRicYcBS31mS7zt28FJTVW5T1pbQymXg1NUBN5XX0EZH49GBwekp6fD29ublSdRMZVUO3cLCwJLS1UAoDt/2kPpCdQn3Wn/g97/9vbAsGEMrl8HAA7Cw5Wwt6czCgSMNilkNVhY8GBvbwd7ezu4uCjh5CRHY6MUTU0E//pXHkJD7eHm5gYvL2d4eWnfxTU3N6OsLANhYd4ICQm5uchoDwhdZSleXsCmTbIez8lUVVUhNzfXZCWq/Px81NbWYvTo0XB0dGSzRUrrVZe6oTM7+trtKhQKZGRkgBCC2NhYgwYzbWhra0NqaqpWo7XeQBtFmpZ6y8vLO/TEGhsbkZWVxc4AaRsEpe/bniJtb28Pe3t7+Pv7Q6FQsHI3ubm5UCgUcHV11ZC7MaYCc3/CbRdk+kphprRQW1tbNnNofw69BBmlEtzjx0EcHG5JxHC5ELu5wbGpCU1//omUoUM1Gvy3asV0uEz1UoVCFRQIUQUIiUQ19NhdGUh9Z09bWVTDi8slkEoBhULlYsnlOtxk9xA4OJCb3xXQ1KT7R1axdqxhYWENCwsgPt4ftrY14POvoaFBjro6d7b/QHfLfD4fWVlZ3WYQumYpqkCiG02ZEILS0lKUlpYaVMm4MyiVSmRnZ6O5uRmjR49myy00W3RxcWFl8Cl5gErd0O/R1dW1130jOuRpaWmJyMhIo7OahEIhUlNT4evri6FDhxqkTNSeak4p0kVFRbh69SoAlUK3g4MD6+7ZG4q0Nrkb2jO6fv067O3tQQiBk5NTp+oCIpHI6H0wY6BfBRld3DH7EgBo49rPz09tx6yJ3hiXaQVNOdR+uBwAIAQymQzV5eWI/te/MGDAAI2mpHo9+NY1sWr5IATsTAmgWmBtbLQvqt7ewFdfae7sa2uB//zHCmIxB0Kh8iaRwgaEcMHlEo0FXCRCp34oFNoCneoxDlxdXREQ4IJhw4ZBKBSCz+ejoqICOTk5cHJygpWVFerr63WaJPf2Br7+uvMspaeD6OoZRGxsLJxuWigYCwzDIDMzEzKZDKNGjeqSIm1raws/Pz/4+fmxLCs+n4/s7GwoFApWQVo9eHcHUw95tra2IjU1VUNo09CgFGk3Nze4uLjg6tWr8PHxgUwmw+XLl7ulSAPQOctp3zNqaGhAYWEh6urqwOfzNQZB6d9MX5IymzdvxkcffYTq6mqEhYVh06ZNmNCuD6yOs2fPYvXq1cjOzoavry/WrFnDOmjqA/0qyHSH3mYZdMdaWFiIsLAw+Pr6dnkOvfRkLCygjI8H9/ffQQYMALiqYUfLtjbILCwQMG8ebNsFmM7E87hcwM1NFQCkUg7WrFEgLk51jXQYszOonrsVhAYPBvbtk6GoqBqlpaUYOnQoKis9sXSpivKcnk6ldW71ab74goebfmg335PBvfd2nlkAmtmFut1vYGAgJBIJcnJyWI+gwsJCNDc3s/Tozha89p+lt2AYBtnZ2WhtbdXIIIwFqm7B5XJ7XKJqz7JqbW3tELxp/6GzoUKxWIzU1FS4urpi+PDhRg8wlM0ZEBCAIUOGGPXcgEobMSsrCxEREazYra4U6d5kOZaWlvDy8kJNTQ0raVNfX8+OKzAMg+PHj4NhmD73ZH777Te8+OKL2Lx5M+655x58++23mDlzJnJycrTS4UtKSjBr1iw899xz2LlzJy5cuIDly5fDw8MDDz30UJ+uheK2CzI9LZfRBaW+vl4nu2Z99mSUCxaAm5YGTmEhlLa2kLW0wFKphOXjj4M7alS3AYZhVNmLOjgcwNubICCgd4utamHKB8NUY86cKLi4uODSJZWGGD09w2jSgrdubb8IWuKHHyT45hvtmQWgyi60tTcYhsH169chEokQHx8PGxsbtmZOvV3oIunu7q73HgHtxzEMYxKhSZpB2NnZ9diPqD3UpVO0SbRQd0kqEMnj8SAUCpGWlgYvL69Os3lDgio5BwUFGX0GCLilwzZy5EgNNfXOKNK1tbXIz8+HnZ0de0/SjVBnXjmdZTm0J9Ne7iYjIwOFhYX4+++/oVQqoVAoMHv2bEybNq3H83qffvopFi9ejGeffRYAsGnTJhw7dgxbtmzBhg0bOhz/zTffwN/fH5s2bQIADB8+HCkpKfj444/vzCDT3Q1PA0B7Kf7OIJFI2AFLuqB1B72Vy6BSU5Z//TVkP/+MtrNnwQsMRMnw4Rj28suw6kLk0saGwMJClVmoJ1VKpWqKvrPyWHdgGAZZWVloa2vD6NGj2fqvhwfg60tgb6+SYxEIOCgs7Pr7ranhwcdHgQ6ZBSHg5OeDU9cCpd0wlWPaTcjlcrbJPHr0aHaBVxc1pA3vkpISXLt2jZ0j8fDw6HPGQe8HGxsbREdHG70HQfuBbm5uBskg1CVaGIZhFYnz8vIgk8ng5OSE1tZWDBw40CQBhg6Z6qLkbAjQANOdDltn5S71jZAuFOn2g6ByuVzDtgBQyd2MHj0a+/fvx7JlywAAgwYNwvvvv4+3334beXl5On8+Osj66quvajw+bdo0XLx4UetrLl26hGnTpmk8Nn36dGzbtg1yuVwvm7x+FWS6A52O1kXMrqmpCenp6RgwYECPhup4PB5k3TUieoAqGxtkjxuHkIUL4efnh+rjxxEEgKfmKd7+x+7nRzB6NNOp0KOfX8+DjEQiQUZGBiwsLODnNwY1NbdunooKzk0ygGqWpv19pX55XYwxgVNUBMt168C9elWVgjk7Q7F4MRRLl0IslWr0ALT9/do3vNvPkdjb27MBR1d5FgqhUIj09HSDLfDdgZp9DRw40GBNbnWoC00OGzYM1dXVyM3NhbW1NSoqKtDY2Mj2cXr6XfYGlCIeGhraZbnaUKipqUF2dnavhD5puYvOialTpHNycuDo6Mh+105OTlqznIqKCkgkEtja2rJZDv3t02xHLBYjJiYGr7/+Ot5//31WMFNXCAQCMAzTYcaKluo6+160Ha9QKCAQCPTCtrytgoy6FH9XQYbWOoODgzF48OAe/YD01ZNRNzqLjo5mG/xcLhf19fXw8vLqdJfg5YVOS1G9kaxvbW1lF1hX1zAsXmyD1tZbz0ulwI0bHFhZcTByZC8/u0gEqyVLwCksVGUvDg5AayssPv0UIisr/BMcDC8vrx7RVOkcib+/Pztwx+fzWXkWGnC6Uz1uampCRkaGSXTAAJUWXWZmJoYMGYIA9eaWkUBdJIcNG4ZBgwZBJpNppfXS71LfVgK0B9Jbq4C+ggbYyMhIDBgwoE/v1RlFmoptqlOk3dzcYGlpicrKShQXFyM6OhrOzs4dBkHp+zY2Nmr0ZHpLAmh/f3dX+dF2vLbHe4t+FWS6+1B0d6BQKLTW0pVKJfLz81FVVcUu7D2FPnoy1Daa2hZQeiTDMPD390dJSQmuX78Od3d3eHp6ah226wkVtyvQFJ82WUtKuGhtVVGjadlNJFJlMXK5irbcm0SOd+oUOMXFqtob/Szu7lBWVYHZuhUB//sf/HsY8NWhPnBHqajqqse0fOHh4aHB1Kqrq8O1a9cQHBwMPz+/Xp27L6DnN9UOXpsXjJWVlYbUDaX1FhQUQCwWayhI97VESTOI9j0QY4EGmIiIiD4HGG3oiiKdlZUFW1tbiMViDB8+nNX809bLuXHjBi5duoShQ4f2+loGDBgAHo/XIWupq6vrVEHC29tb6/EWFhZ6o/T3qyDTHTgcTqc0ZplMhszMTEilUlaapTfoa09GLBYjLS0NlpaWrNGZelNw6NChGDp0KIRCIerq6lhfeX32HigqKipQUFCAESNGdNhB2tjc0gDj8VS+LjIZB0IhpwMtuasSGQWnvFz1P9SCpUwuB8PjwUkohK23t2bdrQ9Qp6KGhISwVr9VVVXIy8uDk5MTPDw8wDAMysrKEB4ebnSZFkCVUV+/fh3h4eEmWWB18YLR9l2qlyhpw5v6u/Rkk0D/HvrIIHoD9fMbYwaq/XdZVlaGgoICODo6Ii8vD8XFxR2IGIBqIzhv3jw8+OCDbAO+N7CyskJsbCxOnDiBefPmsY+fOHECCe0s1Cni4+Pxxx9/aDx2/PhxxMXF6Y10c1sFGUB7pkEZM/b29hg7dmyf0v2+ZDK0D+Th4YERI0awTT5tDX5K6Q0KCoJYLAafz0ddXR2uX78OBwcHeHh4wNPTs1feFlQyp7q6GjExMXDpRoOGapo1N3Pw0UdyZGVxsW5d1zeYlZVm5CG0kSuXg1haQiqVQi6Xw16pBGfgwFsDqXpGe6tfmUzGGomJRCJYWVmhsbERFhYWfRpc7AnUhzyjo6MNoujdHXrrBUOn2KmdMW14U2FU9Wn5rhYhGmAN6UXTFYwdYNqjpqYGRUVFiI6Ohru7uwZFOi8vDwKBAJs3b8aECRPw+++/Y/To0di+fXufS5WrV6/GU089hbi4OMTHx2Pr1q0oLy9n517WrVuHyspK7NixAwCwdOlSfPXVV1i9ejWee+45XLp0Cdu2bcOvv/7a5++Aol8FGV0W0/ZBoK6uDlevXoW/v79eRA1725OhQpu0DwTc4tID2hv8FLa2thq9B4FAwGY5lpaWbMDpaoaEgpbqxGKxBoOsO1hbq/4NHkygVCpgYWGJzmItlwuEhGg+ydx/PyyGDAGnqAgyW1swXC7s5XJweTzIn35ab1lMd7CwsEBzczMYhsHo0aPZoHPt2jWWFUR35oaQUGkvlW9sl0N9esG0b3hTBWnK/HNxcWHLauqVAxrgdNngGAI0gzNVgKutrUV2drZGgGtPkb5x4wZiY2Oxd+9eFBcXg8vl4rXXXsPs2bNx77339node/TRR1FfX4933nkH1dXVCA8Px+HDh9k1qbq6GuW06gBgyJAhOHz4MFatWoWvv/4avr6++OKLL/RGXwYADulK9tjIIDen4bvCxYsXERQUBE9PTxQXF6O4uBjh4eF605yidf7x48frdDy15y0rK0NkZCR7E9HyGIBe757Vp7v5fD6USiV7o2rzIqEMMktLS0RERGhdRIuLOXjqKSuNnozqtSrpmZ9/liEwkODUKUAg0N5MHzCAwZQpWq43OxuyFStgX1oKKy4XcHKCYuFCKF544ZZYmgFBhR4pS0edsk5ZQfS7bGtrg4uLi4YeWF9BnTSbmpoQExNjdIkQ9QAXGxtrUB0sKnXD5/PR0NAAW1tbDBgwAAqFglVRMJbZmTpMHWDUBz27YrE1NjbigQcegL+/P77//nucPXsWhw4dQm5uLi5dunRHKTHfdkHm8uXLGDhwIAQCAftj1qckSH19Pa5du4aJEyd2eyxd1FpaWhATE8OKG1JufFfZS0+hvpOsq6uDRCKBm5sbu0hSHSqqZNtZYKuu5mDRIisNdhmFoyPw448y+Pj0/JYQi8VIT0+HnY0NInk88IRCKENDASOVKuhQG4fDQVRUVLdZSvtFkvYePDw84Ozs3OO/G70XJBIJYmJiupSJMQTUvWBiY2ONGuAUCgXrXtna2qpBn+6J1E1fQUt0pipR8vl8XL16tVuSQ3NzM+bMmQNPT0/s37/f6PeKsdGvggyAbk3JLl++DLFYDFtbW0RFRen9D9TU1IS0tDRMnjy5y+Po5DaPx0N0dLRGg78riRh9oa2tjdVBam5uBgC4u7sjJCSk2x1sdXXH5j6gct3sTYBpaWlBenp6jynK+gIlW1Czq54OWdJFks/ns1I36vTo7urkdMgUgE4BTt9Q94Jpn8EZA5SuT3uAhBA2YxQKhSwRY8CAAb3qMeoCUwcY6mXTHcmktbUViYmJcHBwwMGDB40uaWQK3FZBprGxEf/88w+cnJwwZswYgzRxW1tbcfnyZdx///2dHkO1l9QHPTtr8Bsa5eXlKCgogI+PD6RSKRoaGmBjYwNPT89e78p7AkqRDgwM7PFMkj7Q2tqKtLQ0eHp6snYJfYG6eyWfz9fIGAcMGNBhAZfeHDK1sbFBRESE0VUE1L1gYmJijC6TQ4VG6+rqEBsb24HVKZFIWKmb+vp6WFlZsX2cnjpLdoaKigoUFhYiOjraJD0gqmSgjcWpjra2Njz00EPgcrk4dOjQHekdow39LsjIZDKt7pgVFRXIy8tjmVd94ZN3BZFIhPPnz2P69Olan6+ursa1a9dYJ00AbAYDgJ3gNTToj7umpgZRUVHsj0t9V87n83s0tNhT0N1jWFiYSSjCdMgxICAAAQEBBvneKT2aZozqzD8ul4v09HS4uLh06+ZoCKh7wZgig6IluoaGBsTGxurkDNnY2MhmjDKZrNP5Jl1BSQamDjDDhw/vsi8sFovx8MMPQy6X48iRI0YnhJgS/Ypdpg1KpRJ5eXlsKl5bW6t3C2Z18Hg8tq+ivmgQQlBUVITS0lJERkbC09MTBQVAc7PyZlBUpycDQ4caLnarM8jGjBmj8eO2sLBgGUF0OKyurg55eXmQy+VdDoDqCkp2qKysNFl5gg75hYaGYuDAgQY7D6X0BgQEQCaTsX2c0tJSKJVKODg4mGSK3dReMJTk0NzcjLi4OJ1KdOq9GkIIhEIhBAIBSzemAZw6WHa3aTB1gKGbnNDQ0C4DjEQiwWOPPQaxWIxjx47dVQEG6OeZDG3mymQylq2Tn58PhmEwYsQIg5xfoVDg5MmTmDJlCrsIU2HJ5uZmtsFfUACMHNn5ziszU2yQQENFHq2srDplkGkD/VHTPo5QKOzVAKi60VZ0dLRJUv7y8nIUFhZi5MiRPdah0gcaGxuRnp4OT09P8Hg88Pn8Xvu69Aam9oJR7wHFxsbqpS9KAzgtq9EMnM7ktA+iZWVlKC4u7jNNu7eg98CwYcO63ORIpVI8+eSTqKmpwcmTJ02yITM1+l0mw+FwWJ+MtLQ0ODk5ISYmhm2+8ni8bskBfQH9wTIMA0tLS3ZR53A4GDt2LKytrW8yvboOINrYW30FbbB7eHggNDS0R4uLuqeL+gAonezWZQC0vUy+KRhUNIMyFUWW9qDUlYRDQ0NZX5fy8nLk5ORo2CXrMxCb2gtGqVSyLLq4uDi9BVMrK6sO8iz03pRKpeyGaMCAAairqzNpgKFD1yEhIV0GGLlcjoULF+LGjRs4ffr0XRlggH4YZABVKSQrKwtDhgzpIGjYV3fM7kCb9rQBnJaWBnd3d4SHh7dr8BvsErSCamDpq8He0wFQSlG2tbU1iUw+Lc80NjZi1KhRJsmgqqqqkJub24FB1N7XhTa7+Xw+ioqKYGNjo0GP7m1gMLUXDHXzlMvlPTZb6wnU5VmGDRvG9sVqa2tZ6XtfX1/Wu8WY30NzczPS09MRHBzcpV2BQqHAs88+i8LCQpw5c8YkqgP9Bf0uyBQUFKC4uBgRERFam8n6NBXrDDweD7W1tSgsLERQUBDr3qfuDcHlGmeRJYSwtWdDNdgtLS1ZwUT1AVDqneHs7Izm5mZ4eXlh+PDhRl/cFAoFrl69CplMhtGjR5tkrqC0tBQlJSWIjo7udsjPxsYGgwYNwqBBg6BQKLRKs3Q2UNsZqFWAMe2K1aFOMoiNjdW7UnNXoH0xWuEICAhAW1sbSxtXV5A2JPmBbjqDgoK6FFtlGAbLli1DVlYWkpOTTaJb15/Q74KMjY0Nxo4d22lzrDfumD0B3R0VFhayDf72E/zG+oFTVWlKDzVGaaC9/EVZWRkKCwthYWGBqqoqSKVS9nljLPZSqZRtcMfFxRl1cQM6luh6OvhrYWHRwZSNZjhZWVka9OjO+mLUTZJmscYGtYumM2HGzmIBlU1wWVmZhlQPrTaoqx67urqy5AJ9Zrs0yAcGBnbp6MkwDP7zn//gn3/+wZkzZ0xCCulv6HdBxt/fv8tMxZDlMoZhWI2rESNGsAuDKeZf6O5dIpGYxIceUEl0FBUVITw8HN7e3p2qHXt6ehqkfCUSiZCWlgZnZ+ceGc/pC0qlkqXo6qNEp27KFhwcDJFIxJaB8vPztZqyUbMvU7lJyuVypKWlsUQTUwSY4uJilJeXIzY2VmPzyeVy4erqCldXVwQHB7N9RmpbYGtrq6Eg3dv7h/aHhwwZ0mWQVyqVeOmll5CcnIzk5GST/L36I/pdkOkOhiqX0aE6DocDW1tb2NjYGHWCXx3qDLJRo0aZZP6hqKiIFVmkDUt1Oq9UKmWJA8XFxXofAKW1b19fX70In/YUlFEoEokwatQog0zR29nZYfDgwaziMe3jUCUJe3t7NDY2YsSIESbxoqF2vra2toiIiDB6kAfA3oftA4w2qPcZ1cuU6pbJtEypK2FBKBQiNTUVgwcP7tJwTqlU4tVXX8XRo0dx5swZk2Sc/RXmIINbqbCbmxvCwsLwzz//QC6XdxlguqO695YK3xcGmT6gTlEePXp0p7t3a2trjb4DHQBNT0/v8wAo3b0HBQWZ5MdK+w9KpdJoQV69L6ZUKlFQUICKigpYWloiNzcXdXV17K7cGGVKSpOmUj3Gvg/V1aTj4uJ6LPbZvkxJLZOpf5Ozs7OGgrS2TQwNMH5+fmxfVhuUSiXefPNN7Nu3D8nJyQgKCurx572T0e+CTHc7Vn33ZGpra3H16lUEBgYiMDAQgCoNFwqFLD9f2zUNHUqQmSnuVGiyNzMy+maQ9RS9pShrGwDl8/kaA6A06HS3YFMG14gRI/SmrN0TyGQytjxkqv7DjRs3UFVVxWaRQqEQfD4flZWVyM3NZcuUXS2QfYE6TZr6IhkTNJOurKzsVYBpj/aWyersv+LiYlhZWbHfJ/UcamtrQ2pqKgYOHNhl0CCE4N1338WuXbtw5swZhISE9Ola70T0u2FMhmG6DCJisRhnz57F9OnT+3TzE0JQUlKCoqIilsmmboVaVFQEAGzPQd+SLO2vhQ6XmcpFUZ2irK/au7YBUBcXF7aspt5nUjf6ioiIMAnlky6upuoBqe/eqR98e6iXKRsaGmBtbc0ukH3pO1CIRCKkpqZiwIABetGC6ynUA4yh7QqAW3YaNOgoFAo4OzujpaUF3t7eXX4HhBB88MEH2LJlC06fPo2RI0ca9FpvV9x2QUYmk+H06dOYOnVqrxdChmGQnZ2NhoYG1ipAvf9C9ceoJAufz2d1lvoqydIeVDaHz+cjKirKJMNltETn6emJYcOGGWxxVR8AbWxshL29Pft9VlVVoa6ujlVUMDZoacRUStK98YLR5jekTo/u6T1Kd++mmsOhTL6qqirExcUZfRaKEAKBQICsrCzweDzI5XI4OjqyZUp1qRtCCDZt2oRPPvkEJ0+eRExMjFGv9XZCvwsySqUScrm80+cZhsGJEycwefLkXk0bU0osIYT1/eiuwa++I6+rq0NbWxvc3NzYHXlva+SUQSaVShEVFWUSBhntfxi7RKc+AMrn8wEA3t7e8PHxMZpNMgWd4Pb39zfJDIo+vGC0mbKpT8l39540yPr6+mLo0KEm+Q4KCgrYIGuKYVuxWIyUlBR4enoiJCREg4xRX18PCwsLHD16FOHh4aitrcXHH3+MY8eOYfTo0Ua/1tsJt12QIYTg+PHjmDBhQo9/jK2trWytmfqO9IZBJhKJ2IDT0tLCSoh4enrqfE1isRgZGRmwtrZGRESE0ec/AFXtPz8/H2FhYSbh86v3gPz9/VmFXroj9/T07NHAYm9A2UfBwcFdDtgZCobygtGWNdIspz37jxJf/P39MWTIkLs6wHh4eGjNZJVKJRoaGvD666/j+PHjEAgEGDt2LJ566inMnj3bKPfOhg0b8Nprr2HlypXYtGmT1mOSk5MxadKkDo/n5uYiNDTUwFeoHf2u8d8dOBxOrxhmdXV1yMzM1Gjwa07w605RtrOzY+XlaY28rq4OhYWFbAmoKw2w5uZmZGRkGLw81Rk6oygbE5SmbWNjwzbYfXx82B15XV2dxsBiX7NGbaiurkZOTo7Jgqy6F4w+dcCAjrJB7dl/NODweDxcvXoVQ4YM6ZKiayjQMmFtbS3i4uKMblkNqO5F2ofqrFTK5XLh7u6O+Ph4HDhwAD/88APq6urw66+/YsWKFTh79izuueceg13jlStXsHXrVkREROh0fH5+vsbgsCmEZCn6XZDRZaHvSZChDWWq2uvt7a0xYEnP2dvdmzqVV70EVFpaCmtrazbg0N1jbW0tsrOzERQUBH9/f6PvGttrgBm6saoNQqEQ6enpcHNz6yDyqM4ECg4OZgdAq6urNQZA+8qsokrOUVFRJiEZtJdpMSRN2tLSEt7e3vD29u7A/pNKpbC3t4eFhQWkUqlRJXuoJxKfzzd5gHFzc+u2yf/LL79gzZo1OHjwIOuc+8orr6ChocGgfUShUIgnnngC3333Hd59912dXkN1B/sD+l2Q0QW60pjpzIdAIMDo0aPh7OzcQSJGn1lEew0wunuk3vM2NjYQCoUmL08pFAqTaYDR/oefn18H8VNtaD8ASoM4HQClZUpdB0BpFnfjxg2TKTmb0guGik8SQnDjxg0EBgaCy+WyKg600e3h4WEwq2SgY4AxRT9SKpUiNTUVLi4uXWryEUKwZ88erF69Gnv37u1gzd6dll1f8fzzz2P27Nm4//77dQ4y0dHRkEgkGDFiBN544w2tJTRj4bYMMrpIy9AfMsMwiI+PN/oEP4/HY7MYajLW0NAACwsL5Obmgs/nG6XnQKFOUY6KijJJD4jOAfW2/2FtbY2BAwdi4MCBbBCvq6tjgzjt43RGN6cNdoFAoJf5i97A1F4wwK0+lPos0pAhQzRM2UpKSrTOj+gDhBDk5eWxfwdTBBiqZuDs7NztLNDvv/+O559/Hrt378aMGTOMeJXA7t27kZaWhitXruh0vI+PD7Zu3YrY2FhIpVL8/PPPmDJlCpKTk3Hvvfca+Gq1o981/gF06xdz+fJl+Pn5dSq1QbWGnJ2dMXLkyF43+PUBuVzOKghHR0fD2tqa7Tnw+XyIxWK4ubnBy8vLYGZX9PswVQ8IuGXVbIg5IPUSEJ/Ph1QqZXsO9DtVKpXIyspiG+ymWNhM7QUDqIaPr1271sGuoD20zY+of6d9cVXNy8tjmXSmDDBUzaCrteDPP//EokWL8PPPP+PBBx804lWCVTs4fvw4IiMjAQD33XcfoqKiOm38a8OcOXPA4XBw8OBBA11p17gtgwylGWpTQ62rq8PVq1cREBDATuqaSuSSZg82NjadMsjoNHddXR1aW1vZYUVPT0+9MI0oRZk2dk3BHKICh1FRUQYnGVC6Of1OhUIhnJ2dIZPJwOVyERsba1DXys5gai8Y4BbRISIiokeNYCqxT4M4HapVl2XR9X2o4KipAoxcLkdKSgrs7e27lcs5duwYnnzySfzwww949NFHjXiVKhw4cADz5s3TyMoZhgGHwwGXy4VUKtWp1Pree+9h586dyM3NNeTldop+GWTULZi1IT09HS4uLhp6QuoN/vDwcJappK8Gf09BGWR0UdFl1yqRSFhqdFNTExwdHVlWVW9KO5WVlcjLyzOZRAsdNBUIBIiJiTFJeaqlpYXtQzEMwyode3p66uQjr69rMKUXDKC6F/Lz8xEREYEBAwb06b3EYjGb4TQ0NMDOzk5D7Vjb5yOEsISTuLg4gwiOdge5XM4KfnZXqjx9+jTmz5+PLVu24MknnzTJ36y1tRVlZWUajy1atAihoaFYu3YtwsPDdXqfhx9+GA0NDTh9+rQhLrNb3BE9GcqY4vP5RmnwdwfKIBs6dCj8/Px0vkFtbGxY2qlMJmN348XFxaxsuaenJysD3xnUKcq6mGwZApSeS60KTLGoiMViZGVlwcnJCSNHjgTDMOzimJKSwjqA6rvnoA5Te8EAqrJLQUEBoqKi9HIv2Nraws/PD35+fhriqJ2ZstEA09TUZNIAk5aWBhsbm24DzLlz5/DYY4/h888/N1mAAQBHR8cOgcTe3p516gWAdevWobKyEjt27AAAbNq0CQEBAQgLC4NMJsPOnTuRlJSEpKQko18/xW0ZZNQpzKZu8KuDZlMlJSUYOXJkn7jpVlZWbJOb/pDr6upYGXhaUmuvV9UfKMoymYxtxsfFxRndqgC4VZ6iata0xKCudEwlWbKzs8EwjEbPQR/ECFN7wQBgNfFiYmIMQmlVF0clhLC9scLCQnbGSS6XQyaTGcwyoTsoFArWOqM7y4JLly7hkUcewYcffohnnnnGZAFGV1RXV6O8vJz9b5lMhpdffhmVlZWwtbVFWFgYDh06hFmzZpnsGvtluUwul7MlLm3Iz8+HQqHA4MGDkZqaCicnJ1bUkRDC0puNWR6jBlf19fWIiorqsYNiT87T0NDAEgcIIRpGV9nZ2ZDL5SzJwNgQi8UaEvGmUDGmXjSDBg3SiSatLslSV1cHkUjU5wFQ2mA3FV0duGX21ZnYpqEhFApx7do1tLW1QalUatCjjVWqVCgUSEtLg4WFRbd08StXriAhIQHvvPMOXnjhhX4fYG4X3JZBpqioCPX19WhpacHgwYNZrSXafzEVg0wulyMqKspouzVq51tXV4fa2lpIJBJYWVkhKCgIXl5eRs8gKIvNVCKTAFBfX4/MzEwMHTq0S5vcrkAHQPl8PpqbmzV6Y7oMgNL+R1+z2d6i/SyQKQRH6Yxaa2srYmNjWZdPdR0w9VKlITYjDMMgLS0NXC4XUVFRXZ4jPT0dDzzwAF5//XW89NJL5gCjR9x2QYYQgvT0dPD5fIwcORK+vr4mbfCrz5+MHDnSJPMndHF3cXGBvb09K5BoKDkWbWhoaEBmZiYrt2OKH2lNTQ2ys7P1SnSgvTG6OHY3AFpeXo6ioiJERkaapBdGdcCqq6uNIpWvDe0DTPt7T6lUsjp1fD5fw3NIXzR+hmGQnp4OAN36AmVlZWHWrFl46aWXsG7dOnOA0TP6ZZChTKD2oP2Gmpoa2NnZYdy4cSZt8Dc1NSEjIwPe3t4m27l3RlGmIp50N+7k5MT2cfQt30EX9+HDh5vEJhi41dzWB3uqM6irOFDlaBpwXF1dUVZW1qUXjKGhPkXfWzXnvoIKfra1telEF1ennPP5fLS2trKCsx4eHrCzs+vx74phGNbZNCYmpssAk5OTg5kzZ2L58uVYv369OcAYALdNkKHNZLlcjoEDB6KmpgZjxoxhRS6NWR4DVAtrTk5On8oyfYWuFGV1Ec+GhgadRDx1BdUAM2VpSL33YCy9JqVSyZYq+Xw+JBIJACAwMBCDBg0y+iyOOkXYVDMoPQ0w2qDuWtkbUzaGYVhl7+jo6C4rC9evX8fMmTOxcOFCvP/+++YAYyDcFkGGMoUcHR0xcuRINDQ0oKCggA0ytyuDrC/XQCnKPS3LqIt41tfXs9IhlKmm6/dIyzJVVVUm37lTszNTlIbo4i4QCODt7Y2mpiZ2qFZ9N25I0PJUS0sLYmNjTcLgoooKIpFIbwOv7TNHQkiXpmxKpRKZmZmQy+WIiYnpMsAUFRVh5syZeOSRR/Dxxx+bRH3hbkG/DDLq7pgCgQAZGRnw9/dHcHAw20DMyMhAcHAwPD09jbZrVGeQRUdHm6yhSnes0dHRfVpYqXQI3Y1zOBwNu+nOfnh0UWtubkZ0dLRJ/D/Ur8HUO/f2XjASiUTDItmQA6Dqizs14TM26DWIxWLExMQY5PdISS70exWJRKwpG+05UtuEmJiYLkkvpaWlmDlzJubMmYMvvvjCHGAMjH4dZMrKynD9+nWEhYWxDX5KUS4rK2NlQ2iD25ABR13B2JgMsvbXQFls+qYoU/0vGnCoVlV7EU+FQsHuFk1Fk6YlEZlMZrBFTZdrUF/UOrsG6uVSV1cHgUAACwsLlozR1wFQXa/BkFAqlezQraEtC9QhEonYgNPU1AQejwcul4vw8HC4ubl1GsgrKysxbdo0TJs2DVu2bDEHGCOgXwYZuVyOrKws1NbWIjo6Gq6urp02+MViMWpra1mXSn1rfwGqGzojI8OkDDJq8mUMJ832cyNUxNPNzQ2VlZWwtrZGZGSkSb4HuVzOGm+ZSk1a3QsmKipK54VVfQCUz+f3aQCUNrdp78EUA6+0PCWVSo0aYNpfw9WrV1mH2oaGBnC5XDbDUVfkrq6uxowZMzB+/Hh8//33JpnhuhvRL4NMcXExSktLWbVcSlHursHfXvuLMqq8vLx6XU6hDDIfHx+TCRu2trYiPT0dAwYMQGhoqNF3X21tbbhx4wYqKipACIGLiwu8vLzg4eFh1DKVuky+qQY99eUF09kAKF0cu9og0Ql2AN02tw0FGmBoNmmqANOeaNBekbusrAw7duzAfffdh7179yImJgbbt283yXd2t6JfBhmGYSCRSFjdo95IxMhkMjbgNDQ0wMHBgQ04uvYQKIPMVP7vgGq4kKpKm2r+hE7Q+/r6ws/PT8M7nn6vlKlmKLS1tSEtLU2rm6axYEgvGFr+qaur63IAlGZyPB6v2wFDQ4GWKxUKhcmyKEIIS3bozLqaEIKSkhJs3rwZR44cQVlZGUaPHo2EhATMmTMHYWFhRvk9bdiwAa+99hpWrlzZpUT/2bNnsXr1amRnZ8PX1xdr1qzB0qVLDX59hka/DDJKpZIdyNTHBL9cLmd/wPX19bC1tWUDjjYKL705S0tLDTp30R1MraIM3JrDCQoK6iDwSE2u6PdqY2PDBpzuRDx7AqpiPHDgQFbdwdgwphdM+wFQauPt6uqKwsJC1jribg4w6oKbXfUFGxoa8MADDyAgIABffvklTpw4gYMHDyIlJQWlpaUGz2iuXLmCRx55BE5OTpg0aVKnQaakpATh4eF47rnnsGTJEly4cAHLly/Hr7/+ioceesig12ho9MsgwzAMZDIZS2PW5wS/QqFgF0aBQAArKyt4eXmxCyO9gRsaGkzGIFOf/TDV5DgAVFVVITc3Vyf9LapwTL9XKuLZ1wZ3Q0MDq2IcEBDQq/foK0zpBUNpvDU1NaitrQWHw4G3tzdLyDBmoFHvA3VHETYUqCcNnQfqqqzY3NyMOXPmwMvLC/v27dMIRgzDGPy7o6zDzZs349133+3SbGzt2rU4ePCghufL0qVLkZmZiUuXLhn0Og2NfhlkPvroI/D5fCQmJrLOloaAuoUvn89nz0PNrUxFi6XGTn2lKPcWdBaIZnLu7u49ej2VDamtrdWYb+jpwkhFJk2pJNAfvGAkEglSU1Ph6OiIgQMHssOKUqkU7u7u8PT0NJirKoX6FL2p+kDqrprdWQa0trYiISEBTk5OOHjwoEnYoAsWLICbmxs+++yzbh0t7733XkRHR+Pzzz9nH9u/fz8eeeQRiEQik2SM+kK/7H6NGDECP/74I6ZNmwYvLy8kJCQgMTERMTExei1TqEvm090qh8OBQqHAP//8wz5nKK+R9lCnB5tKFp0OONbW1iIuLq5XmRyXy4W7uzvc3d01RDyvX7/OWiPThbGzHw+1a+6pi6M+0R+8YNTLdNSL3t3dHSEhIWhra0NdXR0qKiqQk5MDZ2dnNnvU5wBofwkw+fn5EAgE3QaYtrY2PPzww7C1tcX+/ftN8jvavXs30tLScOXKFZ2Or6mp6WCH7eXlxVZeTFUu1wf6ZZCZPXs2Zs+ejba2Nhw5cgRJSUmYM2cOXFxcMHfuXCQmJmL06NF6y3AaGxuRmZkJX19fBAcHs74YdXV1uHbtGiunT3fihgg46hTluLg4k/yQGYZhhwtHjx6tl0yOw+HAxcUFLi4uCA4OhlAoRF1dHcrKypCdnc0yqjw9PWFtba2RRVH6uinQH7xgRCIRUlNTWVahehbF4XDg4OAABwcHBAYGagyAFhQUsAOg1AKitxkYFZokhJg0wFy/fh18Ph9xcXFd3pdisRiPPPIIAODgwYMmGRSuqKjAypUrcfz48R4FOG29YW2P327ol+UybRCLxTh+/DiSkpLw559/wtbWFnPmzEFiYiLGjRvX65uf+p6HhIRoZZDRnTidxZHL5eyiOGDAAL0EOlNTlAEVOUJ99sMYg31isZhlAFIRTw6Hw1JSTdEPA/qHF0xbWxtSUlLg7e3d4z6Q+gBofX09eDyehpCnrvcXpUpzOJxulYwNBSpfVFNTg7i4uC4zNIlEgvnz56OlpQXHjh0zidQRABw4cADz5s3T+L4YhmGN86RSaYfv8k4ul902QUYdMpkMJ0+eRFJSEg4ePAgul4sHHngA8+bNw4QJE3T6g6g310eOHKkTg4wQgtbWVjbgSCQStvTj4eHRq0BHKcqDBw/GkCFDTFbzT09PNylriVolt7a2ghCiVxHPnsDUXjCAatORmpqqs+laV6D9Mdp3ZBiG7eNo0/+ioAFGFy8WQ4EQgsLCQlRXV3cbYKRSKZ588knU1tbixIkTJsuAAdXfr6ysTOOxRYsWITQ0FGvXru1gqQyoGv9//PEHcnJy2MeWLVuGjIwMc+Pf1JDL5Th79iz27t2LAwcOQC6X44EHHkBCQgImTZqkld6orv8VFRXVqx0zIQRtbW1swGlra2N/vB4eHjplApS9ZUqKslAoRHp6uknnT6g8ikQiYftutLktEAhgaWmpYTdtqIBjai8Y4BbRwN/fH4GBgXp9b/UBUHXPofYDoP0hwABAYWEhKisrERcX12XZSy6X4+mnn0ZpaSlOnz7dY6KKMdC+8b9u3TpUVlZix44dAG5RmJcsWYLnnnsOly5dwtKlS80U5v4GhmHw119/sQGntbUVs2bNQmJiIqZMmQJbW1vU1NTg0KFDCA8PR1RUlN60t2gTtq6uDq2trXB1dWUXxvbn6C8U5aamJqSnp8PPz6/PO+begpbpAGiVaOmtiGdPQP8epvSCAW79PYxFNNA2AOru7g4+nw8rKyuTBhh1Z8+uGJYKhQLPPPMM8vLycPr0aXh6ehrxKnVH+yCzcOFClJaWIjk5mT3m7NmzWLVqFTuMuXbtWvMwZn+GUqnE33//jb1792L//v0QCAS47777kJqaioiICOzZs8dgP6D2vQZnZ2dWhsXa2trkFGUALKnBlGoGUqkUaWlpOpfp2ot4yuVyDaZab8qVtKlcU1NjMidJ4NY8kKn+HjKZDDU1NSgqKoJCoYCtrW2vLCD0gZKSEpSVlSEuLq7LvwfDMFiyZAnS09Nx5swZk/XPzOgad2yQUYdSqcR3332HF198EYMGDUJVVRWmTp2KhIQEzJw5E05OTgY7t1QqZQNOY2MjuFwueDweIiIiTFY3pvTg8PBwk+38RCIRaxk9YsSIHmcktD9Gv1sq4tmTciUd7KuvrzeZkySg6stlZmZi2LBhGDhwoEmuQS6XIy0tDVZWVggLC9PQ/wLAltQMPQBKmYXdET8YhsELL7yACxcuIDk52WTfmxnd464IMr/++iueffZZbNq0CYsXL8bVq1fZDKeoqAhTpkxBQkICZs+ebbBdG9W9AgBra2s0NjayzW2qp2bo3aK62VlUVJTJglxrayvS0tLg4+PDegT1Fe3LlVSNuzMRz868YIwNPp+Pq1evmrQvpx5gIiMjNQI+pfPTshodAKVBR58sxLKyMhQXFyM2NrbLjZ9SqcSqVatw6tQpnDlzxmQzTGbohrsiyJw8eRKEEEydOlXjcbqT3bt3L/bt24ecnBzcd999SExMxAMPPAB3d3e9LICUouzu7s4219UdKgUCAWxsbFh5G30bWwH9Q0kAUJWFMjMzDSr4SWdGaPaoLuJpb2/PysOb0ocFuEWVDg8P7zCIZyzI5XKkpqay9g1dZZSU7EK/29bWVjg7O7Nltb5kgpR0oUuAoUys5ORkvZMjzNA/7oogowsoXZIGnIyMDIwfPx4JCQmYO3cuvLy8erUg6kJRprpftbW1rJ4aXRSdnZ37vBCrs7eio6NNtmunfSBjloXai6NaW1tDqVTC0tJSbzbBvQGdzzKlogENMLQn1tOSZXsHUDs7OzZ77MkAaEVFBQoLCxETE9Ml6UKpVOKNN97Anj17cObMGYSEhPToes0wDcxBRgvo1HlSUhL27duHK1euYOzYsZg7dy4SEhIwcOBAnX5AlKLcE+0tyqaiul/q0jeurq49DjgymQwZGRngcDg9MtjSN+h3MXLkSJP1gahEC7WO6O2QYl9BZ3EiIyNNRreVyWRIS0tjjfj6+tmp/AmlndPvlhqHdfb+N27cQEFBAaKjo+Hi4tLp+xNC8M4772D79u04c+YMhg8f3qfrNcN4MAeZbkAIwY0bN7Bv3z7s27cPFy9eRGxsLBISEpCQkIDBgwd3ahVQVlbWK4FJCm1CkzTg6ELfFYvFSEtLg4ODg8lMvgBVM7ekpMSkdO32XjAA2CHFuro6KJVKDekgQ31XFRUVKCgoQFRUlMm+C5lMhtTUVNjZ2endFwe4dd/SDJIOgFIHULrRocG2O/kgQgg++OADbNmyBadPn2b/fmbcHjAHmR6AEIKamhrs378fSUlJOHfuHCIiItiAM3ToUMhkMmzatAljx45FTEyM3qRRCCEaiyLDMF0uirS57uXlhWHDhplkBoZKglRVVSEmJsagLL6u0J0XjLqIJ21uU1tkDw8PvWV/NNh2t2s3JGiAoe6ihs7e1FmAdADU1dUV1tbWrL16V8GWEILPPvsMn376KU6dOoXo6GiDXq8Z+oc5yPQShBAIBAL8/vvvSEpKwunTp1lDLZFIhJMnTxqMt08nt6nagEwm05gXaWlpMXhzvTuoEw1iYmJMIlQI9NwLhja3acARCoXsYG13tshdgQ7fmjLYGjvAaINIJEJRURFqamoAAE5OTmwwby8fRAjBV199hY0bN+LYsWMYPXq00a/XjL7DHGT0AGp0NnPmTDAMA5FIBG9vb9aiwJA/aEIIhEIhG3BEIhEIIRg4cCCCg4NN0oNhGAZZWVkQiUQmpQfrwwtGm4inui1yd6C08crKSr1mtj2FVCplPWnCwsJMEmCAW5bmkZGRcHR01OjjWFtbw8XFBeXl5ZgyZQp++uknrF+/HkeOHMG4ceNMcr1m9B3mIKMHXL9+HVOmTMHUqVPx7bffQiwW488//0RSUhKOHj0KHx8fzJ07F/PmzUN0dLTBfuBlZWUoLCyEl5cXhEIhhEIhO6Do6elpFCaVXC5HZmYm6z1iKqKBIbxgZDIZW/apr69n5fQ7o51TNYHa2lrExsaaLJvrLwGGUrYjIyM7CNJSwsvly5fx73//m7VdX7duHVatWmXwQdktW7Zgy5YtKC0tBQCEhYXhrbfewsyZM7Uen5ycjEmTJnV4PDc3F6GhoXq/PmpBfzvCHGT0gKamJvzyyy9Yvnx5h4VGKBSynjiHDx+Gm5sb5syZg3nz5mHUqFF6aTCr9z7UtbdEIhG7C29paYGLiwsrb2OI7EIqlSI9PZ0d6jMV0cAYXjDtbbzbi3gCQF5eHgQCgUnVBKRSKVJSUuDs7IywsDCTeZPU1dUhKyurW8o2IQQ///wzPvnkE8TExCA9PR0VFRWYNm0atm/fbrBe1h9//AEej4ehQ4cCALZv346PPvoI6enpCAsL63A8DTL5+fka5U8PDw+93fc0sND/39raiuzsbFhZWSEmJkYv5zAGzEHGiBCJRBqeOPb29qwnTnx8fK+0t5RKJbKzs9Hc3Izo6OhOd8sSiQR1dXWora1lyz50+FMf5mS0uU4XM1Pvlo3pBdNexBMALC0toVAoulUQNiSobbOpAwxVNeiOvk4IwZ49e7BixQrs3bsXM2bMYC2Xjx07hpUrVxr1M7i5ueGjjz7C4sWLOzxHg0xjY6NBAl96ejr++OMPLFu2DB4eHmhoaMCoUaNgYWGBgoICLFmyBKtWrbotZoXMQcZEkEgkOHXqFPbt24fff/8dPB6PzXDGjx+vU5lJ3a45OjpaZ0VpqVQKPp+P2tpadiKeBpzeLIj9gckG9A8vGPo3aWlpAY/Hg0Kh6LOIZ28gkUiQkpKiYdtsCggEAmRmZuqkarB//378+9//xm+//YYHHnjASFfYEQzDYM+ePViwYAHS09MxYsSIDsfQIBMQEACJRIIRI0bgjTfe0FpC6w1++eUXPPXUU1i7di1efPFFrF69GpaWlli3bh2Kiorw9NNPY9KkSXjrrbf6PaXbHGT6AeRyOZKTk5GUlIQDBw5AoVCwnjj33Xef1uBBS1OWlpaIjIzs9eJFJ+Jra2vZPkNPzMKoPL0pTdeA/uEFo1QqWcJDbGwsLC0tNUQ8RSJRjz2HegMaYKhHkKn+JlT4c8SIEd1mlX/++ScWLVqEnTt3Yt68eUa6Qk1kZWUhPj4eEokEDg4O2LVrF2bNmqX12Pz8fJw7dw6xsbGQSqX4+eef8c033yA5ORn33nuvXq5n165dePLJJ/H6669DIBDg+eefZw3PUlJSMGfOHIwdOxb/93//h4iICL2c0xAwB5l+BoVCoeGJIxQKMXv2bCQkJLCeOFlZWfj7778xduxYvZamaJ+BytvY2NiwAUebTAifz0dWVpZJ7QL6ixcMle7pSg+NUqP5fD5aWlrg7OzMfr/6KFkCt8qWpg4w1Lpg+PDh3Qp/Hj16FE899RR++OEHPProo0a6wo6QyWQoLy9HU1MTkpKS8P333+Ps2bNaMxltmDNnDjgcDg4ePNjjcxNC2L+VRCJhe6b79+9nTcuOHz+O+++/n+3RZGVlYc6cOQgJCcGGDRsQGxvb4/MaA+Yg04/BMAzriXPgwAHU19djwoQJ+OuvvzB//nx8+umnBltEGIZBfX09G3AsLCw0GtuUimrM3kd79BcvGIZhkJGRAYZhdGbUdSfi2Vu6dWpqKtzd3REaGmryABMaGtqtnNLp06cxf/58fPPNN3jiiSdMds3acP/99yMoKAjffvutTse/99572LlzJ3Jzc3t9ztraWnh5eUEsFuO3337DwoULcfLkSUybNg1PPvkkNm7cCF9fXzYoZWdn495778XevXv1VqrTN8xB5jaBUqnEF198gTVr1mDo0KEoKyvD1KlTkZiYiJkzZxp0/kKpVKK+vp7dhVN6aVBQEAYPHmySJn9/8YKhVsVUG643Zcv2Ip40g/Tw8NBZIFUsFiMlJQUDBgwwaYBpbGxEenq6TiKo586dw7/+9S98/vnnWLRoUb8KMAAwZcoU+Pn54aefftLp+IcffhgNDQ04ffp0r873+eef44cffkBSUhKmT5+OmJgY7N69GzweD0eOHMGcOXPwzDPPYP369RqBprW11WTzV7rAOF1IM/qMnTt34vXXX8fPP/+Mf/3rX8jMzMTevXvx4YcfYtmyZRqeOPpQblYHl8tldacKCwtRUVEBDw8PlJeXo6ysTEPexhgBR90LZtSoUSYb9pTL5UhPT4eFhUWfKNuWlpbw9fWFr68vm0HW1dUhPT1dJxFPGmA8PDxMSryg/bmQkJBuA8zFixfxyCOP4KOPPuoXAea1117DzJkz4efnh9bWVuzevRvJyck4evQoAGDdunWorKzEjh07AACbNm1CQEAAwsLCIJPJsHPnTiQlJSEpKalH51UvjQUEBMDb2xujRo1CWFgY9uzZA0C1kZk5cyaOHj2KmTNnQqlUYv369Sw9n2bw6iW3/gRzJnOb4MCBA3BycsLkyZM1HqdqA9SiIDc3F5MmTUJiYiJmz56tN08cmjnQuQ97e3vW0Io2timTysvLy2Aik7r0PowBqmJsbW2tk3V0b0CFJrsS8RSJREhNTYWnp6dOsjmGQnNzM9LS0jB06NBu+3NXrlzB3Llz8d///hcvvPBCv1gYFy9ejFOnTqG6uhrOzs6IiIjA2rVrWQ+qhQsXorS0FMnJyQCADz/8EFu3bkVlZSVsbW0RFhaGdevWdUoU0AaJRIL4+Hg8+uijePXVVwEAU6dOxfnz5xEVFYUdO3YgJCQECoUCHA4HPB4P58+fx8SJEzFnzhz88ssvJisR9wTmIHMHgQ5l0oCTmZmJCRMmsJ44np6evfpBU9ZUW1tbpzIxVE+NzuJQkUkvLy+9UXcVCgUyMjJACDGpbQGdoKeKzsbI3qiIJy2rSSQSuLi4oKWlBd7e3iYtkbW0tCA1NRVBQUHw9/fv8tj09HQ88MADeP311/HSSy/1iwBjKggEAmzcuBE//vgjXn31Vbzyyis4c+YMmpubsW3bNtTW1uLbb79FdHQ0ZDIZu6G6fPkyLl++jP/85z8m/gS6wRxk7lBQuwHqiZOSkoL4+HjWE8fX11enHzid+1AoFIiOjtYpc6B6ajTgiMViuLm5sWoDvQkOMplMg7JtKjUB9QHHESNGmKwfJRAIkJWVBR6PB7lcrhcRz96ABhhd5HuysrIwa9YsvPTSS1i3bt1dHWAoGhoasHnzZnz66ad4++23sXLlSgDA4cOHsWXLFlRXV+Pbb79FbGws8vPzsXfvXqxbt4697/priUwd5iBzF4AQgoqKCtYT59KlS6wnTmJiIvz9/bXeqHRhpz2H3mYjlLpbW1vLqhrTgKPLAGl7LxhTqQn0F3pwW1sbUlNT4ePjg6FDh7JqDuoinrSsZki1gdbWVqSmprJq312BCsg+//zzePvtt/v9wmhoyOVydrO1b98+7Nq1C/v27cMHH3yAV155BQBw7NgxbN26FWlpaViyZAk2bNiAZ599Fp988okpL73HMAeZuwyEEFRXV2P//v3Yt28f64mTmJiIhIQEBAUFgcPhoKCgADk5ORgyZIheVaSpqnFtbS07K0LVBrTtwLvzgjEW2trakJaWZvLmeltbG1JSUuDr68taS6hDJpNpMNWoJXJnIp69hVAoREpKCjuE2xXy8/Mxc+ZMLFq0CO+///5dH2DU8eCDD6KpqQl+fn5IT09HQUEBXnrpJbz77rsAVASJ3bt3459//sHs2bPx5ptvArg9MhgKc5C5i0HLLjTgnD59GqGhoZg0aRJ++eUXzJ8/Hx988IHBbmY6K1JbW4umpiZWRt/T0xN2dnY99oIxFIRCIZs5BAcHm/w6Bg4cyG4GukJnIp4eHh69svJufx2DBg1CUFBQl8cWFhZi5syZmD9/Pj766KPbVklYX1APDt9++y3eeecdXLhwAQEBAaisrMT27dvxwQcf4Pnnn8f777/Pvk4oFLJNfoZhTFYu7g1uqyCzYcMGvPbaa1i5ciU2bdqk9RhjS3DfKaDOm5999hk++OADDB48GBYWFkhMTERiYqLBRS+pjH5dXR0aGhpga2sLiUQCX19fkza1aUmILqi3S4BpDzrrRLMcAGxJzc3NTedFi2ZSAwcOZBWLO0NpaSlmzJiBuXPn4osvvrirA8ySJUs6DHW++eabOHnyJC5dusQ+Vltbi3fffRdff/013nrrLaxfv17jNbdTBkNx2/zVr1y5gq1bt+qs0ZOfn4/q6mr2X3BwsIGv8PYGh8PBlStXsGnTJnzyySdISUnBG2+8gfz8fEyePBnR0dF48803kZaWBqVSqffzW1lZYdCgQYiJiUFkZCQkEglsbW1RVVWFS5cuobCwEK2trTDmnqi5uZktCWkrTRkLtDQ1aNCgXl8HnXUaMWIEJk6cyPbY8vLycPbsWVy9ehU1NTVQKBSdvgelS/v6+nabwdy4cQOzZ8/GzJkz7/oAk5mZyQZ2dYSHh6Ompgbp6ensY15eXpg5cybs7e3xzjvv4LffftN4ze0WYIDbJJMRCoWIiYnB5s2b8e677yIqKqrbTMZQEtx3Mt577z0MGTIEjz/+uMbjQqEQhw8fRlJSEo4cOQJ3d3cNTxx9LiDtvWDal3ysrKzg6ekJLy8vrXpq+gIdLNSn6VlvQDMpPz+/bhf23oAQwop48vl8tLW1aTW6E4lESElJgbe3d7clw+rqakyfPh333nsvvvvuu9uqtGMIKBQKljSzefNmLF68GNbW1khPT8dzzz2H+Ph4LF26lPWtOXfuHL777jusWLECY8aMMeWl6wW3RZBZsGAB3Nzc8Nlnn+G+++7TKcgYSoL7bodIJMKxY8eQlJSEQ4cOwd7eHnPnzmU9cfqyoHTnBaM+Dc/n88Hj8djFsC89hvag2lumFP4EbgUYf39/BAYGGuWcbW1tbEmNEjNcXV1RWVkJb2/vbntjtbW1mDlzJuLi4rB9+/a7PsDs2bMHhw4dwk8//YSamhoMGzYMw4YNw7lz52BjY4MdO3bg/fffR3h4OO655x4EBwdjzZo1mDZtGrvG3W49mPbo90Fm9+7deO+993DlyhXY2Nh0G2SMIcFthgoSiQQnT55kPXEsLS3ZDOeee+7p0TxMT71glEolaxRWV1cHDocDDw8PeHl5dSq/ogtoJqWLuKMhQQOMLuwtQ0EikaCqqgolJSVQKpUdfIfaBxuBQIBZs2ZhxIgR2LVrl9G8c/ozduzYgcWLF+OXX37BI488goyMDDzxxBOwsrLCX3/9BXt7e+zfvx8HDx7EwYMH4efnh9DQUOzevRvA7dmDaY9+HWQqKioQFxeH48ePIzIyEgC6DTLa0BcJbjN0g1wux5kzZ1hPHIZh8MADDyAxMRH33Xdfl0OcffWCUSqVGvI2DMOwAacnTW1qEWxKZWlANeCYlpZm0gAD3PKlcXd3R1BQEAQCAfh8PgQCAaytreHp6Qm5XI7Q0FA0Nzdj9uzZCAwMxG+//WZwuZ8tW7Zgy5YtKC0tBQCEhYXhrbfewsyZMzt9zdmzZ7F69WpkZ2fD19cXa9aswdKlSw16nVVVVVi5ciUkEgk2b94MPz8/XL16FU888QQA4K+//oKzszOkUimEQiEkEgmr+3a7ZzAU/TrIHDhwAPPmzdP4ohmGAYfDAZfLhVQq1emPoA8JbjN0B/XE2bNnDw4cOACRSITZs2dj7ty5uP/++9l5GKVSiatXr6KpqUlvXjBUfoXO4sjlcg15m87uF1qq684i2NCgE/RDhgzpdsDRkJBKpayzZvvBU/Wy5ZNPPon6+noMGDAAzs7OOHXqlFEUgf/44w/weDyW4bZ9+3Z89NFHSE9PZ3sb6igpKUF4eDiee+45LFmyBBcuXMDy5cvx66+/sn4tfQX1eWlubta4l//44w+Wwr18+XIwDIPc3Fw89dRTkMlkOH/+fIfNFX2vOwH9Osi0trairKxM47FFixYhNDQUa9euZV3iukNfJbjN6D0YhsGlS5dYT5zGxkZMnz4dCQkJOHz4MK5du4YTJ04YROhPvaldW1sLiUSiYYVMy3lVVVXIy8szqW0zcEtk0tRkA6rNRqVzuirX8Pl8LFiwAFVVVRAKhRCJRJgzZw6WL1+O+Ph4I1414Obmho8++giLFy/u8NzatWtx8OBBjY3m0qVLkZmZqUEh7iuOHz+OF154AevWrcPTTz/NBop169bhq6++woULFxAREQFCCPLy8rBw4ULk5+ejrKzMZIZ7hka/DpWOjo4IDw/X+Gdvbw93d3c2wNA/JsWmTZtw4MABFBQUIDs7G+vWrUNSUhJWrFih9Rzr168Hh8PR+NddqeTs2bOIjY2FjY0NAgMD8c033+jvQ99h4PF4GD9+PDZt2oTi4mIcP34cfn5+eP755/Hnn38iMDAQR44cQWtrq97PzeFw4OTkhKFDh2LcuHEYM2YMHBwcUFpairNnzyI9PR3Xrl1Dbm4uIiMjzQEGqnml1NRUODk5dRtghEIhazSWkZGBGzdu4NixYxg4cCAEAoHRrplhGOzevRttbW2dBrZLly5h2rRpGo9Nnz4dKSkpkMvleruWy5cvo6CgAEuXLsXq1avxyy+/gGEYrFu3DrGxsXjvvffYHuLw4cPx448/YuHChXdsgAHuAD+Z6upqlJeXs/8tk8nw8ssva0hwHzp0qEsJ7rCwMJw8eZL9765KcCUlJZg1axaee+457Ny5k027PTw89JZ236ngcrmIjo7GZ599Bh8fH3z66ac4f/48Nm7ciKVLl+L+++9HQkICZs2apXdPHA6HAwcHBzg4OCAoKAgikYidpeJwOCgtLYVIJIKnp6dOemr6BKVL66JibEjQAOPo6IiwsLAuv3+RSIRHHnkEHA4Hf/zxB2saN2bMGKPRbrOyshAfHw+JRAIHBwfs37+/U6vkmpoaeHl5aTzm5eXFUuS7s4juCrQYxOFwsGzZMjQ2NkKhUEChUODAgQPYvXs3duzYgYcffhibN2/G5cuXMWfOHADAiBEj7hgWWWfo1+UyY2D9+vU4cOAAMjIydDreWGn3nYpff/0VH3/8MY4ePcpmDoQQZGdnsxYF+fn5Gp44bm5uemfYlJaWoqSkhPWkUReYdHZ2ZqnRtra2ej1ve9AAo4sPiyEhl8uRmpoKOzu7brXqJBIJHn30UQiFQhw9etRku3CZTIby8nI0NTUhKSkJ33//Pc6ePas10ISEhGDRokVYt24d+9iFCxcwfvx4VFdX94rooa1vQgjBpk2bkJGRweqMvfTSSygqKsLLL7+MN954A0OHDsWpU6fuyICiDf26XGYsFBQUwNfXF0OGDMH8+fNRXFzc6bHGSrvvVMyfPx9//fWXRmmKw+EgPDwc69evR2ZmJq5evcoO8gUGBmLu3Ln4/vvvUVtb2+eJf0IIioqKUFpaitjYWDg7O8PW1haDBw/GqFGjMGHCBHh7e0MgEODChQu4fPkySkpK0NbW1teP3gFNTU06G30ZEjTA2NradhtgpFIpnnzySTQ2NuLw4cMmLfNYWVlh6NChiIuLw4YNGxAZGYnPP/9c67He3t6oqanReKyurg4WFhZwd3fv1fm5XC7y8vIwfvx4ZGRkQCgUgsPhYOXKlbh+/TrWrl2LoUOH4vfff8ezzz6LixcvwtnZGefOnev0Ou9E3PVBZsyYMdixYweOHTuG7777DjU1NRg3bhzq6+u1Ht9d2m1G1+BwOF1mBxwOB8OGDcNrr72GlJQU5OXlYfr06di1axdCQkIwc+ZMbNmyBZWVlT0OOIQQFBYW4saNG4iLi4OTk1OHY6ytreHn54fY2Fjce++9GDRoEJqamnDp0iVcunQJRUVFEAqFfQ52jY2NSEtLQ0hIiMkDTFpaGmxsbLq1UZDJZGyT/9ixY3B1dTXilXYPQgikUqnW5+Lj43HixAmNx44fP464uLg+md+Vl5dDLBZj8uTJ+L//+z+cPXsWXC4Xv//+OyoqKvDZZ58BAF588UWsXr0aa9euxeTJk7Fs2bJen/N2w11fLmuPtrY2BAUFYc2aNVi9enWH5w2RdpvRPagnTlJSEvbv349Lly4hLi4OCQkJSEhI6NQTR/31+fn5qKurY+2jewK5XK4hb2NjY8PK2/RUQr+xsRHp6emsdI6poFAokJaWxhrBdRVgFAoFnnnmGeTl5eHMmTMmJUkAwGuvvYaZM2fCz88Pra2t2L17NzZu3IijR49i6tSpWLduHSorK7Fjxw4AtyjMS5YswXPPPYdLly5h6dKleqMwf/jhh9i3bx9qamqwdOlS/Pvf/8b27dtx/fp1vPjiixg2bBh7LB2wVJebuZNx53/CHoIaYxUUFGh93hBptxndg8PhwN/fH6tWrcKLL76Iqqoq1qLgzTffRGRkJOuJExgYqLHoE0KQm5uL+vp6xMXFsU3qnsDS0hI+Pj7w8fEBwzBswElJSWEl9L28vLolLFDJmmHDhrFDd6aAQqFgDekiIiK6DTBLlixBdnZ2vwgwgGqu6amnnkJ1dTWcnZ0RERHBBhigIyFoyJAhOHz4MFatWoWvv/4avr6++OKLL/ocYGjAWLNmDaZMmYJDhw5h/fr1uHz5MkJDQ3H58mWcP39eI8jQ++NuCDCAOZPpAKlUiqCgIPz73//GW2+91eH5tWvX4o8//kBOTg772LJly5CRkWFu/JsAhBDw+Xw24Jw5cwbDhw/XMGFbvHgxJk6ciCeeeELv1sQMw2jI26jrqbm4uGgs3jTAmFqyhmEYpKWlgcvlIioqqssGNMMwWLFiBS5evIjk5GSTBkZTQyAQwN3dvcMmoj0BIDs7G//5z39gYWGBM2fOQKFQ4MqVK4iNjTX2JfcL3PU9mZdffhlnz55FSUkJLl++jIcffhgtLS1YsGABgI5zOEuXLkVZWRlWr16N3Nxc/PDDD9i2bRtefvnlTs/R01mc5OTkDsdzOBzk5eXp74PfIeBwOPD09MSSJUtw9OhR1NTU4MUXX0Rqairi4+MRGRmJv//+GwEBAQaROuHxePDw8EBYWBgmTpyIsLAwKJVKZGVl4dy5c8jOzmblWPpLgElPTweHw+k2wCiVSqxatQrnz5/HyZMn7+oA87///Q9jx45FSkpKh36ceoBRKpUICwtDUlISnn76aUyZMgVOTk4mVW8wNe76TGb+/Pk4d+4cBAIBPDw8MHbsWPz3v/9laZALFy5EaWkpkpOT2decPXsWq1atYjWQ1q5d26UG0vr167F3794OszidlR2oknR+fr5Gc9rDw+OuoT32FVKpFA899BCysrIQFRWFEydOYNCgQUhISEBiYmK3PYi+ghCCpqYm1NbWoqamBnK5HM7OzggICIC7u7tJ/o4MwyAjIwNKpRIxMTHdBpg1a9bgzz//RHJystFUoPsrxGIxRo4cCUdHR3z77bcYNWpUp2VR9cyGYRgIhUI4OzvfsXMw3eGuDzLGQE9nccyeOH0DIQQPPvggKisrcfToUbi5uaG1tVXDE2fAgAGYO3cu5s2bh7i4OIMFHIFAgMzMTAQEBIBhGNTV1UEmk2nI2xijNs8wDDIzM8EwDKKjo7s8p1KpxBtvvIE9e/YgOTn5rjf8ow16uVyOuLg4yGQy/PDDDxgzZozO982doKbcW9z15TJjoSezOBTR0dHw8fHBlClTcObMGSNc5Z0BDoeDF154ASdOnGCFBx0dHfHoo4/if//7H2pra/HJJ5+gvr4eiYmJGD58OF555RVcuHABDMPo7TqobUBYWBiCgoIQEhKCe+65B6NGjYKdnR2Ki4tx9uxZZGRkoKqqymBzVlSIVKFQdBtgCCH473//i927d+PkyZN3fYABVA16hUIBS0tLZGRkwN7eHgsWLMDFixd1dom9WwMMYM5kjIIjR45AJBIhJCSE9fDOy8tDdna2Vkaa2RPHeJBIJDhx4gTriWNtbY05c+YgMTGxx5446uDz+WyA6ar/JhQKWdKAUCjU6krZF9AAI5VKERMT0+XnIYRg48aN+Oabb3D69GmMHDmyz+e/3aGegaiXu8aOHYu6ujr8+OOPGD9+/F1ZBtMV5iBjAnQ3i6MNZk8cw0Mmk2l44hBCMHv2bMybNw8TJ07UedGnASY8PLzD4G5XEIlEbMBpaWmBi4sLG3B6w4qjBASxWIzY2NhuA8ynn36KTZs24dSpU4iKiurx+e400KDS2toKqVQKS0tLDYWD8ePHo7y8HD/99BMmTpxoDjSdwBxkTISpU6di6NCh2LJli07Hmz1xjAuFQoHz58+znjhisRizZ89GQkICpkyZ0umiT43Pehpg2kMikbABp6mpCU5OTuwsji56akqlEteuXUNbW1u3U+2EEHz55Zf44IMPcPz4cYwaNarX132ngAaYgoICLFiwACKRCNXV1Vi/fj1mzJjBmslNmjQJJSUl2Lx5M6ZNm3bXzL70BOYgYwJ0N4ujDWZPHNOBYRhcvHiR9cRpamrCjBkzkJiYiKlTp7LDnadOnYJSqURkZKRejc9kMhkbcBoaGjrYILcHIQTXrl2DUChEbGxslxkYIQTffvst3nnnFfSQdQAAHglJREFUHRw5csToHjD9GWVlZRg3bhzmzJmDZcuW4dChQ/jwww+xbNkyLFq0CCEhIQCAcePGoaKiAteuXbujJft7C3OQMQJefvllzJkzB/7+/qirq8O7776Ls2fPIisrC4MHD+4ggbFp0yYEBAQgLCwMMpkMO3fuxMaNG5GUlIQHH3yw0/NUVlZi7dq1OHLkCMRiMUJCQrBt27Yuh8BMYUl7O0OpVOKff/5hA05NTQ2mTZsGf39/fPPNN0hKSsKkSZMMdn65XA4+n4/a2lo0NDTA1taWzXCo8Vt2djZaWloQFxfXbYD58ccf8dprr+HPP/809/vUIJVK8dxzz4HD4WD79u0AgGnTpqGoqAgNDQ148sknsWLFCnaSPycnp1Obgbsd5tzOCLhx4wYee+wxjVmcv//+mzWn0ocnTmNjI+655x5MmjQJR44cgaenJ4qKirqkQJu9cXoOLpeLsWPHYuzYsfjwww+Rnp6Od999F1999RWCg4PxzTffoKamBrNmzYKTk5PeWUWWlpbw9fWFr68vK8paW1uLf/75B9bW1uDxeJDL5Rg9enS3AWbnzp1Yt24dDh48aA4w7cAwDKZMmcKaI86cORNyuRxFRUX4+OOPsX79ejAMg2XLlmHkyJEYMWLEXU1T7grmTOYOwauvvooLFy7g/PnzOr/G7I3Td/z2229YvHgxfvvtN/j7+7OeONevX8fkyZORkJBgME8cdSgUCmRmZqK5uRkAWD01Km/TXsvtf//7H1544QUkJSVh+vTpBruu2wXavGFu3LiBgQMHIikpCR988AF27dqF4OBg/O9//8OqVavA4XBw8uRJhIaGmuiqbw+Y52TuEBw8eBBxcXH417/+BU9PT0RHR+O7777r8jVmb5y+QalU4rvvvsPevXsxe/ZsjBw5Ev/3f/+Hq1evIjMzE/fccw+2bt2KoKAgzJ07F9u2bUNdXV2fbQLagxCCgoICiMVijBs3Dvfddx9CQ0PZwHPu3Dlcu3YNBw8ehEwmw/79+7FixQrs3r3baAFmw4YNGDVqFBwdHeHp6YnExETk5+d3+RpjySspFApwuVxIpVKUlZWhsLAQADBo0CBwOBw0NDRAqVSyhIumpia89tpr+Pvvv80BRhcQM+4IWFtbE2tra7Ju3TqSlpZGvvnmG2JjY0O2b9/e6WuCg4PJe++9p/HYhQsXCABSVVVl6Eu+I6BUKrt9vqCggGzcuJGMHj2aWFhYkAkTJpBPPvmEFBQUEKFQSNra2nr9TygUktTUVHL06FEiEAg6PN/a2koqKirIgQMHiKurK3FwcCBOTk7ktddeIxKJxEjfEiHTp08nP/74I7l27RrJyMggs2fPJv7+/kQoFHb6mjNnzhAAJD8/n1RXV7P/FAqF3q6L/v1aW1vJmDFjyODBg8ngwYPJtGnTSEFBASGEkL179xJHR0fy/PPPk5UrVxIrKyuyb98+vV3DnQ5zkLlDYGlpSeLj4zUee+GFF8jYsWM7fU1wcDB5//33NR7766+/CABSXV1tkOu8m6FUKklpaSn55JNPyPjx44mFhQWJj48nGzduJLm5uT0OOEKhkKSlpXUaYNr/27NnDxk6dCi7wDs5OZHHH3+clJSUGP27qKurIwDI2bNnOz2GBpnGxkaDXAMNVkqlksydO5ckJiaSI0eOkIMHD5KoqCgSGBhIMjIyCCGEbNiwgYwbN45MnDiR/PLLLwa5njsV5nLZHQIfH58O7Jbhw4drEAraw+yNY1xwOBwMHjwYq1evxrlz51BaWorHHnsMR48exciRI3Hffffhs88+Q3FxcbclNXKzRFZbW4vY2NhuZ2dOnTqFhQsX4u2338Yff/yB0tJSnDp1Cv7+/iwrzZigvSMq+9MVDCWvxOPxoFQq8fXXX8PNzQ0ffPABZsyYgTlz5iA9PR0DBw7E448/DkDV8zx27BgOHTrEPtbd38iMmzBxkLvtwTBMtyUTY+Cxxx4j48eP13jsxRdf7JDdqGPNmjVk+PDhGo8tXbq0y+zHDP1DqVSS6upqsmXLFjJ16lRiaWlJIiMjyVtvvUXS0tI6ZDhCoZBkZGSQI0eOED6f320Gc+TIEeLg4EB++OGHfnGvKpVKMmfOnA73a3vk5eWRrVu3ktTUVHLx4kWybNkywuFwusx+eorvvvuODBgwgLi5uZEbN24QQggRiUSEEELKy8uJj48P2bVrl97OdzfCHGR6CIZhyOnTp8m5c+dMfSka+Oeff4iFhQV57733SEFBAfnll1+InZ0d2blzJ3vMq6++Sp566in2v4uLi4mdnR1ZtWoVycnJIdu2bSOWlpZk7969pvgIZhDVAiwQCMi2bdvIrFmziJWVFQkLCyPr1q0j//zzD2ltbSXLly8n69evJ3V1dd0GmOPHjxNHR0eyZcuWfhFgCCFk+fLlZPDgwaSioqLHr33ggQfInDlzen3u9t9BYWEheeutt4i1tTX5z3/+o3FcdXU1CQoKIr/++muvz2eGOcj0CHl5eeTee+8lkZGRZNiwYcTe3p489thj5NKlS6a+NEIIIX/88QcJDw8n1tbWJDQ0lGzdulXj+QULFpCJEydqPJacnEyio6OJlZUVCQgIIFu2bOnyHDdu3CBPPPEEcXNzI7a2tiQyMpKkpKR0ejytq7f/l5ub2+vPebdAqVSSxsZGsmPHDpKQkEBsbW1JYGAgcXZ2Jtu3byetra1dBpgzZ84QJycn8sUXX/SbALNixQoyaNAgUlxc3KvXv/vuuyQ0NLRXr21PGKDEh4aGBvL222+TgIAAsmrVKkIIIWKxmFy+fJk4OTmRQ4cO9ep8ZqhgDjI9wJNPPknGjBlDTp8+TYRCIbl48SJ58sknycMPP8wyUe5kNDQ0kMGDB5OFCxeSy5cvk5KSEnLy5ElSWFjY6WuMwRC6W/D2228TR0dHMmvWLOLg4EACAgLIypUrSXJycoeAc/78eeLi4kI+/vjjfhFglEolef7554mvry+5fv16r9/noYceIpMmTerx69Tvt6VLl5KEhAQybNgwsnnzZlJaWkqam5vJO++8QxwcHMjw4cPJ7Nmzybhx48iaNWt6fa1mqGCe+O8BsrOzERMTw8qGxMfHw8fHB//88w8rjEduTv1SnwlDui8aGx988AH8/Pzw448/so/paitLhwLN6B02bdqEL7/8EufPn0dkZCTa2tpw9OhR7Nu3DwkJCXBycsLcuXORmJgIOzs7zJ07l1X57g9T6M8//zx27dqF33//HY6OjizhxNnZmSUt6CKvlJSUhKSkpB6fnyokT5s2DY2NjVi9ejVu3LiB//znP6ioqMD777+PZcuWwcLCAj/99BMUCgV+/vlnBAYGgqg243fUb9moMHGQu63w/fffE3t7e/LZZ5+RlpYWnV/XH3aS+sDw4cPJiy++SB5++GHi4eFBoqKiOpTk2oNmMgEBAcTb25tMnjyZnD592khXfOfg4sWLJC0tTetzIpGI/P7772TBggXE2dmZcLlc8sorr/Sr+w5aSqYAyI8//sge076c+8EHH5CgoCBiY2NDXF1dyfjx43tVuqLfw65du0hERAT72/3444+Jh4cHSU9PZ49taGggGzduJFFRUeTFF1/s8B5m9BzmINNDfPrppyQ0NJTMmDGDHD16tMPzOTk55O233yYrVqzolAXDMIyhL9Mg6M3ApzEYQmbcglQq7Vc9GGND/XPLZDKN57777juSmJhICCHk5ZdfJl5eXmw/9fr16+TYsWOEEEIEAgH5+OOPSUREBHnssceMdOV3LsxBphc4d+4cefzxx0lAQAA5fPgw+/iXX35JAgICyIMPPkgef/xx4uPjQxYtWtRpUGEY5rbqTfRm4FMb+soQMsMMbVAPMPv37yfbtm3T6JV+8cUXJCwsjLz33nvEy8uLJCcns89988035PHHHyc1NTWEEEIaGxvJO++8Q1auXGm0679TYQ4yOqC6upqUl5d3ePyee+4hDz74IFEqleTs2bNk8ODB5KuvvmKfv379OhkxYgT56aefCCGE1NTUkO+//55kZWUZ7dr1CX9/f7J48WKNxzZv3kx8fX179D59YQiZYUZ3+Pjjj8ngwYPJSy+9RPLz89nHBQIBiY+PJxwOh/z555/s44WFhWTYsGHknXfe0XiftrY2o13znQxz418HUP93KucOqHw9YmJicPToUXA4HJw4cQKNjY1Ys2YNdu7ciXnz5mHNmjUYNGgQO3WflZWFDz/8EAEBAYiLi0NKSgpefvllTJ06tYNMuDZVWFPjnnvu6SBqeP36ddayQFekp6fDx8dHn5dmhhkAgC1btmD9+vXYs2cPxowZA1dXV/Y5V1dXrFq1Chs2bMC6detQVVWFmpoa/Pzzzxg1ahTefPNNALd+e9SMzoy+oX+tYv0U1B99xowZeOaZZ7B//34sXboUSUlJeOSRRwComGf33nsvbty4gYSEBCQlJcHe3h4nTpyAWCwGAOTn56OiogJCoRChoaEIDAzEwoULcf78+Q4MIBpgGIZhmWqmxqpVq/D333/j/fffR2FhIXbt2oWtW7fi+eefZ49Zt24dnn76afa/N23ahAMHDqCgoADZ2dlYt24dkpKSsGLFik7PExAQoFV9V/087XH27FnExsbCxsYGgYGB+Oabb/Tzoc24bVBQUICtW7fixx9/xIwZM9gAIxKJcPHiRWRnZ+Nf//oXtm3bhuHDh+Ojjz5CWloannrqKfzyyy8AVL+3/ra5u+1h6lTqdsKpU6fI/PnzSUREBJk7dy7ZvHkzW8OdNm2axjQ9Iao0/OeffyaXL18mbW1t5JFHHiEjRoxgn5dIJOS+++4jTz/9NJHL5YQQlZTFV199Rfbs2UOkUqnG+9H+zc8//0xCQ0PZcxsTPR347A1DqK6uTmOm5sSJEwQAOXPmjNbjqXLBypUrSU5ODvnuu+/MygV3IXJyckhISIjGcPQXX3xB5s6dSzgcDvH09CSPP/44+1xzc7PG62+n/ujtBHOQ0QHaGvftb9Cff/6ZBAYGkpMnT2p9jwsXLpCIiAiyceNGQsgt5suGDRvY/oRUKiWHDx8mzz77LAkPDyfOzs5kxYoVrJYS/RHMnj2bTJ06lTQ1NennA/ZzrFy5kgQFBXXKmFqzZk2HHs+SJUvMGmx3GdLT04mDgwP5+OOPSUpKCklMTCSxsbFk+fLl5OTJk+SHH34g3t7e7MaIbuwIMVOUDQlzkOkB2rPB1G9MhUJBnn/+eeLt7U0SExPJl19+STZu3EiKiooIIaoGuZWVFSunQiUt5s6dS+bNm8e+j/qNf+TIERIeHk52796tcR1WVlbku+++uyt+GFKplLi7u3fwvVHHhAkTNHSnCCFk3759xMLCogON1Yw7G59++inhcDhk4MCBJDo6mpw6dYq1CigrKyNDhgwh3377rWkv8i6DufjYA3C5XHZyGIBGH4XH4+Grr77Cn3/+iUGDBuHAgQMQiUTw8PBAU1MTrly5AgAQCAQAAGtra7S2tuLEiROYMWMGAJXj3sGDB/H1118jLS0NM2bMQEREBH7//Xf2PL///jssLS0xZsyYTie5Dx06hIKCAr1/flPgwIEDaGpqwsKFCzs9pqamBl5eXhqPeXl5QaFQsN+3GXcHVq1ahby8PBw9ehRpaWmYPHkyqzTB5XIxYMAAs42FkWFml+kJ5CY7LDY2FrGxsQCAtrY22Nvb46+//kJxcTGioqKwe/duREREICcnB5999hlcXV3x7LPPorGxEbNmzYJQKMSgQYOwfv16uLi4gM/nY/ny5ZBIJLCxscG2bdswYcIE+Pv7az1/ZWUl3n//fQwcOBD/+9//TPFV6BXbtm3DzJkz4evr2+Vx7QMuuen10R8kVcwwLkJCQjo8VlNTg8cffxxubm546KGHTHBVdy/MmYyeQBczpVIJhmEAAPb29gBU1OXa2lps2LABQqEQ7u7uePjhh1FfX48tW7aAy+Vi+/btqKmpwVdffYV9+/ahpKQEK1asAI/HQ0hICGxsbAAAJ06cwJw5c+Do6KhxfrqopqamgsvlstkRvZbbEWVlZTh58iSeffbZLo8zm6+Z0Rlqa2vx2Wef4emnnwaHw8HRo0cB3N6/i9sN5kxGz2hPf2xoaMCVK1cwaNAgTJ48GZMnT8Zbb72F7OxsTJkyheXiy2Qy8Hg8+Pr6soKB9fX1cHZ2RmhoKABVGYzD4WDcuHEdzkP/+++//waHw8GECRO0Xg8FwzDgcDj9mq75448/wtPTE7Nnz+7yuPj4ePzxxx8ajx0/fhxxcXGwtLQ05CWa0c+Rn5+PI0eOIDw8HJ9++ikAQKFQsIK2ZhgBpm0J3fm4fv06uffee8natWsJIaQDLZnixo0bJCIigkRHR5OPPvqIPPzww8TGxoYsWrSI1NbWEkIImTdvHpkyZUoHz3NKAKipqSFz584lCxYsMNjnMRYYhiH+/v7s96YOs/maYfH++++TuLg44uDgQDw8PEhCQgLJy8vr9nXJyckkJiaGWFtbkyFDhnTrTWQs8Pl89n+bacrGR//dxt4hCA4OxtmzZ/H6668DACwtLbV6gw8cOBCnTp3Cww8/jOLiYkybNg0KhQJRUVHw9PQEAJw+fRpz5syBk5OTxmvp+125cgV1dXWIj48HgA5DnG1tbTh+/DgWL16MV155BdnZ2R2ug9yUNTc1Tp48ifLycjzzzDMdnquurmZVFABgyJAhOHz4MJKTkxEVFYX//ve/+OKLL7qsvfd04DM5OVnr8Xl5eX3/sP0MZ8+exfPPP4+///4bJ06cgEKhwLRp09DW1tbpa0pKSjBr1ixMmDAB6enpeO211/Cf//ynV7L8+saAAQMAqO5tdeKOGUaCSUOcGSza05GlUinZv38/uXr1KiFENQTJ4XC6dKF84403yPjx40lOTg77nnTGp7S0lCxbtowMHjyYrFq1isyePZv4+/uTzz//vNP3UygUt61idHfo6cDn3Wy+VldXRwB0qZxtnlUyozOYC5P9BBwOB4QQKJVK8Hg8WFlZITExkX3eyckJq1evxpAhQ7S+XiAQ4Nq1awgICMDw4cPZ96TYunUrzp8/j99++w1jxowBAHz//ffYsmULpkyZgrCwMPz9999IT0/HI488And39zt61+fh4aHx3xs3bkRQUBAmTpzY5evuRvO15uZmAICbm1unx1y6dAnTpk3TeGz69OnYtm0b5HK5uTd2F8NcLutH4HA4Ggs7UStb3Xvvvfj44487/NBpSSwlJQV8Ph/jx4/XeJzD4UAgEODMmTOora3FtGnTMHXqVOzYsQNPPPEEKxIIqIgF//d//4d33nkHDz30EJ588knk5uZqvdb+oqemD1DXxWeeeaZbynN0dDR8fHwwZcoUnDlzxkhXaDoQQrB69WqMHz8e4eHhnR5nnlUyozOYg0w/hvqC1xnlkrLDUlNTcfHiRRw6dAgXL15EU1MTe0xxcTEkEgnWr1+Py5cvIyYmBhs3boSLiwtqa2vZxSE1NRUymQyOjo546qmn0NDQgKVLl6Kurg6AZtCj5+0s2NxOQUiXgU8fHx9s3boVSUlJ2LdvH4YNG4YpU6bg3LlzxrtQE2DFihW4evUqfv31126PNc8qmaEVJi3WmaE3KJVKkpSURCZPnkwsLCyIp6cneeqpp8jff/9Nmpubib+/P9m2bRt7PMMwJC0tjfz6669EIpGQK1eukLCwMPLWW2+xx2RmZhIOh0MuXLjAPtbS0kJ27tzJughqw+3mwzFt2jTywAMP9Ph1d7r52ooVK8igQYNIcXFxt8eapX3M6AzmTOYOAYfDwYMPPohTp05BLpfjq6++QlNTE37//Xc4OTlh2rRp2LVrF0pLSwGoMpHo6GjMnz8f1tbWOHz4MNzd3TF9+nT2PZubmzFy5Ei23JGZmYlx48bhzTffxIIFCxAZGYmamhq2bESzl4iICHz55Zcag6mlpaUoKysz4jeiG3Qd+NSGsWPH3jHyPeoghGDFihXYt28fTp8+3WkfUB3x8fE4ceKExmPmWSUzAJgzmTsJCoWiU7ZTaWkpue+++4ivry9ZvHgx+eSTT8jrr7/OPj99+nSyaNEiDXXp//73v2T8+PHk6tWrpL6+ntxzzz3k/vvvJ9nZ2YQQQj7//HPWaTArK4sIhUKyYcMGwuVySWFhocb5X3vtNRIWFkYyMzMN8Ml7j7fffpt4e3trCJPqioceeohMmjTJAFdlWixbtow4OzuT5ORkDSYdVQMnxDyrZIbuMAeZOxTq9GN1evSxY8fIokWLyJw5c8j3339PCFHZEMTFxZEvvviCPU4mk5GEhATy+OOPE7lcTn799Vfi5+dH0tLS2GPy8vKIh4cHeeihhwifzydXr14lHA6H2NjYkDFjxpAff/yREEJIa2sreeKJJ8jDDz/MLub9QUG6JwOfn332Gdm/fz+5fv06uXbtGnn11VcJAJKUlGTMSzYKAGj9R/+ehHT0DSJENYwZHR1NrKysSEBAQL8ZxjTDtDBTmO9QtFeLppay06ZNY6mmcrkcAPDPP/9AIpEgLCyMfQ1lq02cOBEWFhZIS0uDh4cHoqOjWTHOoUOHQiKRYMyYMXBycsKAAQMQHh6O2NhY+Pr64tNPP4W7uzv8/PxQUVGBWbNmsXIelLJtyqZwTwY+ZTIZXn75ZVRWVsLW1hZhYWE4dOgQZs2a1e15FAoF1q9fj19++QU1NTXw8fHBwoUL8cYbb3Qp63P27FmsXr0a2dnZ8PX1xZo1a7B06dLefdgegOgwjPvTTz91eGzixIlIS0szwBWZcVvDxEHODCOjvScORWlpKRGLxex/r1u3jsTGxpLk5GRCCCEPPvggeeihhwghtwzXTp06RSwtLVknwtzcXMLlcjWcCQlRldVGjRpF3nvvPfLGG2+QL774QqP0cqfj3XffJe7u7uTPP/8kJSUlZM+ePcTBwYFs2rSp09eY3T7NuFNgzmTuMnS2cx48eLDGf4eFhUEkErG2BX5+fjhz5gxqamrg7e2N5uZmfPHFF4iLi0NgYCAAYN++ffDz80NYWBibpUgkEly6dAk5OTkICwtDUFAQtmzZgpMnT2LXrl2sUvWdjEuXLiEhIYEV+gwICMCvv/6KlJSUTl/zzTffwN/fH5s2bQIADB8+HCkpKfj444/NUvVm3F4wdZQz4/ZAZWUlGTZsGJk4cSL5+uuvyYwZMwiHwyEffPABK/oZHR1NVqxYQQi5ZVl96dIlEhMTQ5YtW8a+18mTJ4mjoyO5fPmy8T+ICbBhwwYyePBgkp+fTwghJCMjg3h6epJdu3Z1+hozJdiMOwVmCrMZWsEwDFubb2pqgq+vL44cOYKoqCiUlJRgypQpcHR0RHx8PKysrNDa2oqMjAwkJCQAuFXXv3jxIqytrTF//nz2vXk8HoYNG9YvKc2GwNq1a/HYY48hNDQUlpaWiI6OxosvvojHHnus09eYJ+jNuFNgLpeZoRXqxIG//voL9fX1WLBgATZt2oSmpiY8/fTTiIuLQ2RkJACVCu/AgQNZpQEejwepVIr09HR4eHiwemmAysSNEKLT/MWdgN9++w07d+7Erl27EBYWhoyMDLz44ovw9fXFggULOn2deYLejDsB5iBjRrewsrLCe++9h7fffhuTJ09Gamoqbty4gYMHD7K2A4GBgZg/fz7mz5+PiRMnYseOHaiurkZVVRXuu+8+WFtbAwCEQiEyMjLg7u6OuLg4U34so+GVV17Bq6++ymZzI0eORFlZGTZs2NBpkDG7fZpxp8BcLjOjW0ybNg35+fn48MMPYWlpiUcffRQXLlzAPffcwx7j4OCAjz76CGVlZUhMTIS1tTXS0tJQXFyMsWPHssfl5OSgqKhII7O50yESiToQLng8Xpf6buYJejPuGJi2JWTG7Qx1rxl17xqKtrY2cuzYMSKRSNjHPvnkExIZGamhh3anY8GCBWTgwIEshXnfvn1kwIABZM2aNewx5gl6M+5UmMtlZvQa6rtz6hRJ1AYs7ezsNDxGGIYBl8uFm5ubRnZzp+PLL7/Em2++ieXLl6Ourg6+vr5YsmQJ3nrrLfaYztw+V61aha+//hq+vr7dun2aYUZ/BIeQfuC1a8ZdBYVCwU7+m2GGGXc2zEHGDKOBqLTyupRSMcMMM+4smIOMGWaYYYYZBoN5S2mGGWaYYYbBYA4yZphhhhlmGAzmIGOGGWaYYYbBYA4yZphhhhlmGAzmIGOGGWaYYYbBYA4yZphhhhlmGAzmIGOGGWaYYYbBYA4yZphhhhlmGAzmIGOGGWaYYYbB8P+oQQkymQvTzQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAGRCAYAAAC39s6jAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOy9d5wk5XktfKqq8+Sc0+7OptnAhll2liUJCQwSSkiWkGRLWBcJWXAlcTE219iS9SmDLMwVUcJgWBGEEMlgRNwlbZ6cc86dpnN3he+P2re2uqdz18z0LnX842ftTM/bb3dXv6ee5znPeShBEASoUKFChQoVKwB6rTegQoUKFSrOX6gko0KFChUqVgwqyahQoUKFihWDSjIqVKhQoWLFoJKMChUqVKhYMagko0KFChUqVgwqyahQoUKFihWDSjIqVKhQoWLFoJKMChUqVKhYMagko0KFChUqVgwqyahQoUKFihWDSjIqVKhQoWLFoJKMChUqVKhYMagko0KFChUqVgwqyahQoUKFihWDSjIqVKhQoWLFoJKMChUqVKhYMagko0KFChUqVgwqyahQoUKFihWDSjIqVKhQoWLFoJKMChUqVKhYMagko0KFChUqVgwqyahQoUKFihWDSjIqVKhQoWLFoJKMChUqVKhYMagko0KFChUqVgwqyahQoUKFihWDSjIqVKhQoWLFoJKMChUqVKhYMagko0KFChUqVgwqyahQoUKFihWDSjIqVKhQoWLFoJKMChUqVKhYMagko0KFChUqVgwqyahQoUKFihWDSjIqVKhQoWLFoFnrDaj46IHneXAcB4qiwDAMKIpa6y2pUKFihaCSjIpVgyAI4HkegUAAbrcbFEWBpmloNBpoNBowDKOSjgoV5xkoQRCEtd6EivMfgiAgEAiA4zgIggC/3w+KoiTiIcRC0zS0Wi0YhoFGowFN0yrpqFBxDkMlGRUrDhK9cBwHmqYlkqHpsyVBQRCk/3ieBwAp0tFqtVKko5KOChXnFlSSUbFiEAQBHMeBZVnwPC8RBCEdABEJI5R0+vv7UVlZiaysLJV0VKg4h6Cqy1SsCARBgM/nw6lTp6SoRU4Gse5tSBTDMAy0Wi3sdruUavN6vXA6nVhaWoLD4YDb7Ybf75d+r0KFivSBSjIqFAfHcfD5fGBZFnNzcxAEIeVog6IoSY1GIhmapsHzPHw+H5xOJxwOh0o6KlSkGVR1mQrFIAgCWJYFy7IQBAEMwyi+PgEhHfnvSHqOkFw49ZqaXlOhYnWhkowKRcDzPFiWBcdxABB0mCsRTcQiBkI6REwQiXSIao38/1CyUqFChbJQSUZFSpD3vpC0mPzQJjJlpZ4rXkQiHZZlEQgEgtJv8khHJR0VKpSFSjIqkoa89wVYnsIiPwslB6fTibGxMWRlZSEvLw9GozHmcylV04mHdEifDkmvqVChInmoJKMiKYT2vkQiATnJCIKAiYkJ9PX1obCwENPT0+jr64Ner0deXp70n16vD7uWkkX8eEkn1I1AJR0VKhKDSjIqEkKk3pdIICQTCATQ1dUFq9WKXbt2ISsrC4CoRLPb7bBarZiYmEB3dzdMJpNEOLm5udDpdCuewopEOoFAAH6/HwDCWuCopKNCRXSoJKMiboSmx+JRalEUhaWlJTQ3NyMjIwMHDhyATqeTDm6NRoOCggIUFBQAAAKBAGw2G2w2G0ZHR+F0OpGRkQG/3w+73Y6cnBxotdqVfaGITTqDg4Oora2FyWRSSUeFiihQO/5VxAWe5+H3++OKXggEQcCbb74JQRBQX1+P2tpaKbIhJBNrHb/fD5vNhr6+PtA0DZ/PJ9Vy8vLykJOTA41m9e+V3nnnHTQ2NsJgMEg/C02vEfWaChUfZaiRjIqoIOkxoh6Ll2B8Ph86OjrAcRwaGhpQVVWV1PPrdDoUFxdjfHwc1dXVyMnJgdVqhdVqRV9fX1jSUbo/JxJI+gwIVtkR80/iWBCqXlOh4qMElWRURES03pdoMJvNaG9vl4r4mZmZiu1Jr9ejtLQUpaWlAACPxwObzQar1Yqenh74/X7k5ORIpJOdnb1iKaxQqbac3KKRjly9ppKOivMdKsmoWIZYvS+RwPM8hoaGMDo6ik2bNqGqqgpHjhxRrBkz3DpGoxFGoxFlZWUQBAEej0eKdCYnJ8FxnEQ6+fn5yMzMXJW6iZx0yL7DkY46S0fF+Q6VZFQEgch4u7u7kZOTg9LS0rgOPq/Xi7a2Nvj9fuzfv19Sj61mMyZFUTCZTDCZTKioqIAgCHC5XFKkMz4+DkEQkJubK0U6mZmZq6JcAxCWdPx+f0QLHJV0VJwPUElGhQRyp83zPNxuNwwGQ1yH3Pz8PDo6OlBcXIw9e/YEFeKVIplkDluKopCZmYnMzExUVlZCEAQ4nU4p0hkZGQFFUUE9OiaTKW5RQ7KIRjo+ny+qZFolHRXnGlSSURG294UMF4sGnufR19eHyclJNDQ0oLy8fNlj1spWJhwoikJWVhaysrJQXV0NnufhcDhgtVqxsLCAwcFBaDSaoEjHaDSuaqTDMEzQLJ1Q0pHXc1TSUXEuQCWZjzgi9b7EIgeXy4W2tjYAwIEDB5CRkRH2cWsZycQCTdPIyclBTk4OamtrwfM8lpaWYLVaMTc3h/7+fuh0uqBIRy5ZXinIa2ChpOP1eqXHqFNDVZwLUEnmI4xo1jDRyGF6ehpdXV2orKzEpk2bohbS47GbiRcr3dJF0zRyc3ORm5uLurq6IDeCqakp9Pb2ShY4gNjDE4/vWqqIh3S8Xi8CgQCKiopU0lGRVlBJ5iOIeHpfyJhkOViWRU9PD+bn57Fz504UFxfHfK5w6ySDtTgsGYZBfn4+8vPzAYivn5AOAJw+fTqsBc5KIxzpkDqTXHAR2qOjko6KtYBKMh8xxGsNE1qTcTgcaG1thU6nw4EDB+K+g1fyUFtrcwq5Bc74+Dj27dsnSaZHRkbgcrmQmZkZRDqr4UYgt8DRarVSlMPzvBTpkDqbSjoqVhsqyXyEQKKXRIwt5c7JtbW1WL9+fUJ9Julck0kVGo0GRUVFKCoqAnDWAsdqtWJwcBButzvIjSA3N3fF3AjkI64jpdfUqaEq1gIqyXwEIB+LDMRvbMmyLNra2mC1WrF7927JxDIRpJO6bKVBLHBIGtHn8y2zwMnOzg5yI1CKdKK9N/FODZWr19SpoSqUgkoy5znkvS9A+MFi4RAIBDA3N4e8vDwcOHAg4oyXWDifI5lYCGeBQ0hnenoaLMsuI51k3QjkkUwsRCMdlmXVqaEqFIVKMucp5NYwiTonj46OYmZmBtnZ2dizZ09KB8v5GMkkuw9igVNeXh7RAkfeo5OVlRX3e58IyYRCnRqqYiWhksx5iGTmvgBiTaG9vR0ulwvl5eWK3Ll+lCOZaIhkgUNIZ2xsDAAk0snNzY1pgaPUe6SSjgoloZLMeYZ4xyKHgjgn5+bm4sCBAxgfH4fL5Up5P0qSTLpEMgRKEp/cAqeqqirIAsdisWB4eFjq4wlngbOS700s0gHUqaEqIkMlmfMEJKc+ODiIgoKCuFMt4ZyT4+n4jxfR1kklxXO+I14LHEI4xHpmtfYWjnTkDtMURamkowKASjLnBeTpsampKZhMJmRnZ8f8u0jOyUD6RSDpGMmsJsJZ4JDG0NnZWdhsNkmuvJoWOEB40uF5HjabDT09Pdi7d+8y0lGnhn50oJLMOY7Q3pd4jC2B6M7JQHqSw0eZZEJB07REJgDQ398Pr9cLvV4vWeAYDIYg37XVcCMAzroNEBk8wzDLZunISUedGnp+QyWZcxTy3he5NUwskonHORkQDzEl7GDIXlNFOh1A6Uh2FEXBaDRi/fr1AEQLHNIYOjY2hq6uLmRkZATVdLRa7Yruief5IJEAgTo19KMFlWTOQUQbixzNKyxe52SyjhrJnDsgNxoEGo0GhYWFKCwsBCD2PRHSGRkZQWdn54pb4ITuiSBe0lFn6ZwfUEnmHEI8Y5EjRTLT09Po7u5GRUVFTOdkIP1IRj1goiPWe6zVapdZ4BC59MDAALxer2SBQ5yoU3UjICncWJCTjjo19PyDSjLnCEJ7XyL1sISmueTOyTt27IjLOTncOskiEskkQzxqJBMZiSr1dDodSkpKUFJSAkAUgVitVthsNsUscJJRD8o918gagEo65zJUkjkHkEjvi5wcknVOBlY2kllYWEBHRwc0Gg3y8/PjqhGk0wHiY30Ycg9hD7snabsdpZGqHNxgMKCsrAxlZWUAwlvg5OTkBLkRxIpSSE0mFYQjnUhTQ1XSSU+oJJPGCDcWOR5jS57nMT4+nrRzMllHaZLheR4DAwMYHx/Hxo0bwTAMbDabVCOI5VicLpHMsH0YPa4eDNuHsStj11pvR4KSh2qoBY7b7ZYinYmJCfA8v4x0Qp8/Uk0mFURymCakE8nsU3WYXjuoJJOmSNYaBhDrL4FAIGnnZPJ8SqbLSE9OIBBAU1MT9Ho9OI4L61jc29sLv98vHWJkaFg6wMt60bHQATfnRsdiBzYXbYZRu/LTMWNhpTv+MzIykJGRgcrKypgWOHl5ecjIyFAkkolnb5FIRx1VnR5QSSYNQfLPiRhbAoDNZsPCwoKUHksllaNkJON2u/Hhhx+isLBQ6skhdiQEcsdiuXmkxWLBxMQEOI6DwWCATqeTDrG1OCRGbCOYc82hUl+JBfcChmxD2Fa0bdX3EYrVdE8IZ4FD3AjMZrNkgWMwGMBxHFwuV5AFzkrvLRbpsCwLr9eLwsJClXRWASrJpBHiGYsc6e9GR0cxMDAgpZxSrRUoQTKCIMBut8Nut6OhoQEVFRVxvZ5w5pFdXV3w+/0wm80YGhoKslTJy8tLqN6ULEgUY9KY4KE9yNZlo2O+A+tz1695NLOWFj0URSE7OxvZ2dmoqamRLHBGR0dht9tx8uTJNfm8yN5CScdms2FoaAiZmZkA1KmhKw2VZNIESjgn79u3DzMzM4rsJ1q/TTzw+Xxob2+H0+lESUkJKisrU9qLTqeDXq9HfX19kKXKzMwM+vr6pO72/Px85Obmrkh3+4htBLOuWVRnVcMMM4pMRRhzjKVNNJMuhyKxwMnPzwdN09i6dSuWlpaCPi+9Xi/V3lbbAgeA1PgpH1Xt8/ng9XpV0lEYKsmsMeLpfYmEUOdkrVaLubk5iahSQbz2NOFgsVjQ1taGvLw8VFZWht1PstJWsje5pYq8u31kZAQulwuZmZmSck2Jng8f60PHQgd8rA/TzmnM++eR6ciEn/OjY74DG/I2wKBZnYMyHNLRbJTUZBiGCfq8OI6DzWaDzWZbEwscotIElrcCqFNDlYdKMmsIYg0zPDwMm82GnTt3puScDKQegRAkky4TBAEjIyMYGhqS9jU4OCiNfQ59bKL7ifS6Qrvb/X4/LBZL0NhjuYggHvntsv1CQEVmBYpMReB4Dl69FxvzNkKr1UJLa9dc+ZaOJBNJXcYwDAoKCiRRSiQLHLnSUEkLnGhNopEcptWpoclDJZk1gnxqJUVR4Dguros0mnMykFoEIkeiZOX3+9HR0QGn04l9+/YhJydHWme1D2CdThdWREAmUPI8j9zcXCnSiUdEYNAYcFHVRQDO3AmP0DhYfXDVTCdjYa1JLhziVZeFs8Ahcunh4WEpMlXKAideJwIgsQFuoek1FSJUklllhBuLTFxqYyGWczKgnPQ4EbKy2WxobW1Fdna2lLYjWGt7mnAiAjIMjIgIGIYJagqNtyidTneu51IkEwtarRbFxcVB8nYS6YRa4OTl5SEnJyehdGgiJBOKREhHnRoqQiWZVUSk4n4sYojXORlY3XSZIAgYGxtDf38/Nm7ciJqammUH3VpEMtFAUcuHgS0tLcFisQQVpeWkky7RSjSkI8ko1Sej1+vDWuBYrVb09PRIPVVERJCTkxP1UOd5PuUaHUEs0gHUqaEqyawSolnDRCOZRJyTyVqrETkEAgF0dnbCbrejsbFRKuomuo5S+0kWZKRxbm4ugPD1AZKqyc/PR05OTtod5gTptq+V6PgHgi1wSDqUfGbxWOCkEsnEQiTS+ShPDVVJZoURT+9LJJJJ1DkZUDaSibSO3W5Ha2srMjIycODAgah3+kqSw2pEROFEBKQpVG4cCYjvA5HprjXSKVokUDJiiAR5OjTUAsdqtUoWOHI3ApZlV+0zC0c65IaTRDqhpHO+TQ1VSWYFEW/vSzjn5N7eXszNzSXknEzWUuLAkX8pyJ4FQcDExAT6+vqwfv161NXVxd0smirW6ksX6lbs8XiwuLgIu92O7u5uCIIgHWD5+flr5kSQjumylYpkoiGcBY7T6QySuPM8D4PBgImJiVV3jyD1GoLQWTpmsxlarVZyIzgfpoaqJLNCCB2LHO0iUco5OXStVCAnFooSx+h2dnbCarViz549cfuJRSK9ZL406XC3bjQaUVZWhoGBAezfvz/Ic21kZETq4SE1ndXqbLf5beCcHEpRuirPFw9Ww7ssFuQ1uKqqKvA8j46ODvA8HyT8CHUjWCvSWVxcRGZmJnJycoIGuP3ud7/DwYMHsX///lXZl5JQSUZhkKIf6Q2Jp1OYpmlwHIeJiQn09vYm7ZxM1lKSZHieh8vlQktLC4xGY1KeaOdyJBMN6SIiEAQBd/XdhYXuBbxb8y4ytNHrdquFtYhkYoEU4TMyMlBbWyt9ZlarFXNzcxgYGIBWqw1Kr63WjQIg3pySCAY4G+k888wzqKioUEnmow65NBmIPFgs0t8NDg6m5JxMnlPJdNnU1BT6+/tRW1uLDRs2JNWpfy7VZFJBOBGB3W6HxWJZJiJQcuTx0amj6FzqBE/xeLr7aXxz5zdTXlMJpEMkEw7yWpH8M6urqwPHcREtcMh/KzlDiOO4oMiGRDrEZPRchEoyCiBc70sizsltbW0QBCFl52RAuUiGrDE4OIhdu3ZJhfBEEYlkUrGVSRfE2pNGownqbJePPO7v7w+aPpmfn4/s7OzEnQgEAfeevhecwEFDa/BQ60P48tYvp0U0k46RDBBdXRZqgUNuFIiIoLu7GyaTKehGQcnoNJRkAEijFUIbr88VqCSTIpI1tiTOyYODg6iursbIyIgid0hKkIzT6URLSwsAYO/evVL3fjL4KEUysUBEBMXFxbD5bDAIBkm5NjU1tUwFlZmZGfZa8gQ8AACj1oijU0dxfOo4DIwBBq0BM86ZtIlm0jWSkXuXxULojUIgEJBIZ3R0FE6nU1ELnHAkA0ByPTgXoZJMCkhkLLIccguWxsZGGAwGSfWS6p1fqhLm6elpdHV1ScSX6l2akn0y5wue63sOL/S/gP/4xH+gvLxckt66XC7Jc00uIiCRjtFohCAI+OILXwQAvHTdS7j39L1geRYmygQNrQEFakWimfGlcfzLu/+CX172S5RmxicuOBcjmVggyi+5xJ0o14aGhuB2u5e5ESSSEo1EMm63O2aPXLpCJZkkIC/uJzL3BQjvnEzmlCtBMslKmDmOQ29vL2ZnZ7Fz504UFxdjdHQ0ZYKIRDKkSS0RnOuRDAA4/A78qfdPGLIN4dWhV/E32/4GQPAgMLmIwGq1YnZ2Fv39/dDr9ejn+nFs6hgoisL9Lffj+NRxaBktWI4FxVEwaAyYckwpHs38+viv8fLgy6jIqsDPLv1ZXH+TrpGMkv07Op1umQWOPCWaqAVOuL1xHAePx6NGMh8VkPQY6ZHYsmVLys7JhFiU8hxLdB2Xy4XW1lbQNB0km1aisfN8jGRSeT2vDb+G8aVxZOmy8EL/C7hm/TUoMC4XeoQrSFutVtzx33dIBH3P0XvACRwomgLHcyLRUBS0tBZvjLyhGMmM2EbwdM/T4AUej7U/hpt334zyrMjWRgTnYyQTC/IJr0BkCxxCOqF1uHCRjNPpBAC1JvNRgLz3hWEYScceC/E4JwPKkEyixDA7O4vOzs6wrgJKNHYqRQ7p5oGWDBx+B+45eQ8mliZwSfUlGLGNBEUz0cAwDNpd7ei2d0PLaCFAgI214datt6KarobP54PRaJQOrk2lmxTb9z0nRTIzMkb4eT9+2/zbuKKZdGwQBRKryaSKcBY4ckdwjuMk0snNzQ0bybjdbgA4ZyOZ9LvNSEPIvYfkzsnxDAebn5/HBx98AJPJhKamprB3IyTdtpruyTzPo6enB52dndi2bRu2bNmy7IunxMGejuRwf/P9uPF/bly1fU06JhHgAni+73n0WfrgYl2Ydc0iU5eJF/pfgNljDnq8n/PjpYGXMO+al34mCAJ+dexXAACGYqChROuR96zv4atXfBWbcjZhX/U+VOuqwU/z6DzZidbWVoyPj8PhcCT9WkkUQwln5PgC8Fj7Y5h2TMf825WMGFLBWu2LWOBUVFRg27ZtOHjwIBobG1FUVASHw4H29nYAQE9Pj/S5kT41vV6fstz95z//ORobG5GVlYXi4mJ89rOfRV9fX9S/OXz4sNSKIf+vt7c37udVI5kY4HkeLMsuU4/FsudPxDmZrKtkuizaXaTb7Q6STUfS36cTySi1zpxrDv9x6j/gDrjx/uT7uLjq4pTXjIZZ5yxeH34dW4u24v7m+8HxHChQGLAM4EDlAYzZx5ZFM61zrfjLyF/gZb346y1/DQB4Z/wdnJ49DYY6azHCUAxOzZ7C4fHD0EKLoqIi5OTkSCIColwLFREk0tVOohgtLSqmtLQ27mgmnWsy6UB+oRY4Xq8XH374IfLy8mC1WjE8PIybb74ZFRUVKC4uRnd3NxoaGpJ+T48cOYLvfve7aGxsBMuy+Od//mdceeWV6O7ujikq6Ovrkzz7AKCoqCju51VJJgJijUWORgputxutra0A4nNOjrVeIgi1gwkFmUlTWlqKzZs3Ry1CplNNRin8vu33cPqd4AUe95y8BwcrDyb9pY3n7zoWOjBqH0X3YjeG7cPi34GCw+9A53wncg25eH3kdXyt4WugKAp+zo93J96Fw+fA8enjOFB5AMPWYTzS9gh4gQcncPDz/rN7AIU/dP0B38j+hrQfuYiAWKk4HA5YLBbMzc1JIgK5ci2cinBiaQJP9zwNlmPBU2evA1Kb+f7e76M4I7Kv3kexJpMKCCnX1NSgpqYGHMfht7/9Lf70pz+hq6sL+/btQ1ZWFi6//HL84z/+I3bt2pXQ+q+99lrQvx999FEUFxfj9OnTuOSSS6L+bXFxsdRknChUkgmD0N6XcJ37xAomFDMzM+jq6krIOZmsp1QkAywvTvM8j4GBAYyPj2Pbtm0oKyuLa63zKZKZc83h8c7HwVAMjIwRJ2ZOrGg0M+ucRZ+lD3W5dXhp4CVQoFBoLARDM1jyLcGoNeIXl/0CRaYi6fpqnWvFoHUQWwu3YtA6iOf7nsd/df4Xthdtx72fuDfs8+wu2Y3FnsWIpEfTNHJycpCTkyOJCIjsljQYkl6P/Px8yYmApmhcVn0Z3Kx72ZpZ2qyYJJuONRkimlhpd+hkEFr0ZxgGl112GXw+H06cOIG2tjacOHECb7/9tiLjqO12OwDE5UO4a9cueL1ebN26FXfeeScuv/zyuJ9HJZkQxNv7ElqT4TgOPT09STknA8qTjLyA6PV60draCpZl0dTUFHcBcaXSZRzHoa+vDw6HA/n5+cjPz0dWVnyHViogUUy2LhsUKLg5d8Ro5vj0cZRmlKImpybp5+tY6IAn4EGBoQBt822gQMGoNYIChVx9LmZds5h1zeJg1UEAkKIYDa2BQWNAWWYZnul5BnOuOQS4AG7bdxsaihrCPte7wrsQEN/7wzDMsgZDUoyWT57Mz8/Hg5c+GHMIWCSkY8RAvmPpti8geiNmRkYGdDodDh48iIMHD6b8XIIg4NZbb8XBgwexbdu2iI8rKyvDww8/jD179sDn8+GJJ57AFVdcgcOHD8eMfghUkjkDMveFZdmknJPb2tqg0WiSck4OXS8VyI0tAdHVta2tDcXFxdi6dWtCd3ArkS4jqUSKolBUVAS73Y7x8XEAkO6kSeNhuNeVLORRDE2JB4yJMYWNZhbcC/jJhz/Bupx1+M3HfyM9PhGQKCZLl4W3xt6Cm3WDEzhYPBYwtPgZ+Dk/Hml7BF/Y/AUAZ6OYfEM+jowfweb8zRi2DcOkNcEdcOM/2/8Tv77i12Gfz8/58YnnPoFv7PgGvrP7OwntNXTcsdfrlZpCyRAw+TiDSE4EoUjHSEZeW003RCIZp9OpuLLs5ptvRnt7O95///2oj9u0aRM2bTqrVGxqasLExATuvvtulWQSQTLWMCSSUcI5mTyn0pHMwMAARkdHsWXLFlRWVia1lpKRzMLCAtrb21FWVob6+nqwLCs1HpKaAWk8NBgMEuGQXHAqe3my60ks+ZZAg4bdZ5d+7uN8eLj14SCSeaH/BUw7pmHz2nBi+gT2VwQ738azj46FDlg8FpyaOYXxpXFcXHExvJwXWfosbCvcBg0tfvVIpMTyLN6deBd2nx2nZk9h3D6OHnMPAnwAvMAjS5eFN0ffRNdCV9ho5o3FN9Bn6cPPj/4c12+9HrmG3GTeJgCi7DbUiYBEOqOjo6Aoatk4g3Dfl3Qs/J/LkYxSuOWWW/DSSy/h3XffTepc2L9/Pw4dOhT34z/yJMPzfJA0Od4vBc/z8Hq9ijgnA8pHMiQ9Fq4vJ5G1lCAZOeERpR1R7QHLawZkBLLFYsHQ0BA8Hg8MBgN4nofNZkvKSPJA5QF8y/etsL/bXrRd+t8L7gW8MPACcvW5cAVceKr7Kewr35dQNOPn/LB4LeB4Dv3Wfvg5PwqMBdhduhsGjQFX1l0Z1pqlPq8eNEXjg8kPwAkc5t3zyNPnIVOXiSxdFuZcc2GjGR/rwzOzzwAQe3Eebn0Yt++/Pe79RkM8IgKdThc0zkCv10u1j3Q7zBP9nq8mVtpSRhAE3HLLLXj++edx+PBh1NXVJbVOS0tLXDVdgo8sycQzFjkS7HY7urq6FHNOBpQjGbNZ7LnQ6XRobGxMSVuvBMkQ+52ZmZm4CS90BLLX68Xo6CgWFhakgVO5ublSpGMymWJ+dheWX4gLyy+M+dwv9L+Aedc81ueuRyabiea55rDRzCsLr6BosQjbS7YvW0PH6PC5+s/h5PRJGDQG5OhzMOOawec3fh5Z+izoNcuvFw2twWc2fga3vX2b2E+hMWHJvwSGZqCltfCyXjA0EzaaOdR1CLaADTRFgxd4/Pb0b/GtC76VUjQTCfGKCEj0mcg1LQgCWJ6Flkm9qB0J6VgnIohkd6NUuuy73/0unnzySbz44ovIysrC7OwsACAnJ0dKT99xxx2YmprC448/DgC45557UFtbi4aGBvj9fhw6dAjPPfccnnvuubif9yNJMko4J1dWVmJiYkKx2RKpkowgCBgeHsbw8DBomkZ9fX3KzVup7slut0sNZk1NTcsUMfGSusFgQF5eHlwuF3bv3g2n0wmLxYLFxUUMDQ1Bq9UG1XOSNfUkUUy2LhsMzSBTl4k519yyaObkzEk8OfskBt4dwItfeDHs6+i19OLY9DEUm4ph0pgwvjSOt8bewpe2fini8w9YBvDfg/8No8YIu88OChSsXisytOJ4YJPWBIZi0G/tl0jGx/rwq+NikyZFUaBBKx7NREMkEYHFYgEAHDt2LGicQTQRwYMtD+L49HH8/prfS+lEpZHOJLPS6bIHHngAAHDZZZcF/fzRRx/FN77xDQCiOpbUSAHRAPS2227D1NQUjEYjGhoa8Morr+Caa66J+3k/UiQTq/clGohzssPhQGNjI3Q6XdCHkSpSOdD9fj/a29vhdrtx4YUX4tSpU2sqGxYEAZOTk+jt7ZXIWAnJJfnMyDRK0ktABoONj4+ju7sbmZmZEuHEMiSU44X+FzDlmEKhsRAd8x0waA3QM/pl0cy9zffCz/txeu40jkwcwWXVly3b55PdT8LNulGaUQoBAsweM/7j1H/gkxs+iUydeFfK8ZwkAgCAh1ofgof1oMhYhExdJvysH0v+Jfyfff8Hn9n4GelxWbqz0eChrkOYc82BwtkeGZ5f2WgmGoiIIC8vD9PT09i3b5+UXuvq6gLLssjJyZHSa0RVOOeawxOdT2DJv4S3Rt/CVeuuWpH9raalTKKIli6L1cwdD+L5Lj/22GNB/7799ttx++2p3ax8ZEgmdCxyIgQjd06+6KKLoNVq4fV6JdJS4qJNlmSsVitaW1uRm5srRQtKWdQkQzIcx6G7uxsLCwvYvXs39Ho9JiYmFNlLODAMIxEKcHYwmMViQU9PDwKBgHSoxVJGTTmmUJZZBnfADYvXAo1fg435G6FltJh2ijYqx6eP44OpD6Cn9eAFHr858RtcWnWptOaMcwYL7gW8P/E+MjQZcLNuWDwW+Dgfhm3DeLb3Wdyw4wa0zrXiKy99BYeuPYTdpbulKIblWVh9VmlPXtaLRzsexZe2fClsGuneU/eK6V6KFomGgtiH41/CU91PJaw0UwrkujGZTMjMzJS8u9xut6Rck4sInpt9DmaPGTRF49H2R/Hx2o8HEbBS+ChHMmuFjwTJENv0vr4+7NixI+6LTBAEDA4OxnROVopk4vFCk+9tdHQUAwMD2LhxI2pqaoL2ptQI5kTIyu12o6WlBQzD4MCBAzAYDHC5XKs6tIwMBispKZEONUI6o6Ojkr0KIR2DwSD97Q8P/hAcz+EnH/4EftYPhmbwrQu+hc9t/Jx04N1z8h5wPAcdpQPN0Dg1e0qKZt4eextfeuFL+FrD16ChNWB5FnafHdPOaQhn/u/PfX/GDTtuwC+O/gIzzhn84ugv8MfP/RFezot1uesQ4APLXlOhsTBireL7jd/HkHUI4xPjqKyoDDqkLq2+NJm3WRHIR5ATyG1U5CKCgZkBvDj8IiiWgpExomW6Bc+2PIvPbfuc4qOOz1WSOVfNMYHznGTk6TGWZTE3Nxf3BRbLOZlcDEpEDEBiB3ogEEBHRweWlpawb9++ZXYPSirV4iUIYldTXl4e5HSwllb/od5Q5GbDYrFI89uNRmOQMqrf1o+j00dRllmGJf8Snu9/Hn+17q+QqcvE8enjeG/iPegYHShetNT3cl785sRvcEnlJfjZhz+Dn/Pj2PQxHLr2EGiKxrsT7+KB5gdQllkGi8cCPaPHkbEjeGvsLVAUhXfG38HJmZNoLGvEq3/9akKvz+6zoyKrAn/b8Lc4cuQILj54sSJpSSUQTzqaiAje63sPHsqDirwKCLwAp8OJRzseRYGtANmZ2UGea6nWGZWcJaM0OI4LW09ciT6Z1cR5SzKhxX2tVht3eoscmMXFxdizZ0/YC5uskUj0EQ3xEoPdbkdraysyMzNx4MCBsBflaqbL5NFeOCPQWF5qSu4lFuQzWgBR+UainMHBQXg8HjxrfhYWpwVF+UXIyMjA6NIoXh95HZ/f9Hncc/IeeFgPDBoDWI6FF2LK9NTsKfzHqf9A81wzGJpBn6UP3YvduLLuSrw/8T5MWhPyjfnI1mdj2DaMO9+7EwBgYAwI8AHcdewu/PFzf0z49fzLu/+Cxzoew/988X9Sel9WAvH2yMy55vBs77MwaUxitEgDpVmlmPJOIVAdwLrcdUFS9kQGgEXa17kWybjdbpVk0g3hrGEIUUS7yBJxTqYoSrGIAYhNMoIgYHx8HP39/Vi/fj3q6uqielWtRuHf7/ejra0NHo8nojxZKZJZCWg0GhQVFUmOss2Tzeh9pxd5mjzY7XZJ5n6o9RAuLr0YnMChMqsSgiBgyb0ElmKRqc+EUWPEoS6xOY0BAx48fn7s52BoUQlWlVUlPh+tAc/z6DJ3QUfrpDt9eTQTL8aXxvFE5xMAgF8c+wV+UPCDtHp/4+2ReWXoFSz5l8DxHCYdk9LPA1wAzw8+j2uuvkb6fOQDwLq7uyURAUl/xmNNdC4W/tWaTBohWu+LPPIIF5kk65ysZCRDRAmhYFkWnZ2dsFqt2Lt3L/Ly8mKutdJTNu12O1paWpCTk4MDBw5ETGOcS0PL/jLxF1gDomTYDTc4joMPPozYRvC7t3+H75d8H3mb8qAxafCL938BT6YHe8v2YlvhNnz5pS+D4znwFA8NrUGvuRe/a/0dAlwA40tnVYhjS2NBr0NDaRAQEo9m7j5+t/S/j0wcwdWGq3EptXY1mFDEG8lcXn059Ez4uktdTnCzYOgAMHm9TW5NRP4L1z91rkUyxHHhXJ2KCZxHJBOr94X8W0nn5FgzZRJBpAN9aWkJra2tMBqNuOiii+LqAVnJdJkgCJiYmEBfXx82bNiA2traqIeJPJJJd1xQfEGQPFiOi8ovQrWuGhaLBW/1voUF/wKq/FU4PXoaT3U9JXa4Q5A63QFg1D6Kn176U6m/Zto5jdvfvl36Nyny8wKPwxOHMWAZQH1+fcx9kiiGE8RrmaEYPDnzJG6ibkr1LVAM8UYydbl1qMtNvPM8XL2N9E8tLCxgcHBQ6p8ikY5erz/nSAZQruN/rXBekIx8LHKyzsnbt29HSUlJQs+rdCQjJwZ5r0ldXR3Wr18fd1SwUukyjuPQ1dWFxcVF7NmzJy6L8GRIZsw+hpcHX8aNO2+UuuPjiWScfidOzpzEheUXwqQNP4gtGq5ef3XMxxiyDZibmYOJMaGsoAxTM1MYsA6Ax9nPLsAHwFAMJhwTKM0oxY7iHdDQGjj9Tjj9TrgDy63zjRojyjLjs+r40Xs/Cvo3J3Bod7bjxMxyZ4K1wmr7ltE0jezsbGRnZ6O2tlbqn7JarZiamkJPTw9MJhMYhoFGo0EgEEgbkQRBJFGCqi5bQ4T2vsTq3JeTjBLOyUpGMvK1WJZFd3c3FhcXk/JFW4l0WTh5cjxIhmT+0PUH/GXkL1ifuz6ug5+gba4N74y9gwxtBvaV74v77xLB6ZnTmHRMokBbgKysLOw37sdUYAozjhnk6/MRYANwBpz4RPEncGHphWDcDC77w2X4TP1n8A/7/wH/e+//Tun5m2eb8ae+Py37OQUKPzv6M7z0hZdSWl8prLVvWWj/VCAQgM1mw8jICBwOB9577z3JiSBZEYHSYFl22R4CgQB8Pp9KMmsBUtyX6/HjcU5mWVZyTq6pqcGGDRtSck5WOpJxOp1oaWmBTqdL6DAPt1aqINHD/Pw82tvbE04nkjWA+Emmz9yHw+OH4fA78MfeP+JjNR+DXqOPGcks+ZZwfOY4HH4Hjs8cx7aibUlFM9HgCrjw7sS7CPABuAIuTDmmEOADWPQsQsNoUJFTAYqiMLE0AbvGjssqL8NDrQ+hd7EXM/YZ7DPsQ0NlA/Ly8pI+0P79xL+H/bkAAR9OfQg/54eOSc5WR0mkmwOzViuOp7ZaraBpGlVVVVJTqLxpNxERgdIIF8k4nU4AUGsyqwl570uijqo0TWNoaAhut1sR52SlazIejwdHjx5NmfyULJJbLBZMTEzEPU0z3F6A+Enmub7n4PA7sDFvIwasA3h77G0pmom2Rsd8B+Zd82gobMCgbRCdC52KRzNeVrTqX5+7HhNLE8g15KJjvgN+zg+jxgib1waapqFjdBhYGkAb24Y3LG+Apmk4OAeeGXkGX3N/DT6fL8iFIN4Dbc41h67FLpSYSuDn/Li+4Xp864JvwevxorWtFVddelVaEAyw9pFMJBDhj16vDysisFqtkoiAmLBGEhEoCSJaCiUZt1tMrao1mVVCssaWgKiGIgW0dHNO5jgOU1NTkgEkkWyu5b78fj/m5uYSnqYZikRIhkQxRaYiGDQG0BQtRTPRPmcSxeQZ8qBjdMjSZq1INFNgLMD39n4PPp8PHwQ+wOVNl+Nf3v0XjC6NSh39nMBBy2ihY3T4Y+8fsehZRIY2A17Oi8OLh/GPH/tHFOuKYbFYlqmiIg1sI/ivjv+C3WdHaUYpbD4bXh95Hd/f+32U5pRi3jCPAmNqN01KIt0iGYJwhf9QEYEgCJLfGhERaDQaiXCIiEBJRBoL7XK5YDQa1zyVlwrOGZKJdyxyKOT2K3q9HjU1NYo6J6eaLnO5XGhtbQXHccjMzEyZYIDU1WU2mw2tra1SXjuVfHAiJEOimLKMMrA8i/KMcgxYxGimqaAp4hokitlcsBkAUJZZhn5rvxTN+FgfRu2j2FSwKezfJwr5Pn508Y/Cuh1bPBZc+6drAUG0/mdoBmaPGQ+3PoyfXvpTVFRUoKKiIuhAI/NZ9Hq9RDh5eXnQarWYc83h6Z6nJfIl45sf73wc393+3bQ70NM1kolHXUZRVFwiAkI4ubm5KYsIyDkSLl2WkZGRdp9vIkh7kkl0LLIcoc7Jw8PDiqW3gNTTZTMzM+js7ERVVRXy8vIwODioyL6SVZeFypNZloXX6015P/Gk7wasAzg8fhgBLoDOxU4suBdQllkGD+vBH3v/iMYD4RsVnX4njs8clwwoCbwBL47PHMf2ou14afAlvND/An5yyU+wPm99yq9HDg2tQbY+e9nPf9f2Oyx6FmHQGMRmYEoDmqLxfP/z+NYF35ImYoYeaGRgm9VqxcjICDo7O5GdnY0/zf8JZrcZxRnFkvRZS2vxdM/TuG7ddYq+JiUQK5LpXuyGhhYNSFcTydjKRBIRWK1WKf2elZUl3RQkIyKINBb6XG/EBNKcZFJJj4VzTg6VMKeKZCMZnufR29uL6elp7NixAyUlJVhcXFwTHzSCcPLkoaGhVRsZoKN1OFh5EH7Oj8Pjh0FTNAqNhdhduht5BrH5NNIaWwq2YH3ucvLQMTrYfXa8NPASuha78Hz/87jtwttSfj0AcNR2FJudm1GetdwVwsf6cKjzEFiehUtwwc26pf1bPBYc6jqEfz7wz2HXDR3Y5vP5YLFYcLj1MCiewoJjQXKboGkafs6PdyffxXpKWfJMFdEiGU/Ag1vfuhU6Roc/fe5Pq1pHUqLjn4gISNbB5/OFdf4myrWsrKyYzxkpQ0NIRo1kVgDx9r6EIppzstIkk0wkE+osYDKJNQMlLWoSTZeRlB2RcxNF22oabdbk1OBfD/4rjk0dw/GZ49hUsAlaRosbL7gR1dnVWFpaCvt3mbrMsDLnB1sexJ7SPTgycQSD1kHMu+bxVPdT+NzGz6UczbQvtOPx6cexeHwRv/n4b5b9XstocdOum9Bn7lv2O4qisL88/l4WUqB+6gtPYcG9AI/Hg6WlJSwtLcHhdEDDaLCN2ga34Ibf7096YJvSiBbJvDjwIkbto6BA4dWhV/HZjZ9d1X0pncbT6/UoLS1FaWkpBEGAx+ORlGtyEQFJr4UTEZyvljJAGpIM6X2Znp7GyMgI9u/fHzfByJ2TL7zwQmRnB6cx1jqSmZubk5yKN2/eHHSxK+2DFu++osmTlVSpxbMOx3P4U9+fwPEcavNq0Wfuw0sDL+HmPTfHvQYg9rL89IOfoiKrAnU5dbD5bPDzfkw6JvFM7zP4v03/N6XX8ljnY1hil/Dm6JvoXOjEtqJtQb+nKRrf3vXtlJ4jFOE640mtYHZ2Fk7eiffffx+ZmZlBtYK1KhhHimQ8AQ8e63gMNEVDgID/bP9PXLP+mlWLZla645+iKJhMJphMpiARgdVqlSa5ajQa6TPKy8uDwWCISDLnugMzkGYkw/M8WJaVQkeWZeMmmHick1cikgkEls//CAXP8+jv75eMN8NJgVfTbBMQD4GBgQGMjY1FlCcrRTLx1ohOzpxE23wbyjPLxXSZqRBvjr6JT9d/GnlUdL82OX51/Fdws24MWgdh9Vqx5F+CntEjwAfwXO9z+NLmLyUdzbTOteLI5BFka7LhYT14pO2RsNHMaoDUChiGgdVqRWNjo5S26evrg9/vX7Pej0iRDIliik3F4MFjwDKwqtHMatvKyGtuNTU14HlemuQ6NTWF3t5eGI1GGI1GqTwgFxGc65YyAJAW8g9S3Pf7/eA4DhRFQavVxkUIpL7R1taGLVu2YPv27RHNGpU8yMl6sfbo8Xhw4sQJmM1mNDU1Rew1Wc10md/vx6lTpzA3Nxd1T0p5oAGxoxB5FJOhE79UBcYCWDwWvDTwUtyEd3rmND6Y/AA6RocAH8C8ex4BTrR5oSkaE0sTeKb3maRfxyNtj8Dhd8DDeZCpy5SimZXGmH0Mh8cPh/0deV/IwLYtW7agqakJ+/btQ1FREZaWltDS0oL3338fnZ2dmJ6eVkTQEQ3hIhl5FKNltKIxJgX8Z/t/ws/5V3Q/BGvtXUaG5q1fvx579+7FxRdfjPXr10sE89577+HkyZNobW3FCy+8ALvdnjLJ/PznP0djYyOysrJQXFyMz372s+jrW57KDcWRI0ewZ88eGAwGrFu3Dg8++GBSz7/mkUxocZ907scTdSTqnEw6/pVCrJrMwsIC2tvbpS9+tNSF0pFMpAOZyJNzc3Oxa9euqEOglJywGW4d0htAURT6LH0Ytg2DF/igWgZFUfhg8gN8vu7zcT3Xr47/Cl7Wi0xtJrzwghM40BQNXuDBgIFf8OO1odfwD/v+IeykyWhonWvF22Nvg+M5sAILClRC0QzHi3tJNJoQBAGPdTyGXnMv1uWuQ3V29bLfh64ZmrYhUyhDB7bJ0zapDgSTI1wk8+LAixi2DSNDmwGHzwEA0DN69Fv6Vy2aSbehZWTcBHGO37p1K6xWK44dO4bbbrsNFosFxcXF+MlPfoKPf/zj2Lt3b8Kf05EjR/Dd734XjY2NYFkW//zP/4wrr7wS3d3dEc/MkZERXHPNNbjxxhtx6NAhfPDBB/j7v/97FBUV4brr4lcz3nPPPWtLMtF6X2IRQjLOyUp26AORiYHneQwODmJsbAxbt25FRUVFXGvJD12l9yWXJ9fX1weNa46ElarJ8DyP7u5uTE5OIjs7G/n5+SjIKcCtjbeCF5a/n0atEVm6rJh7IVGMntFDp9FBy4rRMAUKpRmlAAXYvDb4OT9cARdymdyEXsMjbY/A4rWA5VnwAo959zyMGmPE2kzo6//b//5bVGRW4Fcf+1VCz9s634rm2WYs+Zfw6tCruGlXsNtyPNcMmUKZk5ODuro6aWAbkeF6PB7Jyys/Px/Z2dkp3fGHi2R6zD2Sy7XcQTpDm4Fec2/Sz5UI0nWeDCE/IiL47Gc/i09/+tP43ve+h/HxcbS1teE3vxFvZKanpxPq9XvttdeC/v3oo4+iuLgYp0+fxiWXXBL2bx588EFUV1fjnnvuAQBs2bIFp06dwt13350Qydjt9rUhmXh6XzQaTdhJlqk4J69G4V8uPkikU568RiXutELTXESebDab43ZPDrdOKvshBOH1eqXm0927d8PtdsNisWByUhxYFanz3eVyxXweUosxMkY4fU4pBePn/VLnvZ7Rw8t5cWLmBK6suzKh1zG+NA6GYsBTPGiKBsuz0NJaGDQGTDmm4GE9KDQWhrWuf3/yfUma/c2d34y7MVQQBLw08BJ8nA/lmeV4Z+wdXLP+mmXRTKI3JqED27xer+RCMDU1BZ7ng6KcRG1VwkUy/3LRv+AHjT8I+/hIIxaUxlqnyyIhXOGfnC/79+/Hz372M6m2m2ozud1uB4Co58DRo0dx5ZXB34+rrroKjzzySEIO1j/84Q/XhmTIXJfQwWJykDdcfueRqnPySkuYzWYz2traUFhYGFF8EAnyoWqpkow8PeVyudDS0gKtVoumpqaEDDeVHhlgtVrR2tqKgoICbNmyBRzHIScnB+Xl5ZIKx2w2Y3Z2Fv39/TAajRLh6PX6mHuZckwhQ3Mm/KeADG2G+DcU8Jn6z+DGC24UXxdFY2vh1oRfx817bsa/n/h3VGVWwbZog8vowpaCLfjZpT+DI+DAt/7nW6jKrsKvP/ZrcZTwGQiCgN+c/A04gQMv8Ph/p/8ffnvlb+N6ThLFlGeWI0uXhW5z97JoRono12AwoLy8XPosws1mkbsQxJJKh7NI0dAa5BpyU9pnKiCZgnOFZIBgm3+aprF58+aUnkcQBNx66604ePAgtm2LHHnPzs4uu4EvKSkBy7JYXFxMyMdwzdJlsT5oOcloNBpptkoq5pErFckIgoChoSGMjIxg8+bNqKysTPhLL49klNgXz/OSZLqyshIbN25M+D1TMl02Pz+PyclJbNy4EdXV1VI0K38uosKRp3MsFgv6+/vh8/kgCALGxsYkq5vQ9/id698BRYd/3zV0ape6l/Xivwf/GzpGJ3XxV2VXodfSi+MzxzFkHcK0cxoWrwXHp4/jQOUB6W/fn3wfx6aOgaZoMBSDFwdexC17bokZzcijGOIqUGwqXhbNKD0QjqIoZGVlISsrCzU1NZJU2mKxYGxsDF1dXcjKypIinXAd7unoXUa+W+lUkyGIRjJKOjDffPPNaG9vx/vvvx/zsaGfH7nOEo6aE3q0goinHsAwDLxeL3p7e2GxWLBr1y6pEzoZrEQkw7IsTp06BY/HE7Y3J14oSTIURcHtdqO9vR3bt29HaWlp0uukuh9Sd5uamgpK1cU6GOXpHHkUZLfbMTo6GmT1kZ+fj+Nzx/H7tt/j36/49xUxijw5cxJj9jF4WS/6rf2we+1wLDngYT14of8FjNhHkK3Phjvgxh97/4gLyy8EQzNSFOPjRJLM0efAy3njimZIFENTNCYdYjpREATMu+eDohklIploCLVV8fv9UmqNdLjLHYszMzNXfE/JgFzL6RrJhEtBKSlhvuWWW/DSSy/h3XffRWVlZdTHlpaWYnZ2Nuhn8/Pz0Gg0CbvXr7m6LBooikJLSwsyMzNx0UUXpZyLVHL+CyDeZbjdbmRnZ8dUasUDJYQJPp8PQ0NDCAQCOHDgQEqNXKmmy7xeL1paWsDzPBoaGoJywIkcQBRFwWg0gqIo7NixI6jXYHx8HB1dHbhv+j70u/rxdNvT+M6+7yh+kGzI24Bv7PgGAPGQ7e/vl9INJ2dOYtY1iw25G+BlvWibb5Oimfcn38fRqaPgBR4CBHg5LzS0Jq5oxst6UZ1dvUwMUZpRukzyu5oHuk6nC+pwJ3U1i8WCkZERyfKG4zj4fD7FHYuTRbqTTLhUthJTMQVBwC233ILnn38ehw8fRl1d7HHXTU1NePnll4N+9vrrr2Pv3r0Jm4GmZSRDnJNZlkVpaSkaGhoU+RIppS4TBAEjIyOSBfjOnTsV2V+qMmabzYaWlhaYTCYYjcaUL85U0mUWiwWtra0oKiqKeJeWKNGQvZBeA9Jv8NrAa5gcnYSW0uKpjqdQ7apGTVFNkIAg1c+nLLNMkth6PB4cWziGyzdeDrPHjMc7H0eWLgsMzSBDl4EZ14wUzTza/ih8rA+8IIoFPKxH7A8B8Hjn4/jppT+N+JxNFU1oqmiKube1jBrkNvlVVVXgeR5LS0vo7e2Fw+HAhx9+GFRby83NVVQqnQhID166RVhA5HQZcWFOBd/97nfx5JNP4sUXX0RWVpYUoeTk5Eh17TvuuANTU1N4/PHHAQA33XQTfvvb3+LWW2/FjTfeiKNHj+KRRx7BU089ldBzC4KQfpGM3DnZaDSiuLhYsYtCiXQZ2Z/T6URDQwN6e3sV21+yJCMIAsbHx9Hf34/6+npkZGSgp6cn5f0kQzLyvRDvuKNHjyoqhZa/3yzP4tmBZ8EwDGrzajFiH8Fk1iR25OzAwsICnm9+HtnGbOyv3q9IP8gfuv4APsCjEmK64eWBlzGxNIFCUyHm3fMARHfktvk2HJs+hivrrsSx6WOgQMGkNcHsMWNT/iZ8ou4TuKQqvHw0UaRTaoqmaeTm5sJkMiE/Px8lJSWw2WywWCwYGBiA1+uVZOvEhWC1Iot0VZYB4UlGEARFajIPPPAAAOCyyy4L+vmjjz6Kb3zjGwDElhDiswYAdXV1ePXVV/GDH/wA9913H8rLy3HvvfcmJF8GxDMkrUjGYrGgra1Nck4+derUiqjBkv1SkkbG7OxsHDhwAD6fT3EhQaIkw7Isuru7YTabsXfvXuTl5cFisSimCktkPxzHobu7GwsLC9JeyDqp7ifS53V4/DC6FrpQklkiRhLaDLwy9gr+evtfI6s4Cz/o/gGMTiP2VO4J6gfJz89HQUFBQlYr045p3Hf6PgDAbaWim/OsaxYVWcF9UFq9FjRFY9G9iNOzp6GhNNhauBUURaHQWAgP68FVdVehoaghhXfkLJQu/CsBcqCHOhbLzSMnJiYAQIpKiXnkSu8pHREpknG73Sm/J/FcH4899tiyn1166aVobm5O+nk5jsOjjz6aHukyuTpL7pys0WgUJxngrGItXhBV08DAADZs2IDa2lpQFCX1+Sh1J5koycjlyfJpn0q5B8RTkzk+dRxaWostuVvQ0tICiqKCnJwB5Zs6yXvN8iye6n4KgiDAqBHD/mJTMUbsI3hp4CUE+ABmXbNgKAb9dD8+v//z0iFHxkpTFCUdcAUFBVHrB092Pwmr1wpBEHDYchifx+fxj/v/MeLjuxe7cdvbtyHAB7DoWQRNiQec2WPG0z1P48eFP1bkukmnSIYgkrrMaDSGHdg2Pz8vDRYMHdim5J7SUVkGrJ66bDXhdDrx61//eu0jmWjOySthAwMkRjKBQACdnZ2w2+1Bd+fy9ZS6eBMhh2jy5JVoogwHq9eKw2OH4fP4MM1Po7asFlu3bg073nYlIpnTs6cxZB1CQAgEDSxjeRbP9T4HR8ABg8YAjufwWMdj+OT6TwYdcsRqxWw2Y3p6Gr29vcjIyAiqH0w4J3DnkTtx856b8ee+PyNDlwGe5/Gu9V3MueZQkhG5GfiN0TfACuL16wq4JBlyeVY5xpfGI/5dokhHkolnT+EmUJLUGhnYRoaBEal0KpHIuRbJ+P1+BAKBc5ZkTCYTHnroobUlmVjOyUpLjknRL9417XY7WltbkZGRgQMHDixrQFOyS5+sF4sceJ7HwMAAxsfHI8qTlWyijLaf5plmDM0OwWazoWFHQ8TmrqCOf9aL+07dh+s2X4cyY/wNXQTy17UxfyNu2XtLWCuaD6c+xGtDr6EiqwKswGLIOoRXhl7B5zed9UCTW62sW7cOgUAAVqsVZrMZvb29OGU5hWZ3M44sHMGMYwZWrxVV2WJxe9g1jKe7n8b3Gr8Xdp9LviVMOadwRc0VCPABVGVV4fb9t8OgESM8hmIUJYZ0JJlED3SGYVBQUCBJZMnANqvViq6uLrAsG5RaS3SYV7paygDhzxDicnGuWv1rtVpccskla0cyVqsVbW1taGhoQHn58umCABRPl5GJgrHWlPt8rVu3DuvWrQt7Mcu79JUI62ORjM/nQ1tbG3w+X1TLmtVIly26FvFy88ugPBQ2Vm/EMDsMh8+BLP3yuy45ybw29Bqe6noKTr8T/3ThP8W9l3Dvf54hD9dtCilE+v1wthxDz6tv45NcAI4aH8ZKjaApWopm9JrwKTGtVovi4mIUFxej39yP+569Dw6/AxQoNM81o0hXBK/HK3pM0Xr8qe9P+PLWL4eNZo5OHcXk0iQ25W8Cx3MYsY+gfb4dB6sOxv2a40U6RjJKRA1kYFtZWZlUBCekMzw8DI1GI6XViCvESu9ppRAuknE6nQCwonWqlQbLsmtHMnl5ebj44ouj2pwoHcmQNaMdwCzLoqurCxaLBbt3747aeEQscVZjDgxpRszLy8Pu3bujpvtWOl3mdrvx1GFxUuPFmy+GRqNBr7kXrXOtuLj64ojreFkvDnUewpJvCW+MvIEvbPwCNuRtiHsvQIwiptcLzR//iLk3/4CG2VnoaS0Cg2M4usmI2c16DFoHl0UzkfC7tt/B7DWD5VnkG/LhYl1w8A7AKx5WHM9h3DqOBz94EN/b970gQ8kl3xLeHHsTWbosaGgNNLQGWkaLN0bfwN6yvVI0oxTSsfCvNPFRFIXMzExkZmaiuro6qFdqcnISPT09y1Kd4VwI0pFkiPtFuEgmIyMjLfccLxiGWdvCfywfLYZh4PP5FH3eaMTlcDjQ2toKvV4fVEiPBiUbPCO5J8vlyfG4J5OLMtUvOiEH+TpmsxnvnXoPo4FRbFu3DXqd+B7lGnJxbPoYLii5YFk0Q9Z5beg1DFoGsTF/I8aWxvBMzzMRZ90nA7qtDUxzM6bzdZjR5UMQeJgcXlw06MdURQ4mig0we8wx1xmwDOD5/ufBCzwoAWgYdeGSYQ2yfX7ottdgYPc6DLttyMvNQyFTiI6ODgiCIN1RNzuaMWIbQY4uR6q9CIKAfks/Ts2cUjyaSddIZiX3FNorRVKdZGCbz+dDTk5OkFQ6XQv/kexuSI9Mun22iSDtJMyh0Gg0cbnvJoJIJEPuhmpra7Fhw4a4P1glxweEkow8qgoVHcRaB0i9ViQnKwAYHR3F4OAg2AIW/kU/LB4LrB4rANG6neVZdC50oqlyeQMhiWI0tAZ6jR6FxkK8NfoWvrTlS6jPr4+5l3giGbq7G9Dp8LH1n8LHAPRZ+jFuH8OFFhM+v/1vwH3iE3G97odbH4bdZ4cgCPhGG/DN017kcjr4waFgfAiFU+V479N/jaZPXyftiaikZmdnMX36A3xixI1AvgZT26qhyzCKUS8oyeJeaaTbQbTaRpTyVKcgCEFSadL/QW5qPR5Pwua6KwlyHoV+V5WQL68lyA1qWkiYI2Gl0mXyNUlvx/z8fFLeaCsVyUSSJ8e7DpACyXAcqJERaEZGUNjaCq6qCl0eD6wOBxobGzHHzSGvMDzhFWcUh93P4anDGLQMojJbbGLMM+RhwDKAZ3qewZ0X3Sk+0OkEtbAAMAyEsjIg0ToXz4NQkNPvxIxzGn7Oj0WPG0Y+vs9owDKAFwZeAMuzqLUK+JsWAT4a6MxlQVM0puDC5d0dqMzJAc40pkkqKYMBm377W1zy3HOA1wuepuEqK0PzzTeD3rEDBQUFyM/LVzzySNdIZq3SPJEGtg0PD8PpdOLYsWMwGAxBqTUlpdKJIpITwbkeyZDXtKaRTCxp60qTjNPpRGtrK7RaLS666KKEbPDl6ykdyRB5clVVFerr65NyTwaSzNVzHKi33wZ96hQYvx/5IyOYGx2FacsWbPlf/wv6jAzkIjfueSgAEOADeGHkBTj8Dsw4Zs7+nAvgrdG3cP2WL6N+zAHm2DFQViuEMyTDXX45hKqquF+TsGULqLY2CF4vptzT8LJeVApZsGIaXB5QG8deH259GE6/ExnaDByc8yPP70dfgQCKolGZVQmT1gS4TShobwc4DpCRuO7ee6F9+mkIOh2Qnw+aZZE9M4OD99+PoSefhFlm7pmXlyeSTn5+TNv8WEhHkkmnPclVhAaDAfX19ZJUmjToyqXSqQ5sSxThhjYCyviWrSVef/11uN3u9E6XKd0nQ9bkOA7T09Po6upCdXV1Ugc5gZJjkymKwsLCAkZHR7Ft27ak3ZNTcXSmRkZAnzoFoaQEDp6HMxBAkdGIzU4nhIkJCMnMs6CAjTkbsbF0Y9CPeY6HhtHAND4N5s3jgF4Pft06gGVBT0yAeu01BL78ZSDOPgHuggtA9fbC33wCfvsQakBDrxXQvr0WS4ZFVHCBmCOX2+bbkKPPAQAYNR4wFAsdowFF0SgyFYmGlb5pUGxIrTAQgPaJJyDQNEC8pnQ6CNnZ0ExPo6anBxXXXhtUsJ6YmEB3dzcyMzOlZtBkekHSsfCfrlb/NE1Do9GgsLBQylp4vV6pntPR0SENbJO7EKzka1lJ37K1xMmTJ9duMma8UFrCDIgH+dTUFNxuN3bu3Ini4uXpnUSgVLrM5/NhYWEBPM9j//79Kd3BkDA1KfKbmIDAspjzeDA3NwcAKK6rAzU6CmFsDEiCZPSMHrdsuwXr16+XfiYIgjTXXPPKK6ACAfC1teIvGQb8unWg+/tBj4yA37EjvujMZAJ7/fVoLfBiptWKktxK2OvKYastwphzHEO2IWwuEPfv5/zQMcsjiJe+8BLcATcAQHNwFAWDt2M9w4AvKhQfz7KgbTaYGxtRKT8YlpYAhwMIjUrOqACpqSkAywvWctv8rq4ucBwn2eYXFBTEZe6ZTlEDQToOB4uUPjYYDEFSaTKwbXFxEUNDQ9LANkI6qUaeoYhmKXMuk8y3vvUtZGVlfbTSZW63G1arFTRNJzVZMxyUSJcRebJGo5HmcaSKZLvsOZbF4uwsLDyP9evWYWBwUJG9RP291QohtMBJ0wBFAR5PQs9lhhvvlQdgztsEo9YIwA+4puDwO9A824z6vHq8PfY23hx9Ez88+EMx/SWDQWM4KzHekgd8+SvQ/uEPoIbHxNRYIAB/fT2mLrsMQRM5cnMhFBWBnpqCIL+u/H6AosDLCFaOUNt8l8sFs9mMxcVFDA4OBtms5Ofnh5WupyPJpGskE8vpI9rAtvHx8aDIM9LAtkQRz1TMcw08z6OoqAgvvvhiekcySpLM7OwsOjs7YTAYUFRUpJi6JJU9yj3RNm7cCJ/PB7/fH/sP40AyaTyXy4W+xUWUCQI2VlRAk5EBCgDvcomEVV0dc41wiBVVCWVloMfGEESJgYBINDk50hpA7NQQQzHYWbwTLL88zWrUGuEOuPF8//MYsg7hyMQRXL3u6mgbR+CGG8Bv3gzm3XcBux38tm1wHDwI7+hoyBMzCHzrW9D/6EeA3Q4YjQDLgvJ6wW3bBu7yy6Pum7xG0gtCDjibzQaz2Yzh4WF0dXVFNPdMpwNdPpIhnZBMx3+4gW0ktUYGtsml0uEmtsazr/ONZMh78Oabb6Y3yWg0mpRrMjzPo6+vD1NTU9i2bRtsNptiNRQg+ZpMOHny0NCQYqSaaLpsYWEBbW1tqNy+HWW5uaBbWkDNzyNzehq0RgNh714IG+JrnAy3l2jgt20D3dcHenAQfFERKI4DNT8PfsMG8CEDlmKRTK4hF5fVXBbx9/8z/D8YsY1Aw2jw8sDLuLTq0mXRTMjmwR04AO7A2XHKvMsFhJIMgMANNwAeD3QPPSSmzxgG7BVXwPfLX0pps0QQarPi9XphNpuleg4A5OfnK+Y4oRTIdZdOxAcoo3jT6XQoKSlBSUmJNLCNkM7o6KiUDiWkE4+Y6HwmmX/7t39Lb5JhGHF8bbIXh8fjQWtrKwRBwIEDB2AymeBwOBRt8EymJkNUbTqdbkXck8la8aTLBEHA8PAwhoeHz1r8bN4Moa4OwtgYbBSF0iuvBLNjR+KS4jOIlboTSkvBfupTYE6eBDU9DdA0uH37wF14ISCTbqd6aLkDbrw88DIMGgPKs8oxbBuOHc2E22+k10LTCNx8MwI33AB6eBhCXh6EGGNuE4HBYIho7un3++FwOKJ2vK8W0jWSUVpWLR/YRqTSS0tLsFgsmJmZQV9fX1wD2yLVilwuV8o147VGfn7+2tdkokHumpzoxUHMN0tLS7F582ZpLaXrPInWZEjaLpw8WWmSibUWy7Lo6OjA0tJSsAO2RiOqyDZvhplhwG7cmDTBAPHVh4SaGrDV1WIEoNGcVWiFPi4FJdWRiSMYsY1gXd466Bgd9Iw+vmgmUWRkgN++Xbn1wkAuy+U4DhzHIT8/XzL3DAQCkoAgGTPJVHA+RzLRQAa25ebmAhC/XyTKiTaw7XyMZORI+0gGED+seNMBcpficOabSjZPJrIez/Po7+/H5OQktm/fjpKS5aaKSvbcxEqXOZ1OtLS0wGAwoKmpKaJiRgnii1uEQFFSDSbSOslCHsUQVVlFVkXS0Uy6gWGYoI53t9stqdaGh4clhRT5byXTa+kcyaxmdKfRaJYNbCOkMzk5CUEQkJubC57nodVqlwk4zgeSaW9vT2+SoSgqociDzKYJBAIRXYpXIpKJtZ7P50Nra6u0r0iyxNVKl83Pz6O9vT2uZk+lZsGEruHz+dDf3w+dTofCwsK4J1Qmu5fTs6cx756Hh/Wgz9wn/ZzlWbw58uY5TTKhcmF5GqeqqipIITU2Noauri5kZWVJzaBKNx9+VCOZWDAajTAajSgvLw+yIpqamoLP54PdbkdeXh44jkNRUZEiEuZ3330Xd911F06fPo2ZmRk8//zz+OxnPxvx8YcPH8blYUQqPT092JxA+wIhzOuvvz6902VA/KSwuLiItrY2FBcXY+vWrRHvWJSMFgDxMA8EAhF/T+TJ+fn5YWfmhK61kuky+QTSbdu2oaws9jwXpUhGvpelpSU0NzcjMzMTfr8fExMToGlaOvQi9SLENAbt6QHz1lugJybAV1WB+9jHwG/dCgDYXrQdN++5OexryTfmJ/Wa0gWxJMyhCikypyW0+ZC8/6kqL9OxRwZIr3ky8oFtHo8HWq1WGp3++9//Ho888ghKS0vx0ksvoaamBhdddFFC1lIELpcLO3fuxA033IDrrrsu9h+cQV9fX9AASRKNxQtyPe7atSu9Ixkgdte/IAgYHBzE6OgotmzZgsoYhdbVimRC5cnV1dVxuSevVLosEAigvb0dTqcT+/fvj3vantLpstnZWXR0dGDdunWorKyUfr60tASz2Sz1IsilutnZ2VKDaSTCYz74ALp77gGsVghGIzSnT0Nz+DD83/8+uIMHRdVZ9WUpvY6wIPtZQ9JJ9CYgdE4LuaOem5tDf3+/5OtVUFAQsVgdDenYIwOsfSQTCRzHwWQySUrCX//61/jf//t/44tf/CIcDge+9rWvwWaz4bvf/S7uuuuuhNa++uqrcfXViUfpxcXFUm0pFRw6dOjcIJlIpCAf4hXvwak0yYQ7hFmWRWdnJ6xWa8LuySuRLnM6nWhubobJZIpafwkHJSOZgYEBjI2NYefOnSgqKpI6/uUF0/Xr10t32mazGZOTkwBElQrP8+GjRp8PmieeAJxO8Js3A2f2TA8NQfu734GvqRFVXgoefMa5OejuvhvMhx9C0GrBXXEFAl/4ApCfeFSUKlJpxgwdgRyuWE36QAoKCuLqA0nXSCZdrf7DFf6rq6tht9vxu9/9DhdddBG6u7vhcDhWbU+7du2C1+vF1q1bceedd4ZNocUDnufTP10WyVrGbDajvb09riFecqwEycjXiyRPjnetlTDbbG9vR01NDerr6xM+jJaRDMsCLhdgMATJi6NBEASYzWbQNC1Z5kQjrtA7bRLlCIKAlpYWycyQRDma0VHQk5Pgy8slIqGcTsDhADM0BN0//iOEigpwF18M7uMfB1K0T6enp7Hl97+HxuGAkJMDyumE9rHHQHd0wPeLX4RXxrnd0Lz5JqiZGfD19eAuvTTIXDMVKNnxH1qslgsIxsbGQNO0lHorKCgIe8OiRjKJIZq6jKgCGxoaVmUvZWVlePjhh7Fnzx74fD488cQTuOKKK3D48GFccsklCa9H0/S5F8nI+zo2bdqEqqqqhC7olZQwk1RQsqabSqfLZmdnYbVasX379pTMNgVBAAQBVFcXqOZm0QbGaISwbRuEvXuX+3XJ4Ha7pWjkwIEDCfs+URQlSXWnpqbQ0NAAv98Ps9ksDQsrcziwKRCAJhAABYDyekH39oKy2wGOA9PWBuq996B99lnwW7Yg8Pd/D/baa5N6P8DzMLz8MjA9DX7XLokoBJ8PTGsrmHffBReSnqA7O2H45jdBn/EvA0WB274d3kcfhRBGZZgMVupQD7XMD51GKbdYyc3Nla6XdDzMzyWSIRZDq60u27RpEzZtOuuw3tTUhImJCdx9990Jk4zP58Pjjz9+bpAMqcn4/X60t7fD5XJh3759yIkid42ElZAwsyyL3t5eTE5OYseOHWHlyfGupQTJBAIBLC0tQRCEhOov4UBSXVR3N6hXXxV7aPLyALcb1FtvAW43hI9/POzfEjFGRkYGDAZDysaCFEVJc92J15fD4YB5fh62wkIYenvhralB1tISTGcIhvJ6QfE8hNxcUC4X6PFx6O6+G0JhIbimM8PVpqZAd3eD37gRqKkJ/+QsC80zz0D73HOgTp8G5/eDmp2FQKInvR7gONB9fcEkw7IwfOc7oCcnIWRniz1Afj+Ytjbo77gD3v/8z5TeE2D1XJijmXt2d3eDZVnk5eXBYDAsm6i61kilqXulEY5kvF4vOI5L6burFPbv349Dhw4l/Hdmsxn/+q//eu6ky6xWK9ra2pCTk4MDBw4krfMnkYdSXwCe5+FyucDzfFR5cjxQgmQcDgdaWlpAURSqq6tTvkgpioLAsqCam8W7duJflpUF6HSgu7rAXXABIBv2Jhc9bNmyBYFAAHa7PaV9RNobqSfQd9wBzS9/CdPYGLC4CH5pCZxWC40ggM/MBKPVgtLpIBiNoBYWoP3FL0Bdfz20Tz4J5sQJ0StNowG3bx88jz4a9HoAQPvAA9A++qg4UI1hwPj94mtnWQg1NaIAgKKWpcqYo0dBj45CyMo6ay2j00EwGMAcPiwSVZJRJsFaHebhzD3JdFCfz4ejR49KUU5eXt6aWt+Q79W5QjJut+gEng4uzC0tLXEpUUORm5uLJ598Mv0jGZqmYTabMTQ0hI0bN8Y14z4ayIepRBHQarWit7cXgMj2iapwQpEqyZB0XW1tLdxutyIHD0VRgMslpshCBQy5uRDm5kRDyDOHMs/z6OrqwuLioiR6GBkZWXa3HW4SYDx7iXTXzm/bhsBdd4F57z0w77wDursbglYLqr8fHM/D7/FA53aDdrvBBAJg2trAtLSIKTWGgaDXgwoEwLz/PozXXw/PG2+cfd75eWj/9CfAaIRQVAReqwXlcoHiONCjo+DKy0EtLEDIygryOAMAymIRh5uFXhsaDeD1grJYzlmSkUNu7mk0GjE8PIwNGzZIzaAejyeo250oBlcL6Uwy4c4ip9MpTfhMBU6nE4MyJ/WRkRGppaK6uhp33HEHpqam8PjjjwMA7rnnHtTW1kpp6UOHDuG5557Dc889l/Bzm0wmXH755elNMoFAAGazGX6/H/v27VNEUie3qkmWZOR36lVVVZiZmUmZYICz9Y9EDw1BENDf34+JiQkpXUf6H5TYE6fVivb1bjcg087D7QZlMIiOwxBzsC0tLRAEAU1NTZI5oBIKNYKYHmhf/CK4AwdE5VdHB2iKgl6jAXw+cTwzx4EF4DUakTlzZkonRYlpQI0GlNcLprUV9OnT4PfsEd+DgQFQS0vgKyoAAHx+PliLBSaHA5TDAbqnB0JpKQJf/7rUl0PANTRA0OvFkQXyA8PtBoxG0B9+CMpuB7d/f0rqt7UmGTkEQQgy96yvr4fX65VSa3Jzz0SMJFNBOpNMuLOI1GNS/VxPnToVpAy79dZbAQBf//rX8dhjj2FmZgbj4+PS7/1+P2677TZMTU3BaDSioaEBr7zyCq655pqknl8QhPRNl9ntdrS2toKiKBQVFSlCMMDZiyzZugyRJ9tsNjQ2NkpD0JTcWyJRlt/vR1tbG7xeb9Cws3gNMsNiehqUwwGhsFAkCK0Wwo4doF5/XSzyk5rM+DiEhgagtBR2ux3Nzc0oKChAQ0ND0P6VIpl4v3BCRQUC3/gGNI8/DnpsDPTiIvgzjgIUw4DW6aAvLAQ1PS1OsuQ48H4/oNGA0mhA+3xgmpslkhGyssSRyj6fGIFQFDwlJTDk5oJ2OBD49rfBXnWVmDYL3cuGDWA//Wlon3sOAssCWi0ot1t8/+x2GM986bkdO+D505+SimrSIZKRI5y6zGAwoLy8XOp2J0aS09PT6Ovrg8lkWlFzT7KndCMZUisKF8ko4Td32WWXRf3uPfbYY0H/vv3223H77ben9JxyUBSVfpGMIAgYHx9Hf3+/NElxaWlJsfUTtaqRg/h96fV6SSnldDoVlR0D8ZPM0tKSJOltamoKiqaSSr3Z7aD/+EdQra2g3G4I2dkoLC+HcN11EHbvBtxu0B0dQH8/YDBA2LYN/OWXY3p2Fl1dXdiwYQNqa2uXfTGUPADjJSt+5074f/pTcJddBt1//ReoiQnAZgMlCBBMJjBnPn/qTC2F5jhxwBjPQwAwNTgI7/g4CgoKYGpoAL9pE+i2NggVFQDDgPb7Qc/NARoNdPfeC83//A/8N94ILswdn++Xv4RQVgbtH/4AOJ2iDDwEdFcXDN/4BjyvvbZi78lqIZa6TK4YrKurQyAQkHpziLlnTk6O5ECgxGGbTt3+cpBzKFxNJtVUWbpgzUlGfpcrb2Lcs2cP8vPzMT4+rvgI5mRIhtQ7QvtNlO5tARDXejMzM+js7ERdXR3Wr18f9mBP9PChn3oK9LvvQqisFKW1VisK3n0Xvvx84MYbIVx+ObidOwGbTaxPlJSgf2AAExMTuOCCCyJaTyQ9CjrMOgnBYAB37bXwfPKT0N95J7S/+504SMxsBmOxiEPRCNmQ/QkCBJ0OpUePYnDrVpysqYFOp0P5176GdXY7DOPj0HAcMs1mUD6f+D4AoE+fhqG9HT6bDexXvrJsH/5/+if4b70VzLFjMH3qU8tfG8dB8+GHoPv7RZVbAjgXIplo0Gq1Uc09iaIwmuVQPHtKZ5IJ3ZtSkUw6YM1JhmBpaQmtra0wGo1BTYyxbGWSQSIyZvnQs3DyZJKWUuIiJhdUtL3J3Zx37twZcd5ELE+1ZRgfB9XaCqG6GiCpyZIScLOzMJ46BXz1q2JNIT8fyM8XbWpaWuByuYLSdNFeV7w/j4aIxHlGUgy9HkJxcVB9gxoagubZZyGYTKD8fpFYKEr8/zQN8PxZdZjJBG7/fhgWFrDl1CnUfPWrsNntMJvNOHbrrTCdOoWC+XnU/OEP4LOzgdxcUACQlQVqcRH6n/8cfFUV+H37ljer6nRiJBMF1OQkcI6TTCp9MqHmnjzPw2azBY0/Dm3Gjee50plkaJpetrdz3YH52WefxSOPPILa2tq1JxlBEDAxMYHe3t6wd+WROv5TQbyRDHF1Zlk2ojxZrlZTgmSiRUak/uLz+WLKpRONHiibTUyRhYxYZjMzoXO5RAXZmfDd5XKhubkZRqMRTU1NZ6WpgiA+jqaDBAKRoirpZzwvHvAxDspIBynz/vvQ/PnPoGZmAJ0O3PbtYL/6VQhnZJeat94S6ymZmRAYRoxA5DcuGg34nBzw1dWiRJumIbAs6KEhMF7v2emUGzfCs28fHE89BYrn4dFqQXk8oBkGWrcbjMsFamkJhu98B8L69fDfcYdY0JeB37Il4usTKEq0xUkQ6UYySnb8yx0GgMjmnoR0Ipl7prOlTLhzg3T7n8vw+XwYHx9fW5IRBAEdHR1YWFjA7t27pTGzcijdoU/WjHUAWywWtLW1hS1kh64FiBeLkgqzUBDn4pycHNHZNMZzJVr4FwoKIGRmiqkwmf+W1uEAl5MjRTdkTHNVVRU2btwoHSbUwADop58G1dkpHtK7d4P78peBM44M4fZCnzwJ7SOPQNPWBiEnB+xnP4vA3/6tpFYLu8+QdejmZmgffBDw+8UUn98PzZEjoM1m+P71X8W1AgGRWMhEVI4TSUYQREKkadBuN6j5eXBVVeJjvF7xfQhJzRiNRlBlZQBNw6jVgtdoIDidoM+M9aZoGt7MTOiHh6G78054/+u/xDoO2X9dHQKf/jQ0r7wCSu5kQdNg//qvxebOJJBOJLOSHf+hlkNOpxNmsxnz8/MYGBiQzD1Jbw75nqRzJBPNUuZcxRe/+EV88YtfBLDG6TKKopCfn4/6+vqIEsaVIplIawqCgNHRUQwODsZlW0N+p1RdJtzepqen0dXVhXXr1mHdunVxHSgJ14oqKiDs2QP67bdFFVRmJmCxgPF44PjEJ5BpMGB0ZASDg4PLh8FNToL52c+AyUmgpATgeVB/+QuYkRFwP/1pWJKhjh2D/uabQdlsELKyxE78u+4CffIk2M99DszRo6DcbnC7d4O76ioIRUVhXzfzxhugXK6gCIDXaEAfOwbt734H7tJLRQkxy4rNpHq9KB8+894IOTkARYFyu0FZraAWFyFotaDHxoD5eZguvhjcvn0IfPOb4C+4AADgb2yEp7AQWRYLmPx8UF6v+JooClxWFgJGI9waDUzj45j/r/8C/81vBg0K8z70EAy33QbN00+D4jgIGg0CX/safL/8ZfyflwzpVvhfLe8yiqKQlZWFrKwsydzTZrPBbDZjcHAwyNwzmb6s1cD5PhUTSIOaDMm7RoJGo1G8JhOJZMg4YrvdjsbGxrhk0yTFpRQRytNc8npQtMJ6rHXiBf/lLwNGI6jjx4GZGSAnB9arroLrwgux0NEBs9kc1s6HfustYGIC2LpVjAwAID8fdF8fhPfeA7Vv37KDkHnwQVA2G/jycrFhMRAA/H5oX3wRzAcfQKiqEh0FWlvBvP8+/P/2b2HJihkbE+1ayOu2WkH19YGangbz+uugBwdFqXVxMajFRbFfhWWl9JyQkyNGLKOjYs/L0JAU+QhaLSivF5rXXgNz4gS8Dz0kEo1Oh67vfAcXPvQQ6MXFs6RlMgEbNyLTZBL3abXCYLejZ2xMqiWQ1Jtw//2gfvIT0dizqiol9+Z0S5et1X40Gg0KCwtReKYx2OPxSG7eFosFgiCgq6tLinSSmc+iNCKRjNPpPC9IZs37ZIDYYf5qRTLyccSJGjkqOQiNRCBkjIHf78eBAwcSljMm1SeTkQH+K18Brr4aWFoCCgpgHR7G4vy8VH8JF3FS/f1iWkqejtBoxH+PjYG68MLgvXg8oDo6IGRng7JYQDkcYoSh1wMOh1gb0mrB19YCLAu6vx+aV18FwjjR8qWlYNraIACicmxwUOxDMRoh5OWBLyiA9vhxCAUF4LZuBT00JBlVCjodoNWKfTAbN4IeHga3bx+YEyfAl5ScbZ7MywM1NQXt738P329/CwCwNzTA8/rr0Lz8MrSPPw5qbEyMps5EKxTHgWYY5O3ciX379sHn80mH3cTEhBTFF5SUID8zE6m4uqUbyaRLaspoNKKiogIVFRWYnJzEzMwMjEZjVHPP1UYkknG73Ul5M6Yb0rJPJhQMwyhubhdKMkQOnKwdvpKRDMMwcDgc6OzsRG5ubkJjDEL3lDTx5eUBeXmw2WyYm5uDwWDAhRdeGPn9LywUe0zkEAQIHCce0KERiFYLGAxiBOPxiOTCMGJEAAAZGaDMZqCiQvT5ysoCffw4NLW10HZ2gqqqgrBunehmfMUVoDs7RVWWRiP+3eIiaLdbNKVkGDH1xzAQdu0Ct24dhN5eMM3NYk3GYBAjl9lZ8NXV4LdvB3PqVHB3PkVByMgQPc5k76lQWIjADTeA27YNhu9/X2zuLCwUjTkXF8HX1IC76ioAYi2BNCPyPC81I05MTARFOcmOQ04nkklHF2ZBEGAwGKSUs9/vh9VqhdlsDjL3JKRjMplW5T2NJEhwu92okNXyzkUQAj0nSAYQU1mpuvgSEFKQp6OiyYHj2aNSkQzLshgYGEB9fX3YxsZ4kWr/ztTUFLq7u5GTk4OMjIyohwZ/ySVgjhwRU2bl5eLhPT4OFBSAb2paTjIaDfhrrwX9wANimkynE//G6wVoGnxODiiWBcWyEHQ68X+PjmLHv/4rTF4vNCYTuB074L/9dnAHDoCyWKD57/8GNTgoOiqfWQdaLSiWBcxmQK8X7V8KCyHodGKazOsVVXVLS+DLyuD/l38BPTAg7oXngyIzKhAQvdvCfB58YyN8P/oRdA88IJIdw4Dbvx+B226DEEbMIh/SRg48ktYh4wvkM1tipXXSLZJJR5IJvUnV6XQoKSlBSUlJkLnn4uIihoaGoNVqpfd/Jc09o6XLzuXCvxznDMkoPQPG5/PhxIkT4DguLdyTeZ5Hb28vvF4vamtrUVdXl9J6yTZAEuKdnp7Grl27YLFYYvbbCLt3g7/hBtDPPAP09YkHcWkpuG98A1i/Xiymh6TuuO98B0JLC5h33gEsFjGSMZnA63Ri+iw7W0xnLS2B7u8H5XDApNUCZWWig/EHH0B/553wPvww2GuvBdvYCMP3vgeGKMgEQXJWBk0DLAtu924xlabVwnf99eDXrRMdkjMyRNv/7Gzw1dXQ3nefaHhZVCT+rccDBAIIfPazEWXW3Cc+Ac9ll4EeGRFJrKYmbi+yUDdjh8MBs9ksWa5kZGRIB15OTs6yAzwdC//pJheO1vEvN/esrq4Gx3FSb87IyAi6urqCenOysrIUI9Fo6bJzvSZDbnzWnGRi3YGlYgMTCX6/H3NzcygtLY0qT44Xqe7P5/OhtbUVLMtKkUOqSKYmEwgE0NraKo2zzsjIgNVqjU1WFAX+M58Bf+AAqJ4eMTXV0CDJnimKAnw+UCdOiLWemhoIGzbA/9hjoH/5S2jfeANCTg64LVvAdHeD6e2F4PeD7u0FPT4OymYDKAoajgM1NgZ4PODr6kD39oI5dgzcpZdC8847YDo7pf0AOEs0Z2a9CLm58P/qV0Fb53fvDvq3UF0N///9v9D97Gdi3w1FiZHJxRcj8Hd/d2bZCO+rVptwt/7yt/Ls+AJiuUKinK6uLnAcFxTlkNkt6RTJpEtNRo5E9iQ39wQQZO5JBvDl5eVJ6c1UzD3PVwkzcNbFYM1JJh4oRTJEnjw7O4vs7Gxs375dkS9nKpGMzWZDS0sL8vPzsW3bNrS2tirmnpzIOk6nE83NzcjIyAgaW5AQWRUViXf/IdAMD2PT/feDcbvFFFhmJoSLLgJ7003w/5//A6GxUZQsO53g9+0De911oHge9NGjoKemIBgMAE2DoygwggB6YQFUYaEolT7jpKx5803Rrp9hzqa6iEyZokDRtLg30tkfBeznPw9uzx6xidPpBL9jB7iDB5fb9a8CtFptUFqH9IXMzs6iv78fJpMJfr8fTqcTeXl5aXG4pxvpAakRXyRzz5mZGfT19cFoNEqkn6i5Z6T+OlXCvMpQwlpGLk+urq6G1+tVtCs5GRIkKpf6+nppTo5SXmiJpMvm5+fR3t4ujY2Wvy8pOyh7vch86CFoxseB3bvFnhWbDfRrr0FTUgL2r/8a3Cc+Ae7ii0G5XGJD6JkahP7oUdEKhuNAeTyiHT/DiIX6hQUIGRlnXYvdblEkkJcnSpUBkRQCAbGPRaeD/r77oH3hBQS+9jWwX/pSsBouBEJNjRS5pAtC+0KIsWRPTw/Gx8cxMjIi3WFH635faaRrTUaJuko0c8++vj74fD7k5uZKpBPLf4zjuGW1ZlIjSoepmEpgzUkmkemYycLhcKC1tVWSJ8/NzcHlciW9XigSLfzzPI+enh7Mzs4uczpQimSiRiCLi6CGhgCzGVNuNwYEAQ0HD4adfpequSXV1gZmdBSu8nLkkLRCXh4ElwvMW28Bn/mMKH82GMSIZfkGxB6XsTHQgQAorVaMUJaWwO/aJdm2cAcOgO7pEd2S/X5QTudZxRtNQzjTu0IPDkL/ox+BMpsRuPnmpF9XOoAYSw4ODmLLli3QarVB3e+p3GGngtVqxkwEK1UnCjX39Hg8klR9ZGQkprnn+ZwuA8TSxJqTTDxIJV1G5Mm1tbXYsGHDitR4EolkvF6vlBI7cODAsrtNJUkm7DoTE6Dfegu82YxpiwU+ux37t2yB/kzXerh1KKcTWFgQGwYT/aI6HKBYFnxISkAwmeCenob17ruRPzUFXU4O+MsuA/fxj0u9Jtwll0Bz9Cj4rCzwFRUQpqdB+3yAIIDbtQu+n/5UlCADYL/wBTBHjoAZHIRQWSl6qLlcYlNlTo5oOQOxw58ym6F97DEErr8eCKP+igfpdIiSyIEUr2tqasCybNAdtt/vR25ublCUs1KvIV0jmZXeE5lkaTKZpCZz+xlz1VBzz/z8fOTk5Jz3Hf86ne78JZlo8mQl+1rI/uIhBqvVitbW1qh+aEqmy5ZFMjwP6vRpBMxmDAoC6IICrGtshHZ6Gjh9Wuw8l5OBzYbsZ59F9vHj0OTkQKioAP+pT0FobAxe1+EA1dYmprsqKyFs3nyWjKqrIZhM0Mjch3meh7u/HxqzGcVTUwhQFHwcB+bNN+E9cgTCnXfCmJkJ9tOfBnP4sNifAoA786UTLr4YvvvvD3I5FsrK4Lv3XmiefRbMBx8AGzeCq6+H7oknIIQ4NwjZ2aLdf0+PWGs5xxEuYtVoNCgqKkJRUVGQfT4ZZa7T6STCyc3NVcR3jyAdI5m1mCdD0zTy8vKQd2Zsudzcs7OzU3qfdDod8vLypIZrki47l0lmaWkJP/zhD9ODZOJNlyVSkyHRAsdxYbvllexrAWKTFnGa7uvrw8aNG1FdXR3xdUskMzUFanxcbGqsqACqqqLWECKuI4fNBs/wMIaXlpBVUoKqqirxi1dSAszNif0kZJRBIADmoYeQdfgwljIyIGRmgurvB/Pgg+B0Ogg7dwIAqO5u0A8/LA4FEwTRan/vXnDf/rboelxfj8D+/TC8+CIwOQlWr4drZAT6pSUYGAbcpk3QarXgOA4BsxmGt95Ca20tvBdcgIKCAhT+8Ico/PBDaI4ehdlqBS67DNnXX7/cRh+AUF6OwPe+h8D3vifubWAAuqeeEm1kdDqxJ8ZuF40yGQbU3Fzc72c6I1ahPdQ+n+M4KcoZGBiA1+uVohwlhoR9VCOZWAhn7tnW1gaHw4Hjx49Dr9fj+PHjkuVNqjWZd999F3fddRdOnz6NmZkZPP/88/jsZz8b9W+OHDmCW2+9FV1dXSgvL8ftt9+Om266KeHndrlceOONN8Cy7NqTTDxIJJIxm81oa2tDUVERtm7dGjZaUDpdFo20eJ5Hd3c35ufnpUFs0UBTFAwnToAZHxdnj1AUYDCA378fwsc/Hne6KlwtZWp6Gs6JCZRu2IACOdERxZW8+bC7G1RrKwJ1dQiwLJCbCyE3F1RPD+g33xSHl7ndoH//e1ATExDq68UoyOEA9d57oCsrwX/pSwBFwfvtb2PS5YJpfBz2mRnQmzYhZ3paLNZrtQBFgdFowJSUgF5awh4As+vXY3FxEV2jozB5vag7M7GSIr5jcUDYsAHc9u1gTp4Up2IuLYmvleMArRa63/4WfG2tNGb5XEYipMAwTJDHlzzKGR4ehlarlaIcuZNxvEjHSCYdSEYOIuLQaDSor69HdnY2bDYbXnjhBTz88MPw+Xz43Oc+h2uuuQZXXXUVdu/enfD+XS4Xdu7ciRtuuAHXXXddzMePjIzgmmuuwY033ohDhw7hgw8+wN///d+jqKgorr+Xo7S0FG1tbejq6jp/SEbunrx582ZUVlZGnj+yAjWZcA2LXq8XLS0tAIADBw7EpafXz8wg88QJCHV14mwTQFRjffAB+OpqCFHmkYTuiaRRSKPnzOwsLrzwQuTMzoq/oygxhTY9Lc5Skdcn5ubE3hKTCYJs/LWQlwdqZES0TunsFM0n168/m2bLyoKQmwv6vffAf+5zYgSRkYHxj30M0yyLTdXVqGxogPD1rwODg+I+zuyFOvO/aVmqR3v//dD9+MfiXgQBePllLP3nf2Li/vuRV1sb3YKFouD7yU9gvOEG0F1d5I2BkJEBfvNm0AsL0D7yCHy7d8dNXOmIVCXDpI5QWVkJjuOkOsLQ0BA8Ho/kZFxQUIDMzMyYz5WukUy6NYgCouqVYRjJ3PNXv/oVbrrpJuzduxdf+9rX8MYbb+BXv/oV7rnnHtxwww0JrX311Vfj6quvjvvxDz74IKqrq3HPPfcAALZs2YJTp07h7rvvTphkSO17x44da08y8Xw5YkmY5fLkcC7B4dZTOpLxhhTOLRYLWltbo0ZU4aCfnhYlt/J+k9xcYG4OVH9/wiQjN9psOnAAJqcTwltvAQMDoBgGAstCKC2FcOGFwem4M6E6xbJBOX/K6RR9w0gn/JmoIAgGg2gR4/dD0GoxOjoKQRBwwf79ogOxIIC77DJoOjvF13qmoRBmM3ijEf5duyDwPDR9fdD/+MeimsxgAMfzoHgeOQMD8Pz+92j90pckCxZy571MDrphA/zf+Q70//qvoluzwSBavWg04FkWTFcXcMZhIF6kW4e9kvthGEYqTNfX1wc5GY+NjUm/J6m1cLJgNZKJH+HIz+VywWQy4aabbsJ3vvMdBAIBxU2Cw+Ho0aO48sorg3521VVX4ZFHHkEgEEhaAr7mJBMPGIaJaG3icDjQ0tIijW2Ox9+MkIxSTWPy+ocgCBgfH0d/f39c82iWrcWyCHtkaDTLTShj7AkAjh07huzs7LNGmyYT+GuvBTU2BsHhADIyRAuUkENW2LEDQl0dtL29oLOzxYN+fh5gWfCXXSZKi6urRTKyWM5GQYIg9rDs2QNWr0d7SwscDgcAnCUYjgP/6U+DPnECzOnT0vhjwWCA/7rrwG7fDoFloX3uOYDnxd4aMhyNYUCxLErfegsH/9//g8PhwOLiotRzlJWVhcLCQsn+g6IoICdHbACtrAxKN1IsK669Qr5Uq4WVbH6UOxnL1VKjo6Po7u5GdnZ2kN0KEZyk24G+FoX/WCDfhXAkI5cva7XaFfNOk2N2dnbZePmSkhKwLIvFxcWwLQ6RQK7JwcHB9CCZWA1/kfpkyDAvuTw5HshHJisRQpPCP8dx6O7uxuLiIvbu3SupShIBX14OnphFkvRaIAD4fBBqa+NeZ2FhAQBQVla23Fk6M1O0fYmGjAxwf/d34P7jP2AYGBAjjvx88J//PIRLLxUfU1sL/pJLQL/6KrC0BMFoFJ2Vc3Ph+fjHceJMMXPXrl348MMPJVNSAKDz8hC46y4wb78Nqr0d0OvBX3QRsH8/dMR12+1eNpZZgDhFknK5gixYiNGk2WyG2WyW7PQLCgpQtH49qgoLQU9PiyIKmhZtbpaWxKbMNWpaVBKrETnI1VIbNmyQxhfI3+/8/HwEAgHFZ0ClinSMZARBgCAIMUlmNRF6HZFzOdHri5DMO++8kx4kEwuh6S2pxjAzk5R7stx0UwmSIZHW8ePHQVFUxLkr8YBdtw7e9etROjwMmEyiJYrTCWHrVghbt8b8e0EQMDQ0hJGREQBIzsmZ40C9/Tbot94C43CA1eshXHwxuOuuE+33CSgK/N/+LVBeDurwYVBLSxAOHICtqQknnU6UlpZi8+bN8J0xrQwEAmAY5uyUwowMcNdeC1x7rbje3ByYV14BMzsrjg9oaBBrRiwL7kz6j6YoUDwPf1MTAoGA2MdzxilBp9NJ6h1ip282mzFiNmP+r/4KW//4R+gHB8FoNKBoGtyuXQh861uJvTfSS0+fdNBa2biEG19gNpvBsiw6OzuDhrRlZ2ev6XuWjiRDzrRIJLPa71dpaSlmZ2eDfjY/Pw+NRhPUMB4PCDnNz8+fOyRD7ozk8uSmpqaEh3kBZ1NJSsmY3W437HY7KioqsHXr1pQuZtpgwOLBg1in14tmkzwPfvNmCDt2ADHubkhtamlpCRdeeCE+/PDDpF4j/dproP/wB7F+kZkJwWYD9eGHomLs858PfrBOB/6aa8RBZxyHqbk5dHd3Y+PGjaipqZFSJwaDAe+99x7y8/MlZZOciKn+fmgeeECUQmu1QCAAobgYfEMDqK4uUDwPjUYjWu6bTPD/wz8ACHbnpmk66D9ip79+/Xp4d+zA/CWXAG+8Af/8PDzV1RCuuAL5APJYVtE+kdVGOniFyd/v6elpbN26VYosiamkvJaz2lMp05lkQve1VlMxm5qa8PLLLwf97PXXX8fevXsTTteR67GmpiY9SCbedFk88uR4n0+JhkxBEDA2NobR0VHo9Xps27YtpfWAM4Sq1UK48EKxGB8n3G43WlpaoNVq0dTUJNWmEiYZpxPUG29AyMgAqqogeDzwning02+/Df5jH5PcleUQAAwMD2N8fBy7du1CYWGhlHOmKAoXXXQRXC4XFhcXMT09jd7eXmRmZoqEk5+PwmeeEYd+NTSI6SxBAPr6MLt+PfwbN6L6gw9AuVzgDh4E+w//AGb3blA8f7bGw/PSfwCkCId81gaDAYZ9+4B9+8DzvDQLniio5N3wqzWwSgkkm85YSQiCAL1ej4KCAqknhEQ5U1NT0lRKeZSzkgRAhh6mm7qMZFJCPzu3261IuszpdGJwcFD698jICFpbW5Gfn4/q6mrccccdmJqawuOPPw4AuOmmm/Db3/4Wt956K2688UYcPXoUjzzyCJ566qmEn5t8nn/1V3+VHiQTCzRNw+PxoLm5OaY8OV6kqjDjOA5dXV0wm82or6/H1JmRvqkimY5/s9mM1tZWlJWVYfPmzdIHTNM0hEAAGB4W6xo1NbEbOufnQVmtEM4U+SiKEoUIhYXA6CioubllHfQsy6K9vR1OpxP79+9HZmZm0OFPDnxi7lhXVyfd5S4uLqL37bex6dgxMIWF0LvdMJpM4HkeCxoNTIuLyP7Vr+B75JGw7xUQXGMjz0kOFvIa5Gk1mqaDFFRut1uqLQwPD0On00nigXCeX+mkLkunvRCEqsvkppLyIW0WiwUdHR3geT4oyknFOj/SfoDlEcNaI5IYQamBZadOncLll18u/fvWW28FAHz961/HY489hpmZGYyPj0u/r6urw6uvvoof/OAHuO+++1BeXo577703YfmyHIWFhelPMoFAAMPDwwgEAmhqalJs7nUqJON2u9Ha2gqapnHgwAG4XK6gDysVJGJIKXcS2Lx5M6qqqoJ+nzc0BOOf/wzN5KSoBtuwAfzf/R2E7dsjL5qVJRbC3e6zI4gFAXC5pPSZHIT8tVot9u/fD61WGxRVkMM9FPL6iZCdDer55+HSamGz2TC/sAAKgIGmkZmRASHOw4EQCACJaAjphEurkf8t95si3fBms1ny/EoHZ+NISNdIJtqBHm5Im9w632QySYSTm5ubMjmkM8mspG/ZZZddFvUm5LHHHlv2s0svvRTNzc0pPS9J387MzODEiRPpQTKRviBEnqzT6cAwjGIEAyRPMuGiBo/Ho1h9J95IRu4kEFbJNjyMDX/8IxhBEJs6BQF0WxuoX/wC7C9/CVRWio/zekWblexsMdopKgK/ezfo118XRx/rdKDdbnFE8aWXiuOVz8Bms6G5uRnFxcXYunWrRJChKatYoCoroamvR15vL7RVVZhfXITJYIB+fBzT2dkYmppCPs+jqKgI+fn5caU9YkU5pMYXGuXIu+GJ51eos3FmZmZQlJYOSBeSIYqpePcjVwjKxxeYzWZ0d3eD4zjk5eVJkU4yJH+ukcy5PhWToihMTEzg+9//Pt588830IJlwkMuTy8vL8f777yta4EyUZOSOAlu2bEElOaSTWCvWvmKRDJmkScQP4b549NtvQ2+zgd2zB8yZIquQmSnawhw5Av5TnwL9xBOg33wT8Hoh1NeD/+pXIezfL9rBeDygW1uhWVqC3umE8PGPg//qVyU5Mfl8yCwcAFLvEZDgF5phwF53Hbx33QX/yZOoKC6G3ukEqqpguuEGaM9YzJB5HXl5eRIRxCv8CI1y5P9FEg/IPb+qq6vBsiwsFgump6cRCAQkIQOJcla7mA2kXyST1OcvQ6h1vsvlWkby8ignnhsOkr47V0jG6XQmrOZKF5Az+t5770V/fz9+9KMfpR/JyOXJF1xwAYqKiuDz+RK+Q4qFRIiBSDJtNltYRwGlnJPjWWtpaQnNzc3Izc3F9u3bI37JqLEx8DpdcGMnTQNaLajRUTD/3/8H+t13IRgMoObmQB8/Dubpp8F95Svgfvxj8LfcAmF4GN7JSfSPjKDw618XLVkEAYODgxgbG5M+H7S3g3rlFdBjY8C6deA++UnRiTlO8DyPboqC+9prcYHbDa3ZDK6kBPz+/aDWr0chEBRZLC4uYmFhQZoMWVBQgKKiorhTK+HSaoRwokU5Go0GxcXF0Gq1cLlc2LFjB8xms5TmycjIEE09CwtXTbKbbjUZeR0sVVAUFXZ8gdlsRm9vLwKBQFCUE+mGI50iTjmipcvIjdu5BnJGv/HGG7j55pvx7W9/Oz1IhlyQxOtLEIQgeTL5IFiWjaujPx7ESwxEtaXRaNDU1BT2bpVEH0qQYLR9kdk469atw7p166I+l1BRASYQkEYQiz8UpMZO+vhxCEVFoE+fFustFAX4/WAeewzU4CDYZ58Vazjl5XD4fABNB0mkSYGfeuMNMD/9KWCziT5lx46B+ctfEPjhD8Xmyhjw+/1oa2sDz/O44NOfhk6vR6Q2PnlkQQ4dIh7o6OgAx3HSIV9YWBhXZBEurRYryiF7CZ1SScQD7e3tEARBinDCDatSCudbJBMN4cYXkM9/cHAQBoNBIpy8vLygzzQdSSaS4u18GFg2NzeHHTt2AEgjWxkiTy4uLsaWLVuC3nx586RSiGek8+LiItra2lBeXo5NmzZFvFDld8WpyiTDkYwgCBgYGMD4+HjczafC5Zcj8PTTMIyOAmecAqiJCQjFxRAKC8WxxDMzIsHodCLJcJxYuzl9GvQLL4D/ylckebnX60VzczMYhpEk0rzLBe199wEuF4QNG6RIhxoehuaBB+Dfty+qZYvL5UJLSwuysrKwbdu2hN87jUaDkpISlJSUSAXkhYUFTE1NSQOiioqKEoosIokHSCTNsqxkcUTk2TRNQ6vVBhWziWR3YmJCsrshpCPZ3SiIdCEZJSOZaAhNZcoFG/39/fD7/cjJyUFBQQG0Wm3avD9yEHPMUCglYV5LuN1uqcSRFiQzOTmJjo6OZbUOgpWYZhmt9iEIAkZGRjA0NIStW7eiQt7lHgbkUFLCQYAYW5KoKJw8OB4ImzZh4ktfwsb33oN2akrszq+rA//Nb4JaXAQAcZ6K3LZFEESPNJ4HffSoRDKAaJ5H+pMoihI/i64usbeltPSsNJqiIJSWghobAzU0FDFtZrFY0NbWhsrKyoQsgSJBXkBev349/H4/FhcXsbi4iPHxcdA0LUU55OCJhXBRjsvlwsjICLKysqLWcuSSXbn9yvj4uGQyWVhYiPz8/JQaQdM1klnt/YQKNuRjkC0WCwCgt7c36fEFK4Fokcy5Wvgnn3tdXR0OHTqEP//5z+lBMgUFBTHdk1eCZMKtl6ijM1kLUMZBQH4X7fP50NzcDL1ej/379yecclm64AKYr74aZQ6HePjX14tDxSwW0BUVoIaGRGIRn1D83xkZok/amVQT8UCrra1F7ZmIiNzZ0zQd3iKfrBnhoJmcnJRk17EIPFnodLogyxO73Y7FxUUMDw+js7MTubm50qEUr4UHibwKCwuxceNGiWxjNYKG2q8Qk8mRkRF0dXVJd9wFBQUJ24mkG8mQIvta7id0DDIRDTAMEzS+INn3XCmcj6OXyfv4q1/9ClarFT6fLz1IxmQyxbyzTHQ6ZiyEIxlyiOh0urgdnQFIXyolSJCQzOLiIjo7O2Om6mKtxet0y/ti8vPB/dM/gRofB9XdLcqYzzg0CwAojQb81VdjYGAAo6OjACD14Mj7X7B1K4TqalDDw6L9P3VmPs3sLISGBjGFJgNJ+01PT2PXrl0xB7gpBbmxI7GvJ1EOGUVMCCeSRJrUWmpqalBXVyd9mZJpBJWbTHq9XinKGRkZCRoYFo9cOx0sZeRIRwdmQPRZq6+vlz5/EuWMjo6CYZigIW2r4XgMiCSzbDTFmVpTqlMx1xrykQFpQTLxYCUiGfn4gIWFBSl9s3HjxoS/KEqNdCYHRltbGxoaGlK6048mIhAaGxF49VVorr8edGurSBCCAIrjwH71q2jJy4N9ehqNjY04duyYtE5Qg6VeD/aWW6D98Y9BDQyII405DkJpKdhbbgmy1ec4Dh0dHXC5XGhsbFzTnLPRaERVVZXUfGmxWETngd5e+P3+IH81o9GI6elp9PT0YMuWLSiX9QnJkWgjKHmswWCQrPQ5jpPsbgYHB+H1eoMaQcOpp9JRXZZOpAcs76w3Go2orKxEZWWlZDFksVikyDI7O1t6z+MZ0qbUvgjOh8K/HB9pkvF6vRAEAcPDwxgeHkZDQ0PEQyQWlJAxkwZLANi5c+ey2Q6JIqZ7QFER2P/5H9Cvvw7qvfcAjQbeyy/HSYMBzBmHBZK7Zs+YSIY2WPIXXQT/Aw+A+ctfxPpMdTW4q64SZ82cATE11Wg02Ldv36rdKcYDhmGCFEvEX21ubg59fX1SBL1hwwaUlpbGtWYqjaDkcAMQZHdD1FPk96RHRI1kYiOaukxuMUQiy3BD2sh/SqoEz8d0WTikBckoMR0zUZCRya2trZJrcXYC0xHD7S8VEvT5fJJ8m/QHpAr5COaI0GrBf/KTwCc/CbvdjubmZhRmZ6OhoUEiKYZhxJ8XFqKoqAg5OTlBn5mwbh3Y73wn7PJLS0tobW1FQUEBtmzZknYHkBzyvozq6mp0d3djYWEBBQUFGB0dxcjISJBEOt4DJ5lGUAAR7W5Ij0h+fn7aHUbpRnpAYhJmg8GwrH5msVgwPj6+bEhbqr1Q4UiGiEvS7XNNBWlBMvEg0uCyZEG6t3Nzc4Nci5NFKpEMOdwLCgrQ0NCAd955RzERQbzrzM7OoqOjAxs2bFhW4L/ooouklFJLSwsoipIIp6CgIKJSZ35+XurrqampSbvDJxKIos/n80mzgYgsmajVSMGeEE68suRkG0Hl6qmNGzcGdcLzPI/jx49LUU5OTs6akXk6psuS7ZOR18/Wr18Pn88nRTmTk5MgQ9oI6SR6hoQjGbfbDQAqyawFlEyXzc/PY2RkBDqdDnv37lXkS5Hs/og9CzncyaGiVH0n1jryIWekB0d+8BF1lHwYGFFqDQ0NoaOjQ7J5KSoqgslkkkYgkBRkqmm/1QSJKLVabdAcDbksmRw4RDxAiseEBKIRrxyR0mrk/Y8mkSYRV35+Ppqbm1FXV4fFxUV0dXWB47g1s7tJx8ZHpfYU+j1wOBwS4ZBeKHmUE+s5I41eBlSSURyrlS6TH6hVVVWw2WyK3XUlSgyCIKC/vx8TExNn7VmSXCvanqKly0gx3mazYf/+/cjKygpr0R+6ZqhSa2FhAYuLi5K3FE3T8Hq92LVrV1IjqNcKTqcTLS0tyMvLizl8Tq/XSwV7UjwOR7zEXy3VRlC5RFoe5ZAaiNzvixx+09PT6OvrWza7ZSUjjXStySg9S4am6bDjC8xmMzo6OiAIQpBoIxzRRyIZrVa7Jj54K4W0IJl4kGokEwgE0N7eDpfLhf3790vS0bXYXyAQQFtbGzweT9gGSyVJJtI6xMKHjIvW6/VxWfSHwmg0orq6GtXV1fB4PGhtbYXP5wNFUVIthqTVVspaRQmQ5tDq6uqYlj2hkBePN27cKPmrEbsTvV4vRXrxmjrGG+WwLCtFrOQzI02pZG4POfza2tqkFA85/JQWYaRjTSaSiktJhBtfEOprR953ks4MR35klky6vYepIG1IJp7pmB6PJ6m1nU4nmpubYTKZ0NTUBK1Wi0AgoGiNJ15iIHfLJpNJmr8Sbi0l9hYpXUZMNvPz89HQ0CDtXT4ONtGLnHi8mUwmNDY2gmEYqYYxNjYWVMMoKipKqy/SzMwMuru7FWsONZlMEvESifTCwgK6uroQCASCxAPxDugKF+UEAgFMTU1Br9dLUX5oI2jo4SevK/X09Cgu103XmsxqKhpDiT4QCEhET9KZeXl5CAQCy7IzLpcrqZHy6Yy0IZlYSDaSmZubQ0dHB6qrq1FfXx/URKeUczIQHzGQXpyqqiqpYzzSWiuVLiMF/vXr16Ourg5AsEV/Mt3aVqsVbW1tKCsrC3pd8hqG1+uV7u7J9EniKyY3M1xNkPENpB5VWFio+HOESqSdTicWFxcxMzOD3t5eZGRkRFTtRQJ9xqy0u7sbfr8fO3fuhEajidkIGq6uRCTSRK4rbwRNxnrlfK7JJAutVhvksed0OiVjz7a2Nml8wfj4OHw+n2K9Offffz/uuusuzMzMoKGhAffccw8uvvjisI89fPhw0BRNgp6eHmxOwFE9HM5bkiGW9KOjo9i+ffuyHgfyRVVyf9G80Mgsmnh6cZQiQCLTJnsg/UA7duyQLvho9Zd4QBoVN23aFNZ3jsBgMEgNcESOu7CwgJ6eHkmOS0hnNfLRPM+jr69PGvqWinw9Xsidm8kdLiHe1tZWAAjyV4uUXvT7/ZIz+N69e4PIIJFG0Hjsbshe4q0rpWO6bCVqMsmCXAMmkwnDw8PYv38/HA4HZmZm8O1vfxt2ux05OTm477778Fd/9VdYv359Us/zzDPP4Pvf/z7uv/9+XHTRRXjooYdw9dVXo7u7G9WyHrZQ9PX1BX0X5LXiZJE2JBNPuixeUpDXX5qamsIqNZS05wciRx8cx6GrqwtmszluLzQl1WXkwOns7ITVapX6gVIlGCKiIMKFRIYshZoZkrv7qakpSaVD7u5XwrGY4zi0t7fD4/Fg3759azZSWavVnh1BLQiSao+kF7OzsyXiJXe3ZNx1ZmYmtm/fvuwOXV7LIddQPI2goXY3xHrFbDZLkafceiXSoZ2uhf902xMhf71eD6PRiOLiYvT39+Oee+7BE088gT//+c/4wQ9+gJqaGhw5ciThJvF///d/xze/+U38r//1vwAA99xzD/7yl7/ggQcewM9//vOIf1dcXIzc3NykX1c4pA3JxEK8kQwZ2ZyRkSHVXyKtp+QgtHD7kxfXDxw4EPcdupLpskAggBMnTgBASgV+OQhxLi0tobGxMSW5ZejdfTj3ZHLQFhQUpHxHSqaKMgyDxsbGtHEfoCgKubm5yM3NlTrPyfswMjICjUaDnJwcWCyWoHHX0UAO1mQaQeXWK3K7G2Kjn5ubG9buJl0P9HTcU2hqmqZpZGZmYsOGDXjttdfgdDpx+PDhuJ0mCPx+P06fPo1/+qd/Cvr5lVdeiQ8//DDq3+7atQterxdbt27FnXfeGTaFlijOK5Ih9Yba2tqY9vHyGTVKXIChNRmbzSY59pLieiJrKUEyfr8fCwsLKCkpwbZt2xQp8JNDmqZp7Nu3T3HFWKh7ss1mk6ZgktHLhHQSjUCIAWpOTk7Cn8lqQ55e5HkeExMTkpPwzMwMfD5fyiOoE2kEJYRSX1+/bFgYqSkUFBSkbeE/3T5rcu6EvldOp1O6acvMzMSnPvWphNdeXFwEx3HL+tNKSkowOzsb9m/Kysrw8MMPY8+ePfD5fHjiiSdwxRVX4PDhw7jkkksS3oMcaUMysS7MaH0y8p6TeId6yUlGibtZhmHg9/sBQGrOqq+vT6rTXQmSmZubw8TEBIxGozShLrTXItF9ORwOtLa2xtVHogTk0uBNmzYt8xVLpGhus9nQ2tqKiooKRebXrCbMZjOGhoawadMmVFVVSe8DIV+j0SgRTl5eXlyfSyqNoPJhYfKRyD09PfD7/dDpdJiamkJBQUHc6rmVRLqSzEr7loVe49GyNps2bcKmTZukfzc1NWFiYgJ33333+UMysRDJVoaM7/V6vQkN9SJ3aUrJmMlaPT09kpV9smqlVEhGPnCtvLwcPp8PwNm8PHndiWJhYQGdnZ3LrO5XE/LRy2TcsbxoLu+4l984zM3NoaurC/X19dLIgnMFU1NT6O3txbZt26Q709AR1EQi3dnZmdQIaiC6i3S0WTmhI5H7+vrgcDgwOzuL/v5+mEwmaT/xdMGvBM4lknG73SlLmAsLC8EwzLKoZX5+PiH3jf379+PQoUMp7QU4h0iGpMvkbOxwONDc3IysrKwgx+BE11QCgiDAbDZDr9ejqakppQslWZLheR6dnZ2SyICoVlJVkI2Pj2NwcBBbt25NOD+8Uggdd2y327GwsICRkZGgoWSBQAATExPYvn27IkqZ1QKx5hkZGYk6e0ej0Szr9iciCjKCmkR7iYygBhKflUNRFLRaLbKzs7Fp06ag/hDSBS9vBF2t5tx0UpcRRItkUr1OdTod9uzZgzfeeAOf+9znpJ+/8cYb+MxnPhP3Oi0tLSgrK0tpL0AakUw86TJ5oX5mZkYyX0y0Q5tAqUjG6XRibGwMFEVh//79KY92TYZk5C7OxNTR6XTC4/FgaWkpKZUWz/Po7+/H3Nwcdu/erbjqRCnIi+Zyq5uxsTF4vV7o9XpYLBYwDIPc3Ny0u6sNBUn/zs7OYs+ePXHLq+VNgMTqhIgHmpubJWPTREZQA4nNypFft6H9IaQLnkRnxO4mEYPRZJCuhf9wJON0OrFu3bqU17/11lvxN3/zN9i7dy+amprw8MMPY3x8HDfddBMA4I477sDU1BQef/xxAKL6rLa2Fg0NDfD7/Th06BCee+45PPfccynvJW1IJhbIwe33+zE6OorJycm46y+RoEQ/yvz8PNrb26UOXiVmhydKfg6HA6dPn0Zubq4ka+V5HtnZ2TCZTDh16pTU/FhUVBRX3j4QCKCjowNer3dNZb7JQKfTwWq1gqZpNDU1STYvHR0d4Hk+Kbv+1QKZKWSz2dDY2JhSRBxpBDWJ9nJyciQRRbwODNGiHJ7n4XQ6YTKZEAgElkU5oXY3RCLd2toKiqKCGkGVVP2dS+kyj8ejyMCyL33pSzCbzfjxj3+MmZkZbNu2Da+++ipqamoAiC4X4+Pj0uP9fj9uu+02TE1NwWg0oqGhAa+88gquueaalPdCCWkyWo/juKh9MIIg4C9/+Qtyc3PBsix27dqV8ofxwQcfoL6+Pimikjc3bj8z3nhkZARNTU0p7QkABgYG4PP5sG3btpiPnZ+fR1tbG+rq6qTGrdACP8/zklX/wsICWJaVPMXCHbQejwctLS0wGAzYsWOHIsS5WvD7/dKhtXPnzqDXRu6miaGnw+EI24uyVuA4Dm1tbfD7/di1a9eKNqXKR1BbLJa4RlBHA0nVLi0tYefOndBqtUF9b6ES6dC/XVpakkjH5XIpandz+PDhNZ/GGoqJiQnYbDbp7CC49tprcf311+Pb3/72Gu1MeaTN6RHrInI4HADEO6jQLudkkWxNRu5eTJob5+fnFRURxGPRT1wEiKOB3LWXrAMEW5ts3rxZOmjJICZyR1tUVAS/34/29naUlJQkNYZ6LeF2u9Hc3IzsM0PXQg9K+d203K6f1HK0Wm1KB20qIORI07Ri13c0hI6gtlqtWFxcRF9fX8JScdLc6vV60djYKPVixSuRpmlaSncSC6JwdjdEPZfIe0P2kG7XcbSaTFZW1hrsaOWQNiQTDWTmCsMw2LBhg2JfwGRIhtzlMwwjNTeStZTyQotFMjzPo6urC4uLi5KLQLwd/KEHLWn6W1hYwODgoGRRnkoaci1gt9vR0tKC8vLyII+6aJDb9csP2t7eXvj9/iCrm5WU4pJrKiMjA9u2bVv1InWoA0OoVNxkMkm/D61psSyLtrY2cBwXNH8nXFot3kZQg8GwbIwCkXF7PJ5ljaDRPmtyw3UuFf7TKeJSAmlNMsRfanp6GhdccAF6enoUNbVMlGSsVitaWlpQUlKybJSwUg2UsdYivlUcxwVNbSQRTKIKMvKF9vl8sFqtqK6uhs/nk+oX8gmY6dIdHwoygXPDhg1RfZmiQX7Qkp6chYUFycgyMzNTIhwlZ7IQh3ASZa51/458IFptba2kEAtX08rJyZFu/nbv3h315i+SeICIeaJFOaRXiog6ErG7kUuv0wnhSEYQBLjd7vNqYBmQxiTj8/mk/DSRBPf39ytuzx/vehMTE+jt7cWmTZvCHmRKyqEjkQyRbGdnZ2PHjh3S41K1iJEXmkmoTmzhQ2XBJK2WLnbkRF69bds2xaIv+UErL1KTFCNN0xL5JutWDJx1hUhmhs1qIVQhJh8V4HA4oNFopAbRVCXSsRpBgeV2N6QRlNjdyAeFGY3GIHeLdALHcWGvGyWbMdMFaUMy8ouTpD5yc3OD7pCUmI4pRzzEwPM8ent7MTMzgz179kTsV1jpSIaMCaipqcGGDRukvYU2ySUC0sjK8zz27dsXVGiW28ITw8SFhQUsLCxgYGAAJpNJIpx4LeqVhCAIGBgYwPT09IrLq3U6XdDYXTIFc2BgAB6PB/n5+RLpxKvCW1hYQEdHxznVIEquCZ1Oh9nZWSmyM5vNmJiYkMg3kRHUQPKNoKFpPmJ3Q65Ro9EoGdKmib5JgkoyawhiySKfeU+gZLRA1otGDKQYGwgEYjZYkqhICcNNeYRFmvIGBgawbds2ybE3XIE/ETidTrS2tkYskodCPgGTZVnpyyzvtidptZUuWssNOvft27eqUZU8fbNx48ZlFi+EfEk6KdxnQ8YjyLv4zxW4XC40NzejoKAAW7ZsAUVRkr9a6Ahq0hBLIt+VbgQNZ3dDut7ff//9oEbQtR5vHK53h+M4eDwelWRWCqQ/YGZmJqIlSyRrmWQRjbTkqalY+WayFqDMPA2apqUvVHd3NxYWFtDY2Ijc3NwgxU6yHfxmsxnt7e2oqqrC+vXrE15Do9FIKRTSf7GwsCAdLvKCudL9NYFAAK2trRAEYUUMOhNFqNUNsXhpa2uDIAjSnXZhYSG0Wq00JC1aF3+6gnwnysrKlokrQsmXRL6EdMgI6kSH1CXSCCp/LLG7MRgMsFqt2LVr17JxyIRw1sLuJlxNxuVyAYBKMisFlmXh8XiiRgwrEckQU0s5iJsz6T2J5xCW33mlesGSSObkyZNhC/ypEMzk5CT6+vqwZcuWhGdURNormUMiv7OXm1iStFqqBXMySyUjIwPbt29PO8VQaP2CND+Ojo6iq6sLOp0OLMuioaEBeXl5a73dhGC329Hc3By3d5088iUjqBcXFyUTzWSUe7GinHDiAWIpQ8ZJhBMzrIXdjUoyawC9Xo+9e/dGzZ2udE2GDOIaGRmRpkfGC/IFiJRrTQQ+n09SmWzfvl0arZtKBEOsSmZmZrB79+4VO+RC7+xJKqm5uRk0nfxsGLvdjtbWVpSUlGDTpk1pWSSXQ251s27dOnR0dMBqtSI7OxudnZ3Q6/VB46fTrTAth8ViQWtra9LqvXhHUEdLMf7/7V13fFNV/35uZveiC0ppWaXQ0l2mosiQPVRUXKi4GL4Irwri688NrldRUYYbcb1QQLaCLAVEWroXlO6ZdCdp9j2/P+K5JE0KbUmaFPN8Pv0oSXrvye295znnO57HEtrvciyVSKvVao5s6GctFTM0NDRwoXrjRlBbyd10RDJisdhhqzi7C4chmc7AluEynU6H7OxstLa2YsyYMV1uiKI34vWOj1oS83g8xMXFAYDJQ9MdgqHfjTpB9lQOw9j50ZI3DF3NBgQEXDVGTpPkgwYN6pZ1gj1h3MVP+6royl4qlSI3N/eaCgz2BL32kZGRVtn5WrKgNva7Nw4xdmVXYSmsptVqUVlZCXd39w5LpI0LXKjeGy2RrqiosJncTUck01l5n96EXkUyHYW3ugsalqKd4iKRCGPHju3WQ84wzHU1ZBJCUF5ejosXL2LgwIEoKysDgOtO8FN3TpFIZFcnSEsJc6lUiurqahQUFHBqwYGBgSYyIpWVlbh48SKioqJ6XZLcuIs/MTGRu/btV/ZUObmiogJ5eXnw8vLiEub2lLqpra1Fbm6uTQsULKlpt7eg7uq14PF40Ol0yMnJAQBER0dzVuTXyuW0ryRsaWlBQ0MDSktLOXUMSjrXQwiWlKGp7tuNBociGXojdARb5GTUajXOnj2Lvn37IjIy8rrCFtcj0Z+fn4+6ujokJydDIBCgpKTkuhP8NMREG/0cJSRjqQ+FhtXKyso4eRe6qoyPj+91OYzOdvG3V06mUjc0lyMQCEx6cnoqD0XJPTY2ttu+SF1FRxbUdJKn14LK/nQUltbpdEhPTwePx0N8fLzJNWvfCNpe5699IyjNN9Lx0F0OlSEybgTtSpi8o52MvfXzbAGHIplrwZo5GUIIGhsboVAoEBUVZZVehe6QYPsyaVdXVygUCrAsi+rqagQEBHRrYqFGXb0hxNReLZg217W1tYHH46G8vBxKpdLhQkkdgXbx+/v7c2W+nYWx1A3LsmhqaoJUKuU0xYx7cmwldVNWVobi4mK7k3t7C2oq+0P7k6i+Gt1VAIbqw/T0dAgEAsTGxpo9O51tBDXux+mK3A0N87m6unb4d6fn6ihcdqOhV5GMtXIytDS4trYWYrHYas1wXd3J0MnIw8ODK5NmWRZCoRBhYWHcFp1qiXWmJJgKZ5aUlFi1C76noNfrUVZWBoFAgJtvvhlarRZSqdQklERDTY4Yv7ZmFz+Px+NWysaaYrW1tVav3AOuKItXVFQgISGBa2R0BBhfCyr7Q3u1qAW1n58fGhsb4erqapFgOjpudxpB28vd0EZQSjpisZgbr4+Pj8lY6BzmJBkHhDXCZWq1GhkZGdDr9Rg5ciTy8vKsNLqujY/aBoeGhiIiIgLAlVUVwzAYMmQId/NKpVKuJJhqaAUEBJhVvtCwW0NDA5KSkjptduUooCEmV1dXLszh4uICT09PLpREVQeKi4sdrkLLll38ljTF2huSGVfudbXC0dgoLSkpyeHLaNs3XkokEk52Sq1WIycnh9NYs1aJdEdhNQBwc3ODm5ubiap1Q0MDCgoKoNVqTeRu6O+0v1+dJNMD6Iw75vWEy1pbW3HhwgXO3Kutrc3qWmid2cmUl5ejsLAQI0aMQEhIiFkHP10xAYabNywszKQkWCKRcLkLSjgeHh7Izs6GTqfDqFGjbKoabAvQRr+r5Y/EYrGJbpWxtz0VbqQTbU8XOPR0F3/7yr32DbHGUv3XSiYTQpCXl4fGxsbrNkqzB1iWRVlZGXx9fREdHc0tzGhRiYeHh0mJdGd3fN1tBDWWu6FFLg0NDZBIJLh06RL3bLa0tJiUbFtLUubTTz/Fu+++i5qaGkRFRWHDhg24+eabO/z8yZMnsWrVKuTm5qJfv354/vnnOQdNa8ChSOZauJ6dDLVrHjx4MNdMZu1CgmsJblIdNLpa9PX17VKDZfuJxXiS1Wg0EIvFGDx4sMM1KV4L9fX1yMrKwsCBA82khDpC+wotKuZJq5KMJU1svTq0dxd/+4ZY6gRqLHXTkVQ/NRuTy+VITk7udYsTtVrNNehGR0eDx+NxJdLGJcn19fVIT0+/LgtqwFSK6lqNoPS/dAcaFhYGnU6HqqoqFBcXIzc3F3q9HiKRCKmpqWhsbLxukvnpp5/wzDPP4NNPP8X48eOxZcsWTJ8+HXl5eRZ7nEpKSjBjxgw8/vjj2L59O06fPo2lS5ciICAAd95553WNhcJhnDEBQ1XI1SZpmUyGc+fOYfLkyZ0+JhVSLC8vR0xMjEmOQqVS4cSJE5g6dapVQi2pqakICgqyGCqhcihqtRoJCQlwc3OzSgd/Y2MjMjMzuXyNVCqFQqHgVrJdEW20B6jf+4gRI9C3b1+rHJO6PkqlUi5Gbyzmaa2wGr23qBSSI4Ynqc4cDa0Z7/h8fX2Rl5cHjUaDhISEXlFUYQy1Wo20tDR4enoiKirqmn9XYwvq+vp6KBQKeHt7c6TT3cqu9o2g13IEbW5uRm5uLsaNGwe5XI7z589j7dq1KCgoQJ8+ffDEE09g5syZGDVqVJcXjKNHj0ZCQgI2bdrEvTZ8+HDMmzcP69evN/v86tWrsXfvXuTn53OvPfXUU8jMzMTZs2e7dO6O4FA7mc6Ey7oiQkkNlRQKBcaMGWO2SrCmFIzx+NpDoVAgLS0N7u7uGDNmjFU6+IErE/SwYcPQv39/ADBTTL548eJV8zj2Ak0yl5eXW30HYOz6aDzJZmZmArCOmCctHqEWCY4aYjLWmTPe8ZWWliInJwd8Ph9hYWHQaDQQCoUOcW90BiqVCmlpafD29kZUVFSn+2fojm/o0KEmhn3Um6Y7zqiWwmpXcwTV6XTg8/lcY+ptt92GP//8E0899RRaW1tRUlKCWbNmoX///tw92xloNBqkpaVhzZo1Jq9PnToVZ86csfg7Z8+exdSpU01eu/322/HFF19Aq9VaJezsUCRzLfD5/A7L/9pDoVAgPT0dYrEYY8eOtXix6DGsIQUDWM7JNDQ0ICMjA/379zdL8NPf6Y5ETFFRESorKy1O0Ma6UcbSLrRqKyAgAIGBgXZLltMCBZoDsGWSuf0k21Huois7Pkt2w70BtLvd1dUVDQ0N8PHxQVBQENeHQkUs6S7H3oUUHUGpVCItLQ2+vr4YMWJEt4nRuES6Iwvqrlo4dKZEWq1Wc2RjTFBqtRqjR4/Gf/7zH+j1epSXl3fp+9TX10Ov15vlBIOCgjg16vaora21+HmdTof6+nqrRBd6FclQIujIupSCrlpDQkKu6lNPJ3hrmo0ZH4sm+IcPH47+/fubhMdocr87JmM5OTmQyWQYNWrUNfMNHeVxqJwJfYh6KllOd5darbbHcwDGzX7GlXt0x0dLgq+WIDbu4je2G+4toCEmKjLK4/HMRCzz8vKg1Wq56ix/f3+HIVKlUonU1FT4+/tb1UnUkjeNVCrlqtaulte6GtrvcuRyOYqLi7leG+MSaZlMxu2I+Xw+Bg4c2K3v0v6aXCvyY+nzll7vLhyKZK71pYxFKC3B2HuFVm5dC9ZM/tOkoLFtNDU6a08w3Vkl0vJrHo/XLZl7Y1OpyMhIyGQySCQSTiXY1nkcKnFDxVBt7TtzLbSv3KN9F7Rb3DisxufzoVKpTJLMva3Agu4AfHx8MGLECJN70JKIpVQqRVVVFfLz8znZH3uGXNva2pCWloaAgACbiqQae9OEh4ebhFypBTXV3euKNw0t0e/fvz8GDx5sMie0tLTgzJkzCAsL6/a4/f39wefzzXYtEomkw4rH4OBgi58XCATo06dPt8diDIcimWuBVoRZKmNmWRa5ubmor6/nvFc6A2vbJut0Oly4cAEqlYqzLbBGgl8mkyE9PR1+fn5mE0R3YCxnYimPQ1f1gYGBVplU6Pip2ZWjhWKMNbQsiXl6eXlBLpfD39+f08LqTaB5wcDAwGtO0MYilrRCq33I1bhCqyfIlo4/KCgIERERPXr9LeW1qNacsb6av79/h02xlCD79u3L2YfQxWZrayvuuecexMfH45VXXun2OEUiERITE3HkyBHMnz+fe/3IkSOYO3euxd8ZO3Ys9u3bZ/Lar7/+atVdukNVl7EsC61We9XPHD9+HPHx8SYkQlfIABAfH9+lEMypU6cQFRVlFdbOy8tDdXU1fH19ERsbC4FAYBWCoU1+XSnxvR4Y53EaGhpMVrl+fn5dJghaAefIXvYdgRCC2tpa5OXlQSQSQa1WO2QhxdVA+8PoCvp6xmss7yKVSrncha1M6gADwaSmpqJfv34YMmSIQ11vSsBUY804WkBVm2mILzAw0Iwg5XI57rjjDojFYuzfv/+6r99PP/2EBx98EJs3b8bYsWOxdetWfPbZZ8jNzUVYWBheeOEFVFVVYdu2bQDAKYM8+eSTePzxx3H27Fk89dRT+OGHH6xWwuxQO5nO9kcY7zyomVKfPn06ZSPcHtfqbeksGhsbUVlZCVdXVyQkJACwToK/vLwcly9f7lEV4vZ5HKqflZeX1+U8Dm1StJZJWk+DGm1FREQgNDTUZFIxXtX3tIBlZ0FlbsLDw7sd4zeGsbwL7ckxVqSg3jC0XPx6CUEulyMtLQ0hISHXTZC2QHvdvfYW1F5eXlAoFPD39zdzE21ra8OCBQvA5/Px888/W4Wg77nnHjQ0NOC1115DTU0NoqOjcfDgQS4MV1NTY1JQMHDgQBw8eBArV67EJ598gn79+uGjjz6yGsEADraTIYRcU8r/9OnTGDp0KAIDA1FdXY3c3FwMGTKk2yv8s2fPYuDAgQgODu7usFFRUYGCggIunh0XF2em7trVsdHGTalUiri4OIfQkaKy9DSsJpfL4ePjg8DAQLM8DiEEJSUlKCsrQ0xMjNXiuz0JSpBRUVEW7w9jApZKpdBoNCaqA/ZOljc0NCAzM9MmMjeWYOwNU19fDwAmBNzV8ItMJkNaWhpnE97b0NzczOVQtVotxGIxWJZFbW0tJk6ciMWLF6OtrQ2HDx92yB4ra6HXkcyff/6J0NBQyOVyVFRUIDY2FgEBAd0+519//cUpq3YVhBAUFBSguroacXFxnEdKbGzsdXnAaLVaZGVlQaPRIC4uzmGbKY3zOE1NTSbVWVVVVZxMf1cN4BwBtIu/swRp7PgolUrR2trKiXleT6NfdyGRSJCTk4Phw4dbrcm1KzBufJRKpWhra+NKgv39/a9ZFUlDfNTuubdBrVYjNTWVK7KglZ0HDx7EK6+8gubmZri6uuKll17CggULuuU42lvQ60jmr7/+gkajASEECQkJ1y0ZQqtVuvpHpqW4bW1tSExMhJubGyorK7nmwu42trW1tSEjIwOurq6c9XJvAF3F1tXVQSqVAjDU2/ft27dbeRx7gXbxV1dXIyEhodsrTOoLQ/NaIpGIy+PYugelpqYGeXl5GDlypMOocNMFSX19PafCQHc57UuCaQic5iB7G2iZuJeXl1mjqEajwQMPPIDq6mrMmTMHx48fx+nTp3H33Xfj+++/t+OobQeHIhnA8AfqCHK5HGfPnoWLiwvGjBljleqHjIwMeHt7d2m1RJ00XVxcEBsbC6FQCEIImpubuR6QPn36cPL8nR0n3V737du3xytorAG1Ws15eYSGhnKhpO5ej56GcRd/fHy81TTPjMU86+vrubwW/bGmnEtFRQUuXbqE2NhYhw1R6nQ6k+tBpW7otTC22u5toF33Hh4eZlWIWq0Wjz76KC5evIjjx49zZnBNTU2orq5GVFSUvYZtUzgcydBdSntIpVJkZmbCxcUFwcHBGDJkiFXOl52dDVdX104fr7GxEenp6ejXrx9XCmosI8EwDBQKBSQSCZe3oNU3gYGBHVa+0dUnTTD3NsjlcqSnp3Nd2HRlSsNIxtfDx8eHW9U7ihyLcRd/QkKCzfIplvJa3t7eJh453UVJSQlKS0vNqi8dGcYlwbW1tWhra+M68e0RZrweaLVapKammoh1Uuh0Ojz55JPIzMzE8ePHe52V+PXA4UmGmnAVFRUhKioKzc3N4PP5GDZsmFXOl5eX1+njVVZWIj8/H8OGDcOAAQMs2re2fyBomEAikaC5uRkeHh4IDAxEYGAgN6FQDa+RI0f2mNWtNdHU1MR541yrAkilUnHXwziPYy3jre6AuinyeDxuZ9pToNeDhpFcXFw6DCN1BEIILl++jMrKyusK8dkTjY2NyMjIwMCBAyEUCrmSYKonRsOMjla9R6HVapGWlsaFuY3/bnq9HsuWLcOff/6JEydO9Moqy+uBQ5MMlVBpampCfHw8vL29UVhYCL1ejxEjRljlfJ05HiEEhYWFqKqqQlxcHOdUaOwB05nJwLiprb6+HmKxGAzDQKfTISEhoVcmyGtra5Gbm2si0tlZGHfZ19fXg8fjmfTj9MSE4khd/Hq93uR6sCxrojpgifzovSmRSJCYmNgrTa8owQwbNsykAIfqidHrQav3aJjRUWwJtFotLly4AJFIhNjYWDMrhWeeeQbHjx/H8ePHb+gEf0dwOJLRarVgWZZ7+Hk8HuLj47nwRVFREZRKJUaOHGmV8126dAlqtRrR0dEW3zdO8NNCA2s0WLa1tSE9PR1arRaEELtMsNcDKuFTXFyMmJiY696BtS8HpnkcWp1lCxl6an/t7++P4cOHO1RYhop50kWJsX0DNSIzziElJiY6bBXi1UDLrCMjI6+6wrdUvUebYq/WaW9rUIUPoVBokWCef/55HDx4EMePH++VVXLWgEOSDM17+Pv7m/lElJSUcIlZa6C4uBgymQyxsbFm79EEv1gsRlxcHJfgN7Yb6M6NTfMXtLyRYRi0tLRAIpFAIpE4fKKclm5LJBKb+Kj0RB6HFllYowu+J9C+XJzKFbEs2yvNxoArShbdKbPuqNOeLtJ6oipTp9MhPT0dfD4fsbGxJgtDlmXx4osvIiUlBSdOnLBaDrk3wuFIpqSkBLm5uYiIiMCAAQPMHv7y8nJIJBIkJSVZ5XxlZWVoaGjguvQpmpqakJ6ejuDgYM4O2BoeMFRkryOJFUsTrK+vL9fwaO/JRK/XIzs7G21tbYiPj++R1TPNW1ATMjc3N66QojsrWOrE2VNNitYGdYNUq9XcYoeu6K/HI6cnIZVKkZWVZRW7amOtufr6eiiVSvj5+XVZpr8r0Ov1nNNmXFycCcEQQvDqq6/i22+/xfHjxxEZGWn18/cmOBzJUDXgjsovq6urUVFRgdGjR1vlfJWVlaipqUFycjL3WlVVFfLy8rqU4O8MaHlpV1Zu7QsHPD09uQnW3d29R1fgGo2GS5DTnV1Pg/pcdDePQ33fR4wYcV0qD/YCdVgFDDp9PB6P88iRSqXcBEuvib0XJZZQV1eHnJwcqxCMJSgUCm6XQ3d9xhYO19ujpNfrkZGRAZZlkZCQYEYw69evx9atW3H8+PEbtiy5K3A4krmWBXNdXR0uX76McePGWeV8NTU1KCsrw5gxY0AIwcWLF1FRUYG4uDjOW6KrCf72oMnZ2tpaxMXFdbu8lIYIJBIJGhoaIBaLuUo1a+hEXQ3UBM7Ly8usPNNeoCtYuuu7Vh6ntLQUxcXFDt1DcjVoNBqTBLMlQqWqE/X19Vw1I13R2ytvYQxaKNJTjaI0/E6vCQDuHumomOJq0Ov1yMzMhF6vR3x8vMmukRCC//73v/joo4/w22+/WQzB/xPR60iGmipNmDDBKueTSCS4dOkSRo8ejaysLMjlciQkJMDDw8PEza674TGdTofs7GwolUrExcVZrS/EuBJJKpVyIRNbFA7Q/EVISIjDqeBSGHugSKVSyGQyk/6Tqqqq6+7itydoIQxt8usMybdX07Z3cUlNTQ3y8/MxcuTI65KC6i4sFVP4+PhwJOzm5nbVe5tlWa7ZOiEhwYxgPvroI7z77rucVL4TBvQ6kqGqshMnTrTK+RoaGpCdnQ2hUAihUIi4uDiIRCKrVJAplUpkZGRAJBIhJibGZuEl45g0LRywluNlXV0dcnNze13+wjiP09DQAIZhOLVcW+/6rA1agHI9dsOWJPqNw2q2FvOkYUpH2kUqlUruejQ1NUEsFnPPTHvpH5ZlkZWVBbVajYSEBJNnihCCzZs34/XXX8fhw4cxZswYe3wdh4XDkYxer7doSkYhk8lw7tw5TJ482SrnKy8vR15eHkJDQzkzLWsk+FtaWpCRkcGZRPVUeMmahQNlZWW4fPkyoqOjHUYDqyugXfxKpRIDBgzgiNjeK/qugErdBwcHW01qiBDChdVoOTDN9QUEBFi9y76qqgqFhYWIi4uDn5+f1Y5rTdDIACUdvV7PhdX8/PxQWFgIpVKJxMREM4L58ssv8eKLL+LgwYO46aab7PgtHBO9jmTa2tpw6tQp3H777df9IFRXVyMnJwcAMGXKFADgdjBA9xP8dPU/ePBgixVyPQlLhQOUcDoqHKC5qZqaGq4JtreBdvHT6h86MRjv+uiK3rhc3Bb9ON0FVSIODQ21qdmbcZNwQ0MDhEIht6K/XnHTyspKXLx4EfHx8fD19bXiqG0HY+mf+vp6tLa2gsfjYcCAAQgODuZImBCCb7/9Fs899xz27duHW2+91d5Dd0j0OpLRaDQ4duwYpkyZ0u0VKFXaLS8vR2RkJPLy8jBlyhSTBH9XCUahALRag8lYWVkZRowYwTUoCgSAIzRity8ccHFx4SrVaAiJqizI5XLEx8c7jLZYV9DZLv5r5XHs2T1PpXp6WonYuMteKpVCp9N1uym2oqICRUVFvUpLzRiEEOTk5KClpQWhoaFobm5GQ0MD9u/fj/r6egwYMABbtmzB7t27MXXqVHsP12HhcCRzLQtmvV6PI0eO4LbbbuvWqpMm4mUyGZe8O3HiBCZPntzt8JhCAezdy+DiRYPAX//+oXBxuRLj9vQEZs/WOwTRUNDwgEQiQX19PRiGQZ8+fdDS0gKBQID4+HiHWtV3FrTRtU+fPl3u4lepVBwJG/fjWMvlsbOgfTwRERFdluqxJjoiYZrvu1oJPVWD6M0Ek5eXh5aWFiQmJnI5K5ZlcfToUWzZsgW///47dDodpk6dilmzZmHWrFn/OF2yzsDxu7bagW7ddTpdlydBpVLJSdGPGTMGIpGI868pLy9HUFBQt/oK2to0yM2VQCjUIS4uDALBlZitSgXIZMBVNmd2AZ/P58qfWZZFXV0dCgoKOCVp6vTpiIoDHaGlpQXp6end7uKn6r/9+/eHTqfjqvcyMjLAMIyJjpit8ji0h2TEiBF2MRszBsMw8PT0hKenJwYNGsSRsFQqRXFxMZcoby/mSQ3fEhISemWolRCC/Px8NDc3IykpyaQogsfjQa1W49SpU/juu+8wdOhQ7N+/H99++y3q6+uxdu1aO47cMdHrdjIAcOTIEYwZM6ZLgpK0Ki0gIICToqfJ/YqKCtTW1qK1tRXe3t7c5NuZTmGFQoE//sjGmTNhiIrqCw8Pfrv3geZm4O679XDU541Ozn379sXQoUM5qwKJRAKFQuHwzX3AldX/kCFDrC5C2FEeh14Ta+34aAWWvUp8uwJjjxypVMp5whBC0NjYiMTExF5ZKk4lkxoaGpCUlGR2vx86dAgPPfQQvv76ayxYsMDsd3tT1WJPweFIpjPumMePH+/SNrympgY5OTkYOnQoZ4RkKcGvVqu5JHljYyMnwBcUFGQxNNDY2IjMzEx4ew9AamoEfH3Ncy+OTjLUpnfw4MEWTaKUSiVXqdbZwoGeBvXiiYqKsnkXv3FllkQisVoep7y8HEVFRQ5dgdURaP/JpUuX0NzcDIZhHNIz6FqgTdNSqRRJSUlmi8zffvsNCxcuxNatW3HffffZZYzh4eEoKysze33p0qX45JNP7DCia6PXhcsAQ6jnar00FIQQFBUVoaysDHFxcQgICDDr4DdO8IvFYi5cQhvZJBIJSktLIRaLERQUxMXnaVlmZGQkPDxCkJZm069sE1CZm6ioqA7lPVxdXREWFoawsDCTwoHi4mK4uLhwhGOv3hNaZk0tGGwNhmHg4eEBDw8PDBw40CSEdPnyZYvFFFcD9UsqLS1FYmJirwwvAYadZFtbG8aOHQs+n89dk0uXLpnIuvj4+DjEwqQ9aEVlRwRz6tQp3Hffffjkk0+wcOFCO40SOH/+vMncl5OTgylTppjtqhwJvXInc/r0aQwdOvSqvRtUyLGlpYXzaulug6VxklwqlYIQAkIIV6Isk/Hwv//x4ePTO3YylHypP053ErPtrwntPQkMDLyusleJBFCrzf8uYjGB8Z/b+Ds4Spm1cR6HSpjQ1bylPA79DlSJoDf6CRl/h6SkJLOdnKVrYpzbcgQxT/odampqkJSUZLbzOn36NO68807897//xWOPPeZQJPnMM89g//79uHTpkkONyxgORzKAQWX2ajh37hxCQ0M7rOSgJax8Pp+rkrJGBz9t7pPJZPDx8UFTUxNYloWrazDOnAmDv787XF1NJ1eVyvDjKCSj1+uRm5uL1tZWq/nYW9IQ647igEQCPPusCC0t5n8bb2+C997TIDDQcL78/Hw0NjYiISEBLS0eUCotH9PVFejXr+dvcZZlOfsG4w57436cgoIC1NfXcz5FvQ109V9XV9cpwzQaVqN5nLa2NpurJXcGdLFiiSTPnTuHefPm4c0338SyZcscaiLXaDTo168fVq1a5dAFB/ZfRnQDVwuXtbS0cEZU1IvGGh38KpUKGRkZEAgEGDt2LOct09LSgrKyesjl1aiqIvDw8ICnpxc8PNy5launp6FXxt6gCr4sy2LUqFFWS1jzeDz4+fnBz88Pw4YN4xrZSktLkZub2+nCAbWaQUsLAxcXAje3K8TQ1mZ4Xa1moNfrkJWVBZVKheTkZDQ2uuLBB0WQySz/TT09Cb79VtPjRMPj8eDr6wtfX19ERERweZyqqirk5+dzK/iRI0f2WoIxzl90Ju9CczU+Pj4YOnQo2traOMK5ePGiXay4L1++3CHBXLhwAXfccQdeeeUVhyMYANizZw+am5vx8MMP23soV4UDTH3moN20HaEjkqEJ/iFDhnANbFTgEkC3Caa1tRUZGRlc7wUNBRk/NIMHE7S0UKmOCrS1tXFOhkFBfeDublttqGuBlm+7ublh5MiRNivBZRgGXl5e8PLywuDBg7mJpLa2FoWFhfDy8jKxKrAENzcCDw/jVwhUKuZvH/ULYBgGSUlJEAqFUCoBmYyBWEzQnr8M5eNMh7ucnoJxHmfAgAHIzMyEXC6Hu7s7MjIyuDwOLQV2tMmsPWiJL63A6u4OxM3Njcv3GVtxU0dcW5eMl5SUoKKiwiLBZGZmYs6cOVizZg2eeeYZh/ybfPHFF5g+fbrD9+Y4JMlcC3w+30QVgBCCy5cvo6SkBLGxsQgMDLSaBwytvqKd1x0dw8PDMJGEhHgAGIi2tra/y4ArUVWV1+XSaGuitbUV6enpCAoKwrBhw3r0gTGeSDQaDbdytVQ4AFxNAVePrKxMhIeLLJKkiwtgaTF9jchrj4Jaeev1em43bKymnZmZCeDqeRx7gzYpNjU1WdWRUygUIjg4GMHBwSYl4xcvXjQR8/T397fKOUtLS1FWVobExER4mK5okJubi9mzZ2PlypV4/vnnHZJgysrKcPToUezatcveQ7kmeiXJCAQCbndinOCnvTPG+ReGYbrtAUO7lq9WfdUR3NzcEB4ejvDwcKjVai42f+nSJXh4eHCEY+syYGpxO2jQIISFhdn1gRGJRAgJCUFISIhJ4QA1QtPrQ6DVDgUhprelTqdHS4sMXl7eiIkZ5BBeNl0F1VLj8/kmMvHGTbGEEG5yvXTpErKzs3tUKflaIIQgNzcXLS0tFntIrAXj8GtERAS3G66pqUFBQQHXWhAQEABPT88u39NlZWUoKSlBYmKiWbFFQUEBZs2ahaeeegr/+c9/HJJgAOCrr75CYGAgZs6cae+hXBMOSTKdDZepVCpOBHHMmDEQi8VWSfCzLIuCggJIpVKrlJWKxWKEhoYiNDTUpDS6pKSEW81310r4aqisrERhYWGP9I90Fe0VB5qbm5GZ2QS5XA6VSgcvLwFcXFzA4/FQVyeDWOyGIUOGoBfyCzQaDdLS0uDi4oKYmJgOdycMw3B5HNoUK5VKuSZNGmq0R48Sy7LIzc2FTCYz64K3JRiGgbu7O9zd3REeHm4i5llWVgaBQMBdE19f32vu/CoqKlBcXGzRV6ioqAizZs3CokWL8NprrzkswbAsi6+++gqLFi1yiOq8a8HxR2gBfD4fMpkMZ8+eRZ8+fTgTJ2sQjFarRVZWFrRaLUaPHm311ZpQKETfvn3Rt29fk9U8rYaj4aP2fhZdAQ0fVlRUICEhweHVb+nKNSrKD+HhQjQ0sFAo1GhsVEGvZ8HnCxAYCDCMGoDlyU2l6txrPQ2VSoW0tDR4eXlxhSidQft+HLVabdKjRF1ReyKPw7IscnJyoFAoTHS87AGRSMT5AlGPHKlUivz8fGi1WpOdX/vClsrKSk6ws/3CsaSkBLNmzcKCBQvw1ltvOfRu+ejRoygvL8ejjz5q76F0Cg5ZwqzVarlciiVkZWWhpqYGQ4cOxcCBAwHAJAfTXYJpa2szSY735CqBPjBUzoUQAn9/fwQGBnYpNs+yLBczj4+PN4s3Ozponwx1LKUqBDKZFDxevVnhQHU145DVZYDhfkpLS+uWWOfV0N4VFUC37pXOgGVZZGdno62tDYmJiQ4rmmos5knl+Y13fs3Nzbh48SISEhLM+sLKy8sxbdo0TJ8+HZ988olDE0xvhEOSTEfumIQQFBcX4/Lly3B3d8f48eOtluBvampCZmYm+vbtazVzqO6ClkZTwtFoNCaeJx31nRjvwuLj462y4mxoACz1xopEgK0a7GkXf3sXRVo4QGV/XF1dERAQAL0+GAKB5di8vfpkZDIZLly4wOnB2ep+Mr5XpFIpVCqV1fI41G6YukE6KsFYAt35UdIhhCAwMBD9+/c3iRJUV1dj2rRpuPXWW7FlyxaHK7S4EdBrSIb6nDQ1NSE0NBT19fUYNWqUiURMd1cg1dXVyM/Px7Bhw+wqrW4Jxk6XxoKVNFRCJxGan3JxcbHaLqyhAVi3Tthhc+TatVqrEs3Vuvjr6kyVAHQ6HZqbmyGTScGyNeDxeNw1uV6jresF7dUKCwvDwIEDe3TBYqyr1n4135U8Dm081mg0ZnbDvQlU127QoEGc3XJ5eTn+97//4ZZbbsH27dsxevRofPXVV06CsRF6RU5GrVbjwgVDf8TYsWPR3NyM2tra686/GOcuekr7qqswllunfScSiYRLBnt7e8Pb2xs1NTUICAhAZGSk1SZYjQZoaWHg6kq48mC1GpDLgbo6BjU1AF0LiETA9diGGHfxJycnc30LtbVAZSWD114TorX1yt+YzxdBJHKDt3dfbNgwDCKRIdSYn5/PGW3RnV9Phj0bGxuRkZFhEzXozsA4SW68mr+aNH976PV6ZGZmQqfT9WqCqaurQ35+vsmzTQ0L//rrL2zZsgXV1dUICgrCBx98gNmzZ2PYsGF2HvWNB4ffyVALWkNiOIoT38vMzER0dHSnKkoswVheJS4urtflLgAD+ZaUlKCyshKEEE4huaul0U1NgCV3hcZG4L33ROjTx9AcqVIBqak8yGSASsVg/PgrRmze3gRLlui6RTR01axSqRAfH88VW9TWAkuXiiCVMigqYsDjgasuEwiA4cNZEMJgyxYNBgww3MbUOpeGjzra+dkCtFx82LBhCAkJsdl5uoOO8jjtNcT0ej2nChEfH98rqpcsQSKRIDs7GzExMWa2CQ0NDZg5cyaGDh2K9957D7/88gv27t2L/Px8XL582ZmTsTIckmSoBXNdXR2ysrIwaNAgDBo0CIBhEtFqtZxiKo21dkWYUa1WIyMjAzweD7Gxsb0q1mwMGuajVs90Aqmvr+90aXRTE7BxoxAtLebv8flATQ2Dvn0NJKNQAH/+yQPLGkhp8mQ9PD2BtjYD6axapTURsewMqNQNAMTFxZmsmktLGTz2mAgAQWEhD0KhYUx6PcCyQGwsC43GlGTag+78pFIpWlpaOqU40B3U1tYiNzcX0dHRXe6p6mkYa4hJJBIuj+Pn54fa2lrw+XzExcX1WoKRSqXIysqySDDNzc2YNWsWQkJCkJKSYvLs63S6XvudHRkOeUWNE/wxMTEICgoyqx4bPnw4hg8fjubmZtTV1SEvLw96vZ6bQDqqspHJZMjIyICvry9nXtbbQK9PeXk54uPjOf8RWtrZUWl0YGCgWZhEqwVaWvC3XtiVc7S1AQ0NDCwV+QkEACEGxWm6AVSpDMfpaMkiEgHtK6mpkOm1pG5cXAzkIhBc0YC7SvGhCYybYo0LB4qLi7nCge70KNXUXJGrqaurQ3FxKSIjE8GyvgAcbt1mgvYaYgqFAnV1dbh8+TL0ej28vLxQXl7eI83C1gY1r4uOjjYjmNbWVsybNw+BgYHYsWOH2eKyJwmmqqoKq1evxqFDh6BUKhEREYEvvvgCiYmJPTaGnoJDkkx5eTnKy8sxevRoeHl5mXnAGE+StHlt2LBhaG1tRV1dHS5evAiNRsOVddK4vFQqRU5Ojl0SstYCzV00NDQgOTnZYpivfaMjLY3Ozs4GIcREtgQwTOxubkD7Q9HKsvp6wy5GpTKQj3HYCgCUSkNo7b//FZrZTPP5gFhsyNesXKnliEahUHBhUGM9OFvCWHHAWIKeEjElnGv1KNXUMHj0URFkMkPFm1rtAze3m8Dn8+HpCXz5pQZ9+zo20RhDJBKhvr4ePj4+GD58OOd4WVJS0uk8jiOgoaEBWVlZFhU65HI57rzzTnh6emL37t12dXhtamrC+PHjMXHiRBw6dAiBgYG4fPlytyw3egMckmRCQ0Ph7+/PdfB3pv+FYRguCT506FDI5fK/V5jFyM3NhZubGxQKBYYPH+5w8fLOQqczKBCr1WqMGjWqUw8Kj8dDnz590KdPH0RGRnLlrpSIGSYICsVQeHuLQQkHMJBJaytw6RIDlcowsbCsYbciEBhKgzUaw+dOneKhoYGBXm8gFOMNiYsLQVwci+ZmBnV1VLRShpycHPTtGwo/v3A0NXWuHJoWGej1hh+53PBviQRon5sWi4GrGUwKBAIEBQUhKCjIpKkvNzcXer3exKqg/QrXIMoJEKKEUKiEn58XBAIeVCriEIKcXYFWq8WFCxcgEok4NQJj6Z/GxkaTBYqjecFQUJfa4cOHm6lbtLW1YcGCBRAIBPj555/tZilA8fbbbyM0NBRfffUV9xoV9L0R4ZA5GZZlodVqrSYRk5ubC6lUCrFYDKVSySWCAwMDe00+hpYoi0QixMbGXvcDTkuj8/Mb8OGHHnB1VcDPTwxPTw/weJ749VcRGhsZaDSGUBefT6DXA83NDPr1YyEUMpg1yzDr79vHh1ptCJV5eRFuwtdqAa2WwdixerS0MBAKAZlMjaamJnh5eXE5EW9vgjVrzMuhaU7GzY2goIDH9evo9YZju7kZyGX0aD3azxve3sCaNdqrEk1H1+VahQOXLwP33ksgEikRHOwJgcDAqgaDOgbbtmkwaJDDPVZm0Gg0uHDhAid3c7VdSkdeMJ2xcLA1mpqakJ6ejsjISDNFYqVSiXvuuQdKpRKHDh0yk5KxB0aMGIHbb78dlZWVOHnyJEJCQrB06VI8/vjj9h6aTeA4S5F2sIYHjFarRXZ2NlQqFcaOHQtXV1ezEmAfHx/OVtmeD8rVIJfLkZ6ebtXQEi2NDg/3RN++Qnh6akCIDC0trWhqakRTU3+wrAAuLgJ4eRkIgu5uKL+1tRmIRae78ppIZPihoFVrLAtIJGqwbD0GDvSBu7sbAIK2NkOZ9NXMUFkWiIxkuTyMSgXI5QyCg1kEBgJ9+phO6Eolg5aW7ikwG1sVDBkyhLtfqDijl5cXKivF0GpHwN//CsH0NlA9NZoPu9Y9ZSmPY2zh4OnpyRGOh4dHj4Wim5ubkZ6ejmHDhpkRjFqtxgMPPIDW1lb8+uuvDkEwAFBcXIxNmzZxZmN//fUX/vWvf0EsFuOhhx6y9/CsDockmW3btkEmk2HWrFkICAjo1g1L/VNcXFwwatQobuVvnAhWqVSQSCSoq6vjvE6CgoLsIsffEWgYYMCAARg0aJDNHl6tVgg3N7+/K/R0YBgeGEYHjUYNtZqAzxdAIBCBxxNArzeExpqaDGPR6w0eMDod0+6YhpCaXA7U1yvQ3KzE0KF94OnpYlJkYCm8VFtryAUJhTBpBmVZQzjOw8PwnlhMQIiB5K4ck0CpvPZ1qqpi0NZm/rqbGxASQv7+/yv3i1KpRGZmJpRKJViWhVwuB48ngFgshlDokI+SRajVaqSlpcHDw4PT/esqOhKt7Mk8DiWYiIgIsxC4RqPBQw89hLq6Ohw9etSh8h0syyIpKQnr1q0DAMTHxyM3NxebNm1ykkxPYvv27Vi1ahXGjRuHefPmYc6cOQgODu7UJGtQ9M1EUFAQIiIiOrzJXVxcMGDAAAwYMABqtZqrPKJy/JRw7OVcSLuVIyMjbZZHEgoNoaWWFoYTlFQohODx+HB3F/5NMFqoVFoolXKIxSKEhKghELjhyScNE7FMJoSbG0F2Np/buWi1QHGxIcTV1ESg1YrA53ugsZEHDw+CiRNZi/4vgIFgli0TobXVdIej0wESCYOICBZ8PpCTw8fFiwQCgaECbeJEfYfHbI+qKgb33COCXG5+P3l4EPz0k4YjGsBQVl9QUAAASEpKgqenG1xdNWBZFVpamqHR8KHVukKtFqG4+ArhurkB/fs7TuiMEoynp2eXBDuvBmPRSprHoT1DLMua6KpZK4/T0tKC9PR0DBkyxEylQ6vVYvHixSgrK8OxY8e46ktHQd++fTFixAiT14YPH46UlBQ7jci2cEiSefjhh7Fo0SKUl5cjJSUFKSkpeP755zFq1CjMnTsXc+fORf/+/S0STm1tLfLy8rrccS0Wi9G/f3/0798fWq0WUqmUKxxwdXXlCKcnQgGEEJSWlqK0tNTmSgS+vsDy5VqTZkyJxBCO4vMJTp7kgxAxeDzx37kQQ36EYdTIzMzCgAE+YJhBkMmEUKsNTpRqtYFkDMlxFkKhHu7uQuj1PAgEBpfLDtyzARjkY1pbDTbMtBrNQFaGfhyxmIDHA/R6ApY1hOxaWw3VcFc7rjHa2gzfUSwmMO7PNCgamO5wdDodMjIyQAhBYmIiKiqEYBgGLCvicnoFBTzodADLEjz2mA4iEfN3tRkPu3ZpHIJoqCK0j48PRowYYZP7mFbpBQQEmORxLl++bOaP093wNG3QHjRoEEJDQ03e0+l0ePLJJ5Gfn4/jx4/D39/fGl/Lqhg/fjwKCwtNXrt48SInBnujwSFJBjDEgMPCwrBq1SqsXLkS1dXV2LVrF1JSUvDiiy8iPj6eI5yBAweCEIItW7YgIiLCYhNWVyAUCrmVmU6nQ319Perq6lBaWgqxWMwRji18yI29bAwrZs9r/9J1on3/il5v2BlY+mo8HgORSAwXFxGGDBkCtVqKsjI5MjL6gGUBlmWg1xv/Ih8ymQsUCgOxiEQGglAoDGEyudyQY6muvkI8DQ2G/7q6Gnpx1GogJ4cHhQJQKBhkZfHR3MxAqzWMUSg0EM3BgwxcXYEJE/QWx24JYrG5o6ZxLodWXwkEAsTHx4PP58PVFfD0NBCqSmUYv07HgGEAsZiBr68ADKODUqlDQwODtLRCAB52NR5TKpVIS0vj8no9kTNpn8dpb8VNzfu6ksehwqMDBw40m5T1ej2WL1+OCxcu4MSJEw7bFLty5UqMGzcO69atw913342//voLW7duxdatW+09NJvAIavLrgZCCOrq6rBnzx6kpKTg5MmTGD58OFxdXVFcXIwjR45g6NChNjk3bXKsq6tDfX09BAKBSZPj9T64Op0O2dnZUCqViI+Pt1teqKYG+L//E4EQgsxMQ6e9SGTYneh0wKhReuj1DF5+WQuhEPjXvwTYv18AgIAQFjqd4f+pnbKLi4FY1GoGwcGG/x85kkVeniGcJhQCQ4awXFWaQGDQRgsMJNDpDDuLwkIDCbW2MvDyYiGX86DXG8iF/p6np+FWvukmPfr3v3p12aVLDBYsEMPLy7wJtbWVwY4dagwYcKVZ1N8/Bmr1ldCSRGLYsYnFBAoFg2XLDFVwbm5XCh+USsMOa+PGEnh4VHGKA3Ri7akwrFKpRGpqKvz9/REZGekQ/WHGeZyGhgYIhUKuT6mjPI5cLkdqairX52YMlmWxYsUKnDhxAsePH7eLblxXsH//frzwwgu4dOkSBg4ciFWrVt2w1WW9jmSMQQjBxYsXMWvWLDQ1NQEAgoODMXfuXMyfP9+mKzaWZbmueqlUCoZhOMLpjuGYWq1Geno6BAIBYmNj7SpK2NAArF8vRG0tg5wcHsTiK2XJLi4GklGpGKxerUVQELB/P7B4scvf4SJAo2FgTDJ8PgHDGMjjppv04PMZPPGEDhs3CiAWG3IgdIHf1sagocFQCuzqSpCZyYdOB45QjEH/tHy+4f+jogwVaBs2aDB0KLlq+fK1SObbb1vQ0nIePj4+8PWNwsMPu0AmMz+OpyfwyitaPPmkCJ6exKSU2tBPw2DXLjUiIgiX96MTq5ubG0c4ttgVG76PwdMmICAAw4YNcwiCaQ/jPI5UKuXyOMZ9SgqFAqmpqejfvz8GDx5s8vssy+K5557DoUOHcPz4cTMCcsK+cNhwWWdQUlKC6dOnIzk5GV999RU0Gg327t2LXbt24YMPPkBoaCjmzp2LefPmXbMPoKvg8XhcbNm4qz4nJ4frqqfJzmudl3a/+/j4WC0Z2x0YOvwNk9Ajj+hQVwd89ZUAfn4ENLQtEBgmfGPXSU9PgwKAgQRYGJo6TSczliVgWQKlsgnu7i7w8mIhFntw4ptXQFBTw6C83BB266y7JSE0H8NAp2Og1XZu7dS+zNnQ72NwgoyLMzSwlpQYREHFYlggkc47cBrn/ajigLH0T2cVBzoLhUKBtLQ0BAcH29TT5nrRPo/T2trKKUfn5OTA29sbMpkM/fr14zQMKViWxdq1a7F//34nwTgoejXJBAQEYOXKlVi2bBl4PB7c3Nzw0EMP4aGHHoJMJsOBAweQkpKCqVOnIjAwkCOcxMREqxOOcVd9c3MzJBIJCgoKoNPp4O/vj6CgIIt6ak1NTcjIyEBoaCgGDx5st4mgoQF44w0hmpuvnF+lAnJzeXB3J1zlll5vueTYoMxgCIXRr0B3HobvxIBhCBjGFWq1CpmZeWhujoJQyIdYLIZAIOR+z/gStD+W+XkNITyGAUpKDH/TF18UIjiY4MMPNegoLG+Q0SGQyxkTomFZFgwjR2hoH0RGhpn8PVxdr56/sURYHcGS4oBEIjFRHLieiiy5XI60tDT069cPQ4YMcViCaQ9j5Y4hQ4agoaEBmZmZEAgEqKysRHNzM8RiMVQqFZKTk/Haa69hx44dOHHiBIYMGWLv4TthAb2aZDw9PfH00093+N69996Le++9FwqFAocPH0ZKSgrmzJkDHx8fzJkzB3PnzsXo0aOtalbEMAynpxYREYHW1lauLDo7O5sjHH9/f9TX1yM3N9chzNI0GoYLUdGJVKUC3N0NOYeGBtNdhY/PldyDTCaDXi8AIYYdTHtC0OsNJEAIAx7PFeHhLhg9OgEHDvCg1cpRW9sMgYAPV1c3sKwbRCIRBg8mEAgI0tP54PEM+SBCTIUxDceEEbER8HiGirGWFuZvkzPL7BQSYihTNq4ia21tRX5+PgYP7ouxY8O5PpqKCgMRNTUZChX4fMN3p0n/lhaGu07tm0rd3ck1y6rbL1LoSp5WZPXp06dLTpeUYEJCQuy6cLleKJVK5OXlISQkBBEREdBqtaivr8ehQ4fw/PPPw8PDA21tbfjwww/NdjhOOA56dU6mO1AqlThy5AhSUlKwb98+uLi4YPbs2Zg/fz7GjRtnMz0mY4fLuro6tLW1gRDC7WDsbQxVU8Ng7VqhWfiqvt6gxvzCC1oEBV25Vaiqck1NDfbtq8Abb4xHS4vAKGx1BfSr+fsTfPmlGiNGGMqYV6wQwtUVcHFhoVaroVIp0dKig1wuAiCGu7sQaWli8PkEWi3DSfx3BD7f8DNunB4se3ULgPagK+ahQ4ciNDQUVVUMFiwQQaEwVLFVVzMmZd70NiEE8PMj+PFHDfz8zM91vX0y7Z0uvb29ubCamwX2kslkSEtL4+6r3gparGApl0QIwdtvv42ffvoJMTExOH36NNra2jBnzhx88803vZZUb1T06p1Md+Dq6oo5c+Zgzpw50Gg0OHbsGHbu3IkHH3wQDMNg1qxZmD9/Pm6++War6ppRGRcPDw9otVpoNBoEBQWhubkZJ0+ehK+vLydvY68yV0twcTH8BATALPRUXl6OoqIi3HlnLAYM0GHdOgYikaF/hTYjCgQGQtJoGPzf/2kwYYJhwpVKyd9NoIBSyQPgCsAVnp6Au7sGUqkOCoUMLCvkjsMwV3JGluDuTkxyRhUV5v04Li4EffuavkZzacbaV21thnJpsZhAJMJVJy6NhoGnJ0FEhPXXa+2dLmlyvKioyKxwgJb3Wqq+6k2g/Tz+/v4WCeajjz7Cxo0bceTIESQmJoJlWaSlpSEzM9NJMA6If9xOpiPodDqcPHkSO3fuxJ49e6BWqzFr1izMnTsXt912m1Umfr1ej+zsbLS1tZmUKCuVSm6HQ1ertBenp/TUOtrJyOWGncy6dVpOvp4QgqKiIlRVVSE+Ph7e3t6QSIDnnhNZND8DDKoC776rMTE1k0rxd0jLFFIp8NJLIvB4LM6eNVSXGcRSAZa9kkvj8a7sbBjGsKMgxLC70GoZhIcTiESmt7enJ/DFF2r07QtkZDAoLm5EWVkZBg4cCN+/G4a8vQnc3YG77jJUn/H5QF6eoU+HgpIeYNitHDmiQlRUzz1KxoUD9fX14PF40Gq16Nevn1UtuHsaarUaqamp8PX1NasOJYRg8+bNeP3113H48GGMGTPGbuN85ZVX8Oqrr5q8FhQUhNraWjuNyHHxj9vJdASBQIBJkyZh0qRJ2LhxI06fPo2dO3fimWeegUwmw/Tp0zF37lxMnjzZYpjiWtBoNMjIyADDMEhOTjYJj7m6uiIsLAxhYWFQqVSc2sDFixe5voqOwiM9Depn09jYiOTkZK7XIzDQQCKWSAMw9JO0d8009MuaT8wsaziGSMRg3Dj2b5IBGhv1uHRJAF9fJZqbRRAIAJVKAOM8kMEGwGDV7OVF4OJy5fgGmwEGKhWDjAxg2jQR1Oog8Hh9TSYzkQjYvNmQtachsn79WBQXX5m4vb0BodBAalfbXdkKxoUDjY2NSE9Ph7e3N+rr63Hy5EmbSLnYGlTyxtvb2yLBfPnll3jttddw4MABuxIMRVRUFI4ePcr925q53RsJvePu62Hw+XxMmDABEyZMwIYNG3Du3Dns3LkTa9euxeOPP47bb78dc+fOxe23327RNKw9FAoF0tPT4eXlhaioqKvejC4uLggNDUVoaCg0Gg0kEgkkEgmKioq4Dmkqb2MLtBeMNP433YkplUokJyeb7bIMJHL9q3mxmMDLi6C11TDJMIzhx91dAFdXHkJDXaFSMWAYPdRqAr3e0JOj0bAgxGAP7eJCTPpvKGjxQkFBHTSa/hCJmL/zK4Zx63QG0pDJGOh04IiFZU1zTa2tBvUDHx/7BgKampqQmZmJiIgIhIaGciXAEokEly9fRk5OjomUiyOFYo1BVaGpplp7gvn222+xdu1a7Nu3DzfddJMdR3oFAoHAzLvGCXM4SeYa4PF4GDt2LMaOHYt3330XFy5cwM6dO/H666/jySefxJQpUzB37lxMnz7dYkNdc3MzMjIy0K9fvy73KohEIjM9NYlEgpKSEri6uiIwMBBBQUFW0VMTiQh8fAiam81Nt3x8CHg8LS5cSAdgEIi0ZaFCUBDwwQfmu6KKCgbPPy+EUGjIkfD5fHh7G4hBqyXo168NKhUBywoQFETA5wtBCN9MYqasrAx1dc1gmAEQCGBGRJRMaBGDpcgTfc+4GKCn0djYiIyMDERERHDVie3N+2jhALW2uFbhgD1ACcbDw8Miwfz444949tlnsWfPHtx66632G2g7XLp0Cf369YNYLMbo0aOxbt06Z5WbBThzMt0Eyxoa9nbs2IHdu3ejqKgIt912G+bOnYuZM2fC19cX27dvh0qlwrRp08yE/K4HVE+NxuNFIhFHONfTOW7cjGkMQlQoK7sAV1dXjBw50m5hgZISBo88IoKrK0FhIQ9a7RWrAb0eGDqUhbs7i/p6Fp6ebeDzVeDxDH04YrEYarUAdXVqPPtsKry9Y/DAAz4WBTLVagZbtqixfr0Ily8z4PMNpGJcwk0vgYcH4OpKcOyYGmFhPfco0Wq4YcOGdVqh21hpvLGxEe7u7hzheHp62iVprtVqkZaWxt1b7XNJKSkpWLJkCf73v/9hxowZPT6+jnDo0CG0tbUhIiICdXV1eOONN1BQUIDc3FybCtr2RjhJxgoghCA/Px87d+7E7t27kZubi7i4OGRnZ+Pdd9/FI488YrMHmOqpUXkbPp9vIm9zveelagTWNEzrLijJXFFhNnw3tdpAju+8o4FYTPDvf4vg4kIgFhOukk+j0UKj4UGj4WPr1ha0tQVgwQLXDkkmJUUNoZBg+XIRPD0NjZgq1ZWdDSGGbv+33tIiMpL0KMHU19cjKysLw4cPR9/2pXKdBF2oSKVS1NfXc/dNQECA1RQHrgUqPioWiy0qcuzduxeLFy/G999/j7lz59p8PNcDhUKBwYMH4/nnn8eqVausfnyWZUEIAZ/PByGkV1XROcNlVgDDMBgxYgT+7//+Dy+++CIee+wx7NixA/Hx8Vi5ciV27tyJuXPndskTp7MwJhWWZTlP9qysLDAMg4CAAAQFBXVr4qCeHSEhIQ7RNe7iQjj1Y7XatGfG3Z2AxzNUgl1RSGYAiAGIoVK1Qa/Xw8eHQXX1ZZSX14CQ5L/DXVe+l0535ZjBwVdUmt3cDIrQFG1thlDZ0KE9SzDUp2XEiBEd5gOKihiLOmuA4doMGUK4fEJwcLCZLBLLslwOx1aFAzqdjrMTt0QwBw8exOLFi/HNN984PMEAhlLzkSNH4tKlS1Y9rkQiQWBgIHd9qqqqsGXLFgDAtGnTkJSU5PAW8s6djBVBCMEDDzyA8+fP49ChQxg0aBDKysqQkpKCXbt24a+//sLo0aM5tYGOPHGsAZZlOXkbiUQCvV7PEY6fn981Q140HNNVXx5bo6bGIB+zdKnIJHdEBTjd3IBNm9ScOKbBbCwfGo0W0dHR8PQUIDiY4Pff5bj77j7QaAy7EoZhuB+RCDh4UAUvL+D++0VmmmWAQVpHrQa++06DQYN65hGSSCTIzs5GdHR0hzL2RUUMYmOvrt6dmanEkCGWx2xcOCCVSqFUKuHn58ftcqwxoVGC4fP5iI2NNbsXjx49ivvuuw+fffYZFi5ceN3n6wmo1WoMHjwYTzzxBP7v//7PKscsKCjA0qVLce+99+KJJ55AS0sLBg8ejNjYWOTl5SE0NBSzZs3Cv//9b7sZK3YGTpKxMvbv348xY8aYmSURQlBVVcV54pw5cwYJCQmcJ054eLjNCIeaR9FeHK1WayJv0/4hp46cI0aM6HY4xpa4eJHBHXeIIRSah7q02iuqx1qtFhkZGQAMFrftV+QZGQyamw19Sk1NTWhuboZKpUJIiDtuuskdOl0gHn/c46q7gm++0XD9Q7ZEXV0dcnJyMHLkSAS2rwU3Qno6g5tuujrJ/PGHEvHxnRuzQqHgCKczigPXgl6vR3p6OhiGQVxcnNm9d/LkSSxYsACffvop1yDtiHj22Wcxe/ZsDBgwABKJBG+88QZOnjyJ7Oxsq5mPVVZW4pFHHoFOp8ODDz6IpqYmlJWV4aOPPkJbWxteeuklnDlzBhMnTsSaNWvg5eVllfNaG06SsQOoJ87u3buRkpKCU6dOITo6miMcWyrmEkIgk8k4wlGpVFxPhb+/P2pqalBUVISYmBiHdBUErpCMh4e5tL5cbiCZ8HA1Lly4AJFIZHG13BHay7hotQHw9AyEn5+fWcm2qyt6hGBqa2uRm5vbKTM+a5OMMa63cECv1yMjIwMsyyIhIcHsb3L69GnceeedeP/997F48WKHJRgAuPfee3Hq1CnU19cjICAAY8aMweuvv25mq9xdsCwLHo+HmpoaPP3001AoFNDpdFiwYAGeeOIJAIa/x8svv4wTJ05g0qRJePbZZ7mGYkeCk2TsDEIIGhoa8PPPPyMlJQW//fYbIiIiOMVoW3riEEKgUChQV1cHiUQCuVwOhmEQHh6OAQMGOGys91ok88MPrZDJUrk4eXeT2LQxViKRoKmpqUf6lNqjpqYG+fn5nSZ9W5KMMdpXOF7LdEyv1yMzMxN6vd7irvLcuXOYN28e1q1bh6VLlzo0wdgKlFgAQ1EEbRNobW3F0qVLsWPHDixatMjEQVOv1+O1117Drl27MHnyZLzzzjt210FsDyfJOBAIIWhubsa+ffuQkpKCX3/9FWFhYRzhXM+EeTXQLv76+noEBwejubkZMpkMvr6+3KTqSE18VyOZ1lbgpZfOIiZGbNVqOOrkKJFI0NDQABcXF+7a2MpwjPa2xMbGdrostqdIxhi04IQSsnHhgL+/PxiGQWZmJrRaLRISEswIJi0tDXPmzMHLL7+MFStW/CMJxhiHDh3C9OnTIZfLcd999+H999+Hv78/VqxYgYyMDDzxxBN44oknODIhhOA///kPpkyZ4lB9RBROknFgtLa2cp44hw8fRlBQEObMmYP58+cjISHBKhOocRd/fHw8FxJSKpWcvE1LSwu8vb25SdVettAUHeVklEoCuVyDTz4pwe232y7HpdfrTVbx9fUecHUNgJ+fnwnheHig25VnlZWVuHjxIuLi4uB3NYvPdrAHyRjDuHBAIpFApVJBIBCAx+MhISHBLEGdmZmJmTNnYs2aNXjuuef+8QSzf/9+zJkzB+vXr8fmzZuRkJCAHTt2gMfjoaWlBcuXL0dRUREWLlyIJ5980qEWfx3BSTK9BAqFAocOHUJKSgoOHjzIeeLMmzcPo0aN6laDpHFiPC4ursNttlqt5iaNpqYmeHp6coRjj6qWykoDyRhL3rCswS7A05OHffsIrNj7elWUlBBMmiSGSmUYA2BQiWAYBi4uTLeaNCsqKnDp0iXEx8d3OcZ+vdVl1gTLskhPT4dcLodYLIZcLoe3tzeX8wOA6dOnY8WKFfjPf/7zjycYiv/+97947rnnEB0djaysLACGnbRIJIJMJsO//vUvXLp0CXPnzsXy5cvtvui7Fpwk0wuhVCrx66+/IiUlBfv374erqytmz56NefPmddoTR6VSIT09HS4uLoiJiek0SWk0Gi4s0tDQAHd3d05twN3dvccmispKhiMZajbWv39/DB0acl3+LV1Fbi6DadNcIBAYbAZYlgXLstBoCPR6Bps25WPsWE/Oq/5aOHOmBoWFVYiMjISnpyf3uocHEB7eue/VmT4ZW4MqYigUCiQmJkIkEnE5rhUrVuC3336Dh4cHoqOjsWHDBiQkJPyjSUav13PP4Pr16/Hpp5+iuroar732Gl588UUAV/I0bW1tWLp0Kf7880/s3LkT0dHR9hz6NeEkmV4OjUaDo0ePIiUlBXv37gWPxzPxxLG0O7FWFz91KqRhI5qnCAoK6jGZEtrPY6zf1ZOgJOPiQmBcJ6HRGHJEn312CT4+lVAoFOjTpw/Xb1JXJzYTI71wQYKVK/sCEJhdO5EIOHlS1WmisScIIcjJyYFMJrPYLHjp0iXMmTMH8fHxEIlEOHToEHx9ffHf//4XCxYssNOoHQO0+VKtVuPbb7/FkiVLsHr1arzxxhvcZ3Q6HRiGwblz5zBu3Dg7jrZzcHb893KIRCLMmDEDM2bMgFar5TxxHnvsMWi1Ws4TZ+LEiRCLxTh58iRKSkowfvz46+7iFwqF6Nu3L/r27WuSp0hNTYVQKOQIx9vb2yaEQ3tHHLWfh2EYDBgwAFFRoVy/SVVVFU6dKsVrr42HVivkQmt6vQ4qVRAUCiE8PWHig0OVoeVyO36ZToIQgry8PMhkMm4HY4ySkhLMmjWLK1Xm8XhQq9U4duyYVfX9eguMdzArVqzAvn378Ouvv2LIkCFYtGgRGIbBkiVLwLIs1q1bh7S0NDz11FP45ptvOIJxdJkZuwlRrV+/HsnJyVx8f968eSgsLLzm7508eRKJiYlwcXHBoEGDsHnz5h4Ybe+AUCjE5MmTsXnzZq7x08vLCytWrMDAgQNx9913484770RdXZ3Ve3H4fD6CgoIwcuRI3HLLLYiMjOQ6u3///XcUFBSgsbGRy1tcL6qrq7nmREckmPZwd3fHwIEDMWrUKERFJf9NMFrweArweAoIBCq4uPBBiEF8Uyi88tNL7GA4Db/m5mYkJiaaJaXLy8sxY8YMzJ49myMYABCLxZg+fbrdwj7r168HwzB45plnevS8xgTzwQcfQCAQoLS0FA888AAKCwshFAqxaNEifPnll3jnnXcwevRo3HbbbZg8ebJJP44jEwxgR5I5efIkli1bhj///BNHjhyBTqfD1KlToTC2H2yHkpISzJgxAzfffDPS09Oxdu1a/Otf/0JKSkoPjrx3gM/n45ZbbsHHH3+MsrIyrFy5Er/88gsGDx6M999/Hw899BB27dp11et9PecOCAhAVFQUbrnlFkRFRYFlWWRnZ+PUqVPIy8tDfX19twmnoqICBQUFiIuLu2r3u6NCLBaDz+fD01MMDw8BhEI9XF0ZsKzBKE2v16G3RbEJIdxCwhLBVFdXY+bMmZg6dSo2btzoMM6d58+fx9atW7lChJ4EJZhp06Zhx44diIuLw9tvvw2FQoE5c+YgPz8fAoEADzzwAFJTUzFz5kx8/vnnWL9+PQBYbcFmazhMTkYqlSIwMBAnT57EhAkTLH5m9erV2Lt3L/Lz87nXnnrqKWRmZuLs2bM9NdReh48//hhr167Fjh07MHXqVKSlpXGK0VVVVZg8eTLmzZvHeeLYCrQPiDZ/Uj016uDYmeKDkpISlJaWIj4+Hj4+PjYba2fRPvFPodMBOh2Dw4fNbZkLCxnMmSOGSKQGj6eBm5sbeDw+mpsJysp48PTUQSg0xN15PB70eh60Wj6OHFEhOtohHlcTEEJQWFgIqVSKpKQks2qn2tpaTJ8+HWPGjMGXX37pMA6ScrkcCQkJ+PTTT/HGG28gLi4OGzZs6NExnDx5Evfddx+OHDnC7U6qqqpwxx13oLm5GTt37sTIkSPNfs+4cdPR4TCjbPnbHP5qPQFnz57F1KlTTV67/fbbkZqaCq093aMcHL6+vjh69CimTZsGHo+H5ORkvP322ygoKMDp06cxcuRIvPfeewgPD8eCBQuwfft2NDU1WX01zTAMfH19ERkZiZtvvhkJCQkQi8W4ePEiTp48iaysLNTW1kJnLIX8NwghuHTpEsrLy5GUlOQQBAMYqr7EYgKdzqD6TH90OgZiMYElYQBCCHQ6HXQ6HUcwBlAnUAF4PDEIEUCnY6BWs9DpdLh8+TKkUin0xhaddgYhBBcvXuyQYKRSKWbPno2EhAR88cUXDkMwALBs2TLMnDkTkydPttsY2traIJfLOcFTvV6PkJAQbN26ldMuy8nJAQCT56K3EAzgIIl/QghWrVqFm2666apx2draWjP12aCgIE7iojfE5u2BBx54wOLrPB4PcXFxiIuLw+uvv8554nz66adYvnw5br31VsybNw+zZs1Cnz59rBr7NXZwHDJkCORyOerq6lBcXMwZP9FKLIFAgIKCAtTX1yMpKcmhFGfDwgh++01tMSlvqRmTEILS0jLo9QPB47mZmMTp9QY1ab0ef7uCGiZkHg8Qi1l4ejIoKCjgBE6p3pwtpPg7A0IIioqKUFdXZ5FgGhoaMHv2bERGRmLbtm12G6cl/Pjjj7hw4QLOnz/fY+e0tPu4+eab4enpif/+979Yt24dR8JBQUGIi4tDU1MT5syZg9zcXIfvh+kIDvFXX758ObKysvDHH39c87PtJzq62nb05Jejw9gT56WXXkJRURF27tyJr776CitWrMBNN93EeeIEBQVZnXA8PT3h6emJIUOGcHpq5eXlyMvLg0gk4kQVHYlgKDrbbEnDSgqFDD4+EVAqzSvGwsIINmzQIDjY9JiGPpmBICScEzgtLi5GTk6OCSH3pN7c5cuXUV1djaSkJDNF5ubmZk5d/IcffnAoPa2KigqsWLECv/76q5noqa1gnOSvrKyEu7s7fH194eHhgSVLlmDPnj3w9PTECy+8AMCww/H19cUnn3yCO+64A2+++SbeeOMNh68kswS752Sefvpp7NmzB6dOncLAgQOv+tkJEyYgPj4eH374Iffa7t27cffdd6Otrc2hbuQbBYaVdynniXP+/HmMGTOG88QJCQmxqSdO+45xHx8fTm2gpyYIa4BWXtHEeH29m1mfDGDwwwkN7fwj2V6Kv6euz+XLl1FZWWlxZ9na2oo5c+agT58+2L17t8P9nfbs2YP58+ebhO70ej2XA1Or1VYN6xkTw9KlS3Hu3DlUVlbiueeew/333w9XV1esW7cOu3fvhp+fH+Lj4/Hzzz9j2rRp+OqrrzBu3DjceuutWLdundXG1JOwG8kQQvD0009j9+7dOHHiBIYOHXrN31m9ejX27duHvLw87rUlS5YgIyPDmfjvARBCUFlZiV27dmHXrl2cJ868efMwd+5chIWFWY1wqCy8TqdDQkIChEIhVCoVJ2/T3NwMLy8vrhfHkUMJtHekqakJSUlJNpt026tG20r+p6SkBGVlZUhKSjJTo5bL5Zg3bx7c3Nywb98+h/y7yGQylJWVmbz2yCOPIDIyEqtXr7ZqKbVxiOz999/HRx99hLfeegupqan43//+h+nTp2Pt2rUICgrCmTNnsGnTJnh4eCA8PBwvv/wyWJbF+PHjcdddd+Hf//631cbVk7AbySxduhTff/89fv75ZwwbNox73dvbm7sxX3jhBVRVVWHbtm0ADDd3dHQ0nnzySTz++OM4e/YsnnrqKfzwww+488477fE1/rEghKC2ttbEE2fkyJEc4VxPo6dWq0V6ejqXM7IUy9doNBzhNDY2wsPDA0FBQXbTU+sIhBDk5uaipaWF6+/qCbRXjXZ1deVCatejGl1aWorS0lIkJiaayN4Ahl3VnXfeCYZhcODAgR6zQ7AGbr31VptWl6WmpuKbb77BjBkzMH36dADAd999hzfeeAPjxo3Dv//9bzMvmurqajz66KNcg3NvSvYbw24k09FN/tVXX+Hhhx8GADz88MMoLS3FiRMnuPdPnjyJlStXIjc3F/369cPq1avx1FNP9cCInegIhBDU19dznjjHjh3DsGHDOBO2rnjiaDQaXLhwAWKxuNOaalqtllOMbmxshKurK0c4Hh4edothU/0uuVxusXekp6DT6dDQ0MDJ/wgEAq503NfXt9PXp6ysDMXFxUhMTDQrdVcqlbj77ruhUqlw+PBhMwJydNiSZI4fP47p06dDLBZj69atuOeee7j3duzYgVdeeQVjxozB4sWLuS7+9PR0bNq0CUVFRTh8+LDDejt1BnbPyThxY4H2wuzduxcpKSk4cuQIwsPDOU+c6OjoDldkKpUKaWlp8PT0vOrnrgZjMy2pVAqxWMwRjq18XyyBNp+2tbVZlFexF6j3C90FAjDpVeromldUVKCoqAgJCQnw9vY2eU+tVmPhwoVoamrCr7/+avb+Pw2WkvPvv/8+XnnlFdxzzz146aWXMGDAAO693bt34/HHH8drr72GpUuXcq9nZWVh+PDhvT7X7CQZJ2yK1tZW7N+/n/PE6du3L+eJEx8fz01qZWVlKCsrg7+/v9XcQPV6PbeCl0qlEAgEXI7Cx8fHpgUL1KMnISHBYQimPeiCgBJOR6XRlZWVnPVA+/4kjUaDBx98EFVVVTh69GiXvG9uRLR3t2RZltvBvvfee/jggw/w6KOP4vHHHzchmtTUVCQlJZkd40aAk2Sc6DHI5XITTxw/Pz/Mnj0bI0eOxJo1a/Daa6/hkUcescnkT1fwdXV1kEqlYBiGIxxfX1+rPdQsyyIzMxNqtRqJiYm9ZhVKCOFKoyUSCZRKJfz8/CAUClFXV4fExEQzgtFqtXjkkUdQVFSEY8eOdcoe+kaG8Q7miy++wKFDhyCXy7nmSj6fjw0bNuC9997DQw89hCeffBJhYWEmxzAudb5R4CSZv7F+/Xrs2rULBQUFcHV1xbhx4/D222+bFCW0x4kTJzBx4kSz1/Pz8xEZGWnL4fZ6KJVK/PLLL9iyZQuOHj2KmJgYJCcnY/78+Rg7dqxNG/dYljWRtyGEdCpkdC1QH3tqM9xbCMYSFAoFioqKuJAateKmDbQ6nQ6PP/44srOzceLEiV6pIWcrvPvuu3jrrbewcuVKDBw4EI899hjGjx+PvXv3ws3NDZ9++ineeustTJs2Da+99hqCg4PtPWSbwiGaMR0BVLAzOTkZOp0OL774IqZOnYq8vLxrVisVFhaaJEIDAgJsPdxeD1dXV/j6+uLMmTNYt24dRowYgV27duH+++8Hn8/nTNg68sS5HvB4PPj5+cHPzw+RkZFoaWlBXV0dCgoKoNPpTEJGnV1V0pJrvV7f6wkGMIQ5GxoakJiYCDc3Ny7keO+990KlUiE4OBhlZWU4c+aMk2CMcPnyZXz99df44osvMG/ePPz+++8QiUSYM2cO17C6dOlSKJVKnDt37oYnGMC5k+kQnRHspDuZpqYmh9HS6i0ghGDcuHF47LHHsHjxYu51rVaLEydOICUlBXv27IFWq8Xs2bMxd+5c3HrrrTat0LLkT08Jh8rbWIJer0d6ejoIIYiPj3co+ZTuoK6uDrm5uYiNjUWfPn1M3qupqcGqVatw7tw5NDc3Y9CgQZg/fz4WLlzo8A6NPYH09HQsXLgQBQUF+OOPPzBjxgy88sorWLVqFRQKBX766Sc8+uijJr9zo+Vg2uPG/WbXic4IdlLEx8ejb9++mDRpEo4fP27rod0QYBgGp06dMiEYwOCJM2XKFGzevBmVlZVISUmBh4cHnn76aQwcOBCPP/449u/fD6VSaZMxeXt7Y+jQoRg3bhxGjRoFDw8Prow+PT0d1dXVJmKsOp0OFy5cAAAkJCT0eoKRSCScT097gmFZFu+++y4yMzPx559/QiqV4tVXX0VJSQkOHz5spxE7Fvz8/ODj44MNGzZgzpw5ePHFF7Fq1SoAhj6/HTt24MyZMwAMixpCyA1NMIBzJ2MRhBDMnTsXTU1N+P333zv8XGFhIU6dOoXExETOLnXz5s04ceJEh7sfJ7oHvV7PeZrv2bMHDQ0NuP322zFv3jxMnTrV5g2YVL5FIpFAJpPB19cX/v7+qKmpgVAoRFxcXK9P2EqlUmRlZSEmJsYs5MuyLF544QXs2bMHx48fx5AhQ+w0SgM2bdqETZs2obS0FAAQFRWF//u//+MaHW2F9ruO9v9ubGzEokWL8Msvv2DRokX47LPPABju33vvvRcymQz79u3r9eHUrsBJMhawbNkyHDhwAH/88UeXfeNnz54NhmGwd+9eG43OCZZlkZqaynniVFdXY8qUKZg3bx6mTZtmU08cwFC0UFNTg9LSUuj1evj4+HC9OI6m09VZ1NfXIzMzE9HR0WZK5yzL4pVXXsH333+P48ePX7UYpqewb98+8Pl8juy++eYbvPvuu0hPT0dUVJRNz11dXY2srCxMmzYNgHlfTFpaGu6//36EhobipptuQkhICH7++Wfk5eUhIyMDnp6eN3yIzBhOkmmHrgh2WsKbb76J7du3mxirOWE70JJhSjjFxcWYPHky5s6di5kzZ8Lb29vqJdFarRZpaWkQi8WIjIzkmj+pXhglnPbKxI6KhoYGZGZmYsSIEWaJaEII1q1bh88//xzHjh2z+QR+PfDz88O7775rFoK1JrRaLRYuXIiqqiqsWbMGc+fOBXCFaOh/acf+77//Dn9/fwwcOBAffvghfH19odPpen1YtStwkszf6I5gpyXcddddaGxsxLFjx6w8QieuBSpEuXPnTuzatQv5+fmYOHEi5s6dazVPHCp74+rqipEjR5qsRjUajYm8jbu7u4m8jSOisbERGRkZGD58uJkfEyEE7733Hj7++GMcO3bMLhbFnYFer8eOHTuwaNEipKenm2mAWRsXLlzAiy++CL1ejyeeeAJ33XUXgCu2I4Ahv6dSqcCyLNzc3Lj+lxuxD+ZacJLM3+iOYOeGDRsQHh6OqKgoaDQabN++HW+99RZSUlJwxx132OV7OGEAddKkhJOZmYmbb76Z88QJDAzsMuFoNBqkpaXB3d39mrI3Wq0W9fX1qKurMxGoDAwMhKenp0N4gjQ1NSE9PR2RkZHo16+fyXuEEHz00Ud49913ceTIESQmJtpplB0jOzsbY8eOhUqlgoeHB77//nvMmDHDpuekO5W8vDysXLkSLMviscce4/TI6PsKhQLvvPMOYmNjubmgN3rBWANOkvkb3RHsfOedd7B161ZUVVXB1dUVUVFReOGFF2x+ozvRNRBCUFJSwnnipKamYuzYsZwnTr9+/a758KvVak5XLSoqqkvxdCpQWVdXh/r6eohEIo5wbBHO6wyam5tx4cIFDBs2DCEhISbvEUKwadMmvPHGG/jll18wevToHh9fZ6DRaFBeXo7m5makpKTg888/x8mTJ22+k6FkUVhYiJUrV0KtVmPx4sW47777ABiKRJYtW4affvoJly9fNiPwfxqcJOPEPwqEEFRUVGDXrl3YvXs3zpw5g8TERE7Ac8CAAWaTPhXu9Pb2RlRU1HWRQns9NT6fbyJv0xOE09zcjPT0dAwdOtSssIUQgi+++AIvvfQSDh48iPHjx9t8PNbC5MmTMXjwYGzZssUqx6uvr+9QKocm7i9fvowVK1ZAoVDg0Ucfxb333ouVK1di+/btOHfuHIYNG/aPDJEZw0kyTvxjQQhBTU0Ndu/ejV27duHUqVOIiYnhPHEGDx6M4uJibNmyBYsWLcKIESOsSgLtFZEZhkFAQACCgoKsqqdmjJaWFly4cAFDhgxBaGioyXuEEHz77bd47rnnsG/fPtx6661WP78tMWnSJISGhuLrr7++7mOdPXsWkyZNwr59+zBp0iSLn6FEU1ZWhn/961+QyWRQqVTIysrCuXPnEBUV9Y8nGMDZjOnw2LRpE2JiYuDl5QUvLy+MHTsWhw4duurvnDx5kjPIGjRoEDZv3txDo+1dYBgG/fr1w7Jly3D06FFUV1fjySefxOnTp5GUlITk5GRMnjwZJSUliIyMtPoug8fjwd/fHyNGjMAtt9zCFRLk5ubi5MmTyMnJgVQqhV6vt8r5WltbceHCBQwaNMgiwfz444947rnnsGfPHocnmLVr1+L3339HaWkpsrOz8eKLL+LEiRO4//77rXL85ORkTJs2DQsXLsTRo0ctfobH44FlWYSFheHTTz+FUChESUkJzpw54yQYIzh3Mg6OrvYDUPfQxx9/nJswly5d6nQP7QIIIUhPT8fUqVPh7++P0tJSDBo0iLMo6GpOpjvnb2lp4XY4Go0G/v7+CAoK6pKemjFkMhnS0tIQHh6O8PBws/d37tyJpUuXYseOHTZvaLQGFi9ejN9++w01NTXw9vZGTEwMVq9ejSlTpljtHCzL4pFHHsHu3bvx008/dXhd6I6mvr4eSqUSoaGhToIxgpNkeiGu1g+wevVq7N2716RP56mnnkJmZibOnj3bk8PstSgqKsLEiRNx55134oMPPjDxxPnll1/Qr18/jnDi4uJsTjjtJfj79OnDEU5nOsflcjlSU1MRFhZmsfdr7969WLx4MX744QfMmTPHFl+jV+Oxxx7Djz/+iB9++AGzZ8+2+Bnj5sp/ahVZR3CSTC9CZ/oBJkyYgPj4eHz44Yfca7t378bdd9+Ntra2f5ScRXdRVVWF7777Ds8995zZZCGXy3Hw4EGkpKTg0KFD6NOnD2bPno358+cjOTnZ5l3ccrkcEokEdXV1UCgU8PPzQ1BQEAICAiyaoykUCqSmpqJ///4YPHiw2fsHDx7EokWLsG3bNudOF6YEYfz/S5YswTfffIPvvvsO8+fPt+cQex2cJNML0JV+gIiICDz88MNYu3Yt99qZM2cwfvx4VFdXmzXcOdF9tLW14ZdffkFKSgoOHDgAd3d3zJkzB/PmzcPYsWNtHi5pa2vjCIfqqdFKNbFYzBFMSEgIBg8ebEaYR44cwf3334/PPvsMCxcutOlYewNoiEuhUEAul0Mul5sQ89NPP43PPvsM27Ztw913323HkfYu/HO0DXoxhg0bhoyMDK4fYNGiRVftB2g/mdB1hHMLb124ublh/vz5mD9/PlQqFY4ePYpdu3Zh4cKFEAqFnCfOTTfdZJMdpJubG5djUalUkEgkqK2tRWFhITw8PNDW1oa+fftaJBiaJP/0009x7733Wn1svQ2UYOrq6nDXXXehtbUVZWVlWLx4MVatWoWQkBB8/PHHEAqFePTRR6FQKPDII4/Ye9i9As6dTC/E1foBnOEy+0Or1eL48eOcJ45er8esWbMwb9483HrrrRbDWtYELVMWCARQq9Xw9PREQEAAFAoFRo4ciT/++IPLNy1evNi5+PgbTU1NSE5Oxs0334zly5ejsbERt99+Ox555BGsXbuW29WsXLkSH374ISorK//xjZadgZNkeiGu1g+wevVq7Nu3D3l5edxrS5YsQUZGhjPxbwfodDr88ccf2LFjB/bs2YO2tjbMmDEDc+fOxeTJk62u2qxUKpGamoqAgAAMGzYMWq0WUqkUaWlpePDBBxESEgKZTIYnn3wSb7zxxj9GCfhaYFkWq1evRklJCXbu3AkAuPPOO5Gfn4/y8nJMnjzZxI794sWLiIiIsOeQew+IEw6NF154gZw6dYqUlJSQrKwssnbtWsLj8civv/5KCCFkzZo15MEHH+Q+X1xcTNzc3MjKlStJXl4e+eKLL4hQKCQ7d+6011dw4m/odDry+++/kxUrVpCwsDDi6elJFixYQL777jsilUqJQqG4rp+Ghgbyyy+/kNTUVCKXy83eP3jwIBk5ciSJiYkhrq6uZOjQoWTNmjWkurra3pemx8CyLPf/Go2GEEKIXq8nhBDy3XffkZMnTxJCCLnvvvtIUlIS0el05JdffiF8Pp/Mnz+f5OXldXg8JyzDSTIOjkcffZSEhYURkUhEAgICyKRJkziCIYSQRYsWkVtuucXkd06cDh6GSgAAFvJJREFUOEHi4+OJSCQi4eHhZNOmTT08aieuBb1eT/7880/y7LPPksGDBxN3d3cyb9488vXXX5Pa2touE0xjYyP59ddfyfnz5y0SzJkzZ4ivry955513CMuyRC6Xk507d5KFCxeSiooKe1+OHkdGRgYhhBCFQkEWL15MysrKSHNzM9FqteSPP/4gMTEx5K+//iKEEHL8+HESHR1NhEIhOXHihD2H3SvhJBknnLAz9Ho9SUtLI2vXriWRkZHExcWFzJo1i3z22WekqqrKImm0J5gjR46Qc+fOWfzsuXPnSJ8+fcjrr7/uECvvdevWkaSkJOLh4UECAgLI3LlzSUFBQY+d/5tvviEMw5Cff/6ZDB48mMybN8/k/V27dpHBgweT0tJSQgghKSkp5OWXXyatra09NsYbCc6cjBXBsiwYhnEmUp3oNgghyM3N5SwKCgsLTTxx/Pz8TO4vjUaD1NRUeHp6Ijo62uzey8/Px4wZM/Dkk0/i1VdfdYh7c9q0abj33nuRnJwMnU6HF198EdnZ2cjLy7O5jTZguGarVq3CZ599hpiYGJw/fx7Alb6Y/Px8JCQkYPr06QgNDcUXX3yBd955B0uXLjX5nBOdhF0prpdDr9eTY8eOkVOnTtl7KE7cgGBZlhQUFJA333yTJCQkEIFAQCZOnEg+/PBDUlxcTIqKisiECRPIwYMHiUwmM9vBZGRkkODgYPL8889zeQdHhEQiIQC4fEhPYMmSJcTT05Pw+Xyya9cuQojheut0OkIIIb/99hsZNWoUmTFjBtm8eXOPjetGhJNkuomCggIyYcIEEhsbS4YNG0bc3d3JwoULydmzZ+09NJvj008/JSNHjiSenp7E09OTjBkzhhw8eLDDzx8/fpwAMPvJz8/vwVH3brAsS4qKisjbb79NRo8eTfh8PgkODibJyckkJyfHLEyWk5ND+vfvT1asWOHQBEMIIZcuXSIASHZ2ts3OQcmDQiaTkdbWVvL8888THo9Hvv32W0IIIVqt1uRzarWa+39Hv46OCifJdBMPPPAAGT16NDl27BiRy+XkzJkz5IEHHiB33XUXuXTpkr2HZ1Ps3buXHDhwgBQWFpLCwkKydu1aIhQKSU5OjsXPU5IpLCwkNTU13E/7B9+JzqGpqYnExsaS6OhocvPNNxOBQEDGjBlD1q9fT/Ly8kh+fj4ZMGAAWbJkicNPjCzLktmzZ5ObbrrJZucwvs/++OMP8tNPP5G8vDyiUqkIIYSsXr2aMAxDtm3bRgghJCcnh0ybNo0UFhaajNOJ7sFJMt1EfHw8Wbx4sclrJSUl5KeffiIlJSWEkCs3pl6vd/iH/Xrh6+tLPv/8c4vvUZJpamrq2UHdgFCpVGTMmDFkxowZRKVSEZZlSWVlJfn444/JxIkTCZ/PJ2KxmNx///294p5bunQpCQsLs1mFm/E1mD59Ohk1ahQJCgoikyZNInPnziUymYwolUry2muvEYZhyPz584mHhwd57rnnbDKefyKcJNNNfP7558Td3Z188MEHXao6udFWRDqdjvzwww9EJBKR3Nxci5+hJBMeHk6Cg4PJbbfdRo4dO9bDI70xwLIs2bZtG1EqlRbfq62tJU899ZRZ2McRsXz5ctK/f39SXFxs83MtXryYxMXFkZqaGkIIIbfddhuJiIjgFoSEELJz506ydOlSk8XSjfa82gNOkrkOvP/++yQyMpJMmzaNHD582Oz9vLw88vLLL5Ply5d3mNTsDatNS8jKyiLu7u6Ez+cTb29vcuDAgQ4/W1BQQLZu3UrS0tLImTNnyJIlSwjDMD2a6HXCccCyLFm2bBnp168fuXjxos3PV1dXR2655RZy9OhRQgghr776KgkMDCR//vknIYSQyspKIpfLCSGmOZne+mw6Gpwkc504deoUue+++0h4eLhJ8vvjjz8m4eHh5I477iD33Xcf6du3L3nkkUc6vHH1en2vylGo1Wpy6dIlcv78ebJmzRri7+/f4U7GEmbNmkVmz55twxE64ahYsmQJ8fb2JidOnDDJ0bW1tXX7mMY7jqysLHL27Fnyxx9/EEIMIcbY2Fhy6dIl8uabbxJ/f3/y+++/E0IM+a233nqLHDt2zLlrsRGcJNNF1NTUkPLycrPXx48fT+644w7Csiw5efIkCQsLIxs3buTev3jxIhkxYgT5+uuvCSGE1NbWks8//9ymFTU9iUmTJpEnnnii059/4403SGRkpA1H5ISjwlKlIQDy1VdfXfexP/jgAxIVFUU8PDyIv78/Wb58OSHE8HwOGTKEBAcHm3Ttnz9/niQkJFx1J+7E9cEp9d9F7Nq1Cz///DM2btyIoUOHAjCo7iYkJODw4cNgGAZHjhxBU1MTnn/+eWzfvh3z58/H888/j/79+6O8vByAwSPmnXfeQXh4OJKSkpCamopnn30WU6ZMMWv2Mnbdc1QQQqBWqzv9+fT0dKe3zT8UxEb93++88w5effVVfPnll/Dw8MCpU6fw0UcfITk5GZs2bcKsWbMwYsQIJCcnQyqVQiKR4N5778XkyZM79Gdy4vrhJJkuIjExEQcPHsS0adNwyy23YPbs2di/fz8OHz7M+Uvk5uZiwoQJ2LZtG7Zs2YKUlBS8+uqrUCqVSExMBAAUFhaioqIC/v7+iIyMRGNjIx5++GH8+OOPuPnmm03OSQlGr9eDYRi7E87atWu5bmiZTIYff/wRJ06cwOHDhwEAL7zwAqqqqrBt2zYAwIYNGxAeHo6oqChoNBps374dKSkpSElJsefXcOIGwrZt27BmzRqcO3cOycnJAIC4uDgcOHAAx48fx0MPPYRPPvkEixYtQnx8PHQ6HTw8PDBmzBhs3rwZQO9YzPVGOEmmixg9ejT279+PY8eO4bPPPsMrr7yC8PBw/Oc//8Edd9wBwGB5GxQUBF9fX6xZswZr1qzB5cuXcfbsWURERKCtrQ2nTp3CwIEDcfr0aQDA3XffjYKCAnz++ecYO3YsBAIBKioqsHfvXgQFBWHOnDkmPiTUZGn79u148803ceLECQQFBfXINairq8ODDz6ImpoaeHt7IyYmBocPH8aUKVMAADU1NdyODTDIeDz77LOoqqqCq6sroqKicODAAefq0QmroKmpCT/99BMiIiIgkUi410NCQhAaGsr9e9asWSgqKsKePXsgEonQt29f3HbbbQCuPE9O2AB2Dtf1OlhK3Le0tJj8+9tvvyWDBg3iqlna4/Tp0yQmJoa89dZbhJArkuPr16/n8hRqtZocPHiQPPbYYyQ6Opp4e3uT5cuXc8lRWiQwc+ZMMmXKFNLc3GzxXCzLOhOaTtzwOH/+PLnvvvvI+PHjuabKr7/+mvD5fE5xuaOybmcVmW3hJJluon01mPFErtPpyLJly0hwcDCZN28e+fjjj8lbb71FLl++TAgxyLKIRCJOVoV2Hs+ZM4fMnz+fO47xQ3Ho0CESHR1NfvzxR5NxiEQi8tlnnzmJxIl/PNLT08nChQvJrbfeShYvXky8vb05XbLeVLl5o8EZgOwmeDyeyfbaOFHP5/OxceNG7N+/H/379+ccEQMCAtDc3MypvtbX1wMAxGIxZDIZjhw5gmnTpgEAmpubsXfvXnzyySe4cOECpk2bhpiYGPz888/ceX7++WcIhUKMHj3aoirs7t27sXbtWmg0GrP39Hq9dS6Eg2P9+vVgGAbPPPPMVT938uRJJCYmwsXFBYMGDeLi9E70HsTFxeGFF15ASEgIUlJScMcdd2D+/PkA4FRNtifszXI3IiztKmiz1++//05uueUWMmrUKLJs2TLS0tJCzp49S+6++27Sr18/otfrSWNjIxkzZgyJjo4m06ZNI/7+/mTIkCHE29ubvPDCC1y39+zZs8m0adPMQmX0/B9++CHh8XgdjulGx19//UXCw8NJTEwMWbFiRYefo26iK1asIHl5eeSzzz5zuon2YhQUFJCHHnqIjBs3jmzZsoV7/Z/4DDgCnDsZG4CumliW5XYM1CcjOzsbdXV1WL9+PeRyOfr06YO77roLDQ0N2LRpE3g8Hr755hvU1tZi48aN2LVrF0pKSrB8+XLw+XxERERwvvBHjhzB7Nmz4enpaXJ+lmUBAOXl5Rg9ejRkMhk3pp07d8LFxQX/+9//euRa2AtyuRz3338/PvvsM/j6+l71s5s3b8aAAQOwYcMGDB8+HI899hgeffRRvPfeez00WiesiWHDhmHt2rWIjIzEtm3b8M477wBw7mbsBSfJ2BDtQ2qNjY04f/48+vfvj9tuuw1ff/01CgsLsWnTJuzduxdz5swBYKjG4vP56NevH1xdXeHh4YGGhgZ4e3sjMjISAHDgwAEwDINx48aZlV3SczY0NCAwMBBKpRIA8Pvvv2P//v3QaDSoqKgAcKVnoaysjPv9S5cuQaVS2eiq9AyWLVuGmTNnYvLkydf87NmzZzF16lST126//XakpqZCq9Xaaoi9EqdOncLs2bPRr18/MAyDPXv22HtIFkGJJigoiLvXnbAPnCTTg2hoaMDly5e5XhmNRoNBgwZh9uzZcHNz4z53//33w93dHffccw/ee+89LFiwAO+++y5uvfVWDBo0CADwxRdfYNy4cQgPD7d4LpVKBVdXVzQ2NiIwMBAA8OWXXyIoKAghISGIiYkBAM4JcMyYMXjmmWdACOHO2Vvx448/4sKFC1i/fn2nPl9bW2tW/h0UFASdTsflzZwwQKFQIDY2Fhs3brT3UK6JwYMHY+PGjfj4448B2K4J1Imrw9kn04MYOnQoTp48CZlMBgAQCoUWrVxDQkLw22+/YevWrSguLsbUqVOxZ88exMXFcYRx7NgxvPrqq/Dy8jI7DyEELi4uqKmpgZ+fHwDg+++/R3Z2Nl599VWUlpZCLpdzn/3oo4/g7u6Of//732AYBlu2bIFCoeDe701hhoqKCqxYsQK//vorF1bsDNp/Rzoh9abv3hOYPn06pk+fbu9hdBpUVcLZaGk/OEnGDqA5lI4mMEII/P39sXbtWgCGHU9AQAAGDx4MANi/fz9aW1tx0003XfXBuXjxIu666y5UV1fjk08+wfz58zFhwgRs374dra2tAIC9e/di//79eOaZZxAaGoq0tDSIxWJER0ebjFGv14MQAoFAgMuXL+P8+fOYP38+RCKRQ03EaWlpkEgk3G4RMIz91KlT2LhxI9RqtVnTXXBwMGpra01ek0gkEAgE6NOnT4+M2wnbwkkw9oOTZBwQDMOAEAKWZcHn8yESiTBv3jzufS8vL6xatQoDBw7s8Pebm5vh4eEBlmWxZcsW+Pr6YsmSJfD09MSpU6fw73//G1qtFi+99BLGjBmDhx9+GACwcuVKaLVanD171uSYxhPzsWPH8OSTT0IikcDf39/q3/96MGnSJGRnZ5u89sgjjyAyMhKrV6+22NU9duxY7Nu3z+S1X3/9FUlJSRAKhTYdrxNO3OhwkoyDgmEYkwnROGw1YcIETJgw4aq/L5VKMWDAAOzduxdDhgzBgw8+CD8/P5SUlECn02HQoEF45513UFZWhp07d6JPnz4oKytDQUEBPvjgA+44J0+exO7du9HU1IQVK1YgIiICp0+fxk033QR/f3/IZDIUFBQgOjoarq6utrkYXYCnpye3C6Nwd3dHnz59uNfba6s99dRT2LhxI1atWoXHH38cZ8+exRdffIEffvihx8fvhBM3Gpx7yF4C45BUZxopg4KCcOHCBWRnZ2P8+PGYO3cuACArKwszZszA9u3bsWvXLjz66KOIiIgAy7L4888/0dzczH32hx9+wLRp03D58mXI5XIsXboUO3bsQGpqKqfTVlNTgyVLluDFF18EAE6JubS0FO+8806XlJl7Cu211QYOHIiDBw/ixIkTiIuLw+uvv46PPvoId955px1H6YQTNwjs057jhK2hVCrJ3XffTaKiokyaNQ8dOkTuvvtu4ufnR+666y5O2qatrY3cc889ZMKECYQQgxZUXFwcefrpp7nf3bZtGxkwYABxdXUlUqmUnD59mtx2221kwIABnAkUxYoVK0ifPn1IZWVlD3xbJ+wBAGT37t32HoYTDg7nTuYGhYuLC3766Sfk5OTA29ubq5by9fXFjh070NTUxDWsAYbw2tGjR/HAAw8AMDRt+vj4YNGiRdwxx48fD6VSiTFjxsDf3x8BAQGoq6tDXV0dJk2ahMmTJ6OgoACAoTjh0Ucf5UqDdTpdT359J2wEuVyOjIwMZGRkAABKSkqQkZFhsjN0wgljOHMyNyhYlgXLshAIDH9iGm4LDQ3F/PnzERYWhvj4eC7Xk5qaira2Nq4/pqCgAOHh4RgyZAh3zPr6eohEIsyaNQuAoRcnNDQUU6ZMwfLly/Hll19Co9GgrKwMxcXFmDZtGnd++l8nejdSU1MxceJE7t+rVq0CACxatAhff/21nUblhCPDuZO5QcHj8SxO7P369UNKSgref/99AAby0el0+OabbxAeHg4fHx9otVqEhYWhoqIC3t7e3O+mp6ejpaUFd911FwAgMzMTpaWlmDlzJgYPHow333wTMTEx2LlzJwBD0UBycjImTpzI+eY4cQWdEe88ceIEGIYx+6E7xp7GrbfeCmJQbzf5cRKMEx3BSTL/MOj1erPCAYFAgJdffhmff/45AEOT6LBhw3Dp0iXk5+cDAI4fP45PP/0Uw4cPx4ABA9DW1oasrCy4ublxK1saEvvqq6/g5+cHf39/bN68GT4+PnjllVdQU1PTg9/UsXH+/Hls3bqVU164FgoLC1FTU8P9UOtvJ5xwdDhjGP8wdOT+l5CQYPLvu+66CwcOHEBSUhJmzpwJiUSC7OxsTqKjpKSEq1zj8/nQarUQCoUoLy9HXl4evvvuOyxcuBAA8PbbbyMyMhKlpaVcB/Y/GcbinW+88UanficwMBA+Pj62HZgTTtgAzp2MEwDMdZ0CAwNx4MABnDp1CrfffjuWLVsGAJzfTWtrKzIzMzkBSiokuW3bNgwcOBDjx483ObZQKHSIPhpHQFfEOyni4+PRt29fTJo0CcePH7fh6Jxwwrpw7mScAGAucUO1nhITEzmJlqSkJE5lYOjQoXBxcUFxcTE0Gg0n8Pn999/j9ttv5zTWAAPxjBgxwqLO2j8NVLyTGtddC3379sXWrVuRmJgItVqNb7/9FpMmTcKJEyeu2ZDrhBOOAIa0X8I64YQRSDuhSFqNptfrsW3bNvznP/8BwzA4cuQIfHx8EBISgv3792PGjBncMUaMGIHbb78db7zxBuer809ERUUFkpKS8OuvvyI2NhaAIZEeFxeHDRs2dPo4s2fPBsMw2Lt3r41G6oQT1oMzXObEVUGrmYz/DRhyO4888giqqqpw7tw5DB06FCdOnICPjw/XewMYvGlqa2sxYcKEfzTBAKbinQKBAAKBACdPnsRHH30EgUDQaUvsMWPG4NKlSzYerRNOWAfOcJkT3YZerwePx0NISAgAYOHChZg+fbqJU+fmzZvh6emJESNG2GuYDoPuiHdaQnp6urOAwoleAyfJONFttJ8UWZY1q4AaNWoU+vfvj379+vXgyBwT3RHv3LBhA8LDwxEVFQWNRoPt27cjJSUFKSkpPT5+J5zoDpwk44TVYMmz45577rHDSHov2ot3ajQaPPvss6iqqoKrqyuioqJw4MABk5yXE044MpyJfydsCr1e3+kwkBNOOHHjwUkyTjjhhBNO2AzO6jInnHDCCSdsBifJOOGEE044YTM4ScYJJ5xwwgmbwUkyTjjhhBNO2AxOknHCCSeccMJmcJKME0444YQTNoOTZJxwwgknnLAZnCTjhBNOOOGEzeAkGSeccMIJJ2wGJ8k44YQTTjhhMzhJxgknnHDCCZvh/wGhLbIoECsIZQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "iris = DataSet(name=\"iris\")\n", + "\n", + "show_iris()\n", + "show_iris(0, 1, 3)\n", + "show_iris(1, 2, 3)" + ] + }, + { + "cell_type": "markdown", + "id": "1b073794-0308-42cc-b348-650d6d38bcd5", + "metadata": {}, + "source": [ + "## 距离函数\n", + "\n", + "在很多算法中(比如*k-Nearest Neighbors*算法),都需要对项目进行比较,找出它们的*相似度或*接近度。为此,我们有许多不同的函数供我们使用。以下是该模块中实现的函数。\n", + "\n", + "### 曼哈顿距离 (`manhattan_distance`)\n", + "\n", + "最简单的距离函数之一。它计算两个项目的坐标/特征之间的差异。为了理解它的工作原理,想象一个2D网格,坐标为*x*和*y*。在这个网格中,我们有两个项目,分别位于`(1,2)`和`(3,4)`的方格中。他们两个坐标之间的差是`3-1=2`和`4-2=2`。如果我们把这些加起来,就得到`4`。这意味着要从`(1,2)`到`(3,4)`我们需要四次移动;两次向右,两次向上。该函数对n维网格的工作原理与此类似。" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "d697d325-01cd-4ebb-94a4-66b9d420efdc", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Manhattan Distance between (1,2) and (3,4) is 4\n" + ] + } + ], + "source": [ + "def manhattan_distance(X, Y):\n", + " return sum([abs(x - y) for x, y in zip(X, Y)])\n", + "\n", + "\n", + "distance = manhattan_distance([1,2], [3,4])\n", + "print(\"Manhattan Distance between (1,2) and (3,4) is\", distance)" + ] + }, + { + "cell_type": "markdown", + "id": "7f8393e1-37dc-49e5-a218-5fdfcce7476b", + "metadata": {}, + "source": [ + "### 欧几里得距离 (`euclidean_distance`)\n", + "\n", + "可能是最流行的距离函数。它返回两个项目中各个元素之间的平方差值的平方根。" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "9d0f44a7-0f02-498f-b1ce-afdec5208797", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Euclidean Distance between (1,2) and (3,4) is 2.8284271247461903\n" + ] + } + ], + "source": [ + "def euclidean_distance(X, Y):\n", + " return math.sqrt(sum([(x - y)**2 for x, y in zip(X,Y)]))\n", + "\n", + "\n", + "distance = euclidean_distance([1,2], [3,4])\n", + "print(\"Euclidean Distance between (1,2) and (3,4) is\", distance)" + ] + }, + { + "cell_type": "markdown", + "id": "1f40b7c9-ae57-4203-9068-ca94456d4da9", + "metadata": {}, + "source": [ + "### Hamming Distance (`hamming_distance`)\n", + "\n", + "这个函数计算两个项目中单个元素之间的差异数。例如,如果我们有两个二进制字符串 \"111 \"和 \"011\",该函数将返回1,因为这两个字符串只有第一个元素不同。该函数对非二进制字符串也有同样的作用。" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "631e612b-8c62-4b72-9e50-eb63802a7956", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hamming Distance between 'abc' and 'abb' is 1\n" + ] + } + ], + "source": [ + "def hamming_distance(X, Y):\n", + " return sum(x != y for x, y in zip(X, Y))\n", + "\n", + "\n", + "distance = hamming_distance(['a','b','c'], ['a','b','b'])\n", + "print(\"Hamming Distance between 'abc' and 'abb' is\", distance)" + ] + }, + { + "cell_type": "markdown", + "id": "dde6b656-6f02-4f77-bdf6-16c246913ac7", + "metadata": {}, + "source": [ + "### 平均布尔误差(`mean_boolean_error`)。\n", + "\n", + "为了计算这个距离,我们找到两个项目的所有元素中不同元素的比率。例如,如果两个项目是`(1,2,3)`和`(1,4,5)`,不同/所有元素的比率是2/3,因为它们在三个元素中的两个不同。" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "7cec4dd5-bca9-4cc6-b6d0-fac128193b2c", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mean Boolean Error Distance between (1,2,3) and (1,4,5) is 0.6666666666666666\n" + ] + } + ], + "source": [ + "def mean_boolean_error(X, Y):\n", + " return mean(int(x != y) for x, y in zip(X, Y))\n", + "\n", + "\n", + "distance = mean_boolean_error([1,2,3], [1,4,5])\n", + "print(\"Mean Boolean Error Distance between (1,2,3) and (1,4,5) is\", distance)" + ] + }, + { + "cell_type": "markdown", + "id": "caed8d04-3682-4125-bd33-25e41d9778bb", + "metadata": {}, + "source": [ + "### 平均误差 (`mean_error`)\n", + "\n", + "这个函数找出两个项目之间单个元素的平均差。例如,如果两个项目是`(1,0,5)`和`(3,10,5)`,它们的误差距离是`(3-1) + (10-0) + (5-5) = 2 + 10 + 0 = 12`。因此,平均误差距离是`12/3=4'。" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "d1b62250-6e06-4f11-9382-57f16aab8db4", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mean Error Distance between (1,0,5) and (3,10,5) is 4\n" + ] + } + ], + "source": [ + "def mean_error(X, Y):\n", + " return mean([abs(x - y) for x, y in zip(X, Y)])\n", + "\n", + "\n", + "distance = mean_error([1,0,5], [3,10,5])\n", + "print(\"Mean Error Distance between (1,0,5) and (3,10,5) is\", distance)" + ] + }, + { + "cell_type": "markdown", + "id": "e0a0b54f-8eb6-4592-a6fd-57c74f7768f6", + "metadata": {}, + "source": [ + "### 平均平方误差 (`ms_error`)\n", + "\n", + "这与 \"平均误差\"非常相似,但我们不是计算元素之间的差异,而是计算差异的*平方*。" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "b7f4f479-1904-4113-9cb6-c3f82e1af201", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mean Square Distance between (1,0,5) and (3,10,5) is 34.666666666666664\n" + ] + } + ], + "source": [ + "def ms_error(X, Y):\n", + " return mean([(x - y)**2 for x, y in zip(X, Y)])\n", + "\n", + "\n", + "distance = ms_error([1,0,5], [3,10,5])\n", + "print(\"Mean Square Distance between (1,0,5) and (3,10,5) is\", distance)" + ] + }, + { + "cell_type": "markdown", + "id": "ad6cd218-d3a5-4335-b9d3-63c0f3828dc1", + "metadata": {}, + "source": [ + "### 平均平方误差的根(`rms_error`)。\n", + "\n", + "这是 \"均方误差\"的平方根。" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "b3b6c315-a28c-4db7-bd09-75b72176f408", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Root of Mean Error Distance between (1,0,5) and (3,10,5) is 5.887840577551898\n" + ] + } + ], + "source": [ + "def rms_error(X, Y):\n", + " return math.sqrt(ms_error(X, Y))\n", + "\n", + "\n", + "distance = rms_error([1,0,5], [3,10,5])\n", + "print(\"Root of Mean Error Distance between (1,0,5) and (3,10,5) is\", distance)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/data/house_tiny.csv b/data/house_tiny.csv new file mode 100644 index 0000000..ade22ea --- /dev/null +++ b/data/house_tiny.csv @@ -0,0 +1,5 @@ +NumRooms,Alley,Price +NA,Pave,127500 +2,NA,106000 +4,NA,178100 +NA,NA,140000 diff --git a/data/iris.csv b/data/iris.csv new file mode 100644 index 0000000..1a75486 --- /dev/null +++ b/data/iris.csv @@ -0,0 +1,150 @@ +5.1,3.5,1.4,0.2,setosa +4.9,3.0,1.4,0.2,setosa +4.7,3.2,1.3,0.2,setosa +4.6,3.1,1.5,0.2,setosa +5.0,3.6,1.4,0.2,setosa +5.4,3.9,1.7,0.4,setosa +4.6,3.4,1.4,0.3,setosa +5.0,3.4,1.5,0.2,setosa +4.4,2.9,1.4,0.2,setosa +4.9,3.1,1.5,0.1,setosa +5.4,3.7,1.5,0.2,setosa +4.8,3.4,1.6,0.2,setosa +4.8,3.0,1.4,0.1,setosa +4.3,3.0,1.1,0.1,setosa +5.8,4.0,1.2,0.2,setosa +5.7,4.4,1.5,0.4,setosa +5.4,3.9,1.3,0.4,setosa +5.1,3.5,1.4,0.3,setosa +5.7,3.8,1.7,0.3,setosa +5.1,3.8,1.5,0.3,setosa +5.4,3.4,1.7,0.2,setosa +5.1,3.7,1.5,0.4,setosa +4.6,3.6,1.0,0.2,setosa +5.1,3.3,1.7,0.5,setosa +4.8,3.4,1.9,0.2,setosa +5.0,3.0,1.6,0.2,setosa +5.0,3.4,1.6,0.4,setosa +5.2,3.5,1.5,0.2,setosa +5.2,3.4,1.4,0.2,setosa +4.7,3.2,1.6,0.2,setosa +4.8,3.1,1.6,0.2,setosa +5.4,3.4,1.5,0.4,setosa +5.2,4.1,1.5,0.1,setosa +5.5,4.2,1.4,0.2,setosa +4.9,3.1,1.5,0.1,setosa +5.0,3.2,1.2,0.2,setosa +5.5,3.5,1.3,0.2,setosa +4.9,3.1,1.5,0.1,setosa +4.4,3.0,1.3,0.2,setosa +5.1,3.4,1.5,0.2,setosa +5.0,3.5,1.3,0.3,setosa +4.5,2.3,1.3,0.3,setosa +4.4,3.2,1.3,0.2,setosa +5.0,3.5,1.6,0.6,setosa +5.1,3.8,1.9,0.4,setosa +4.8,3.0,1.4,0.3,setosa +5.1,3.8,1.6,0.2,setosa +4.6,3.2,1.4,0.2,setosa +5.3,3.7,1.5,0.2,setosa +5.0,3.3,1.4,0.2,setosa +7.0,3.2,4.7,1.4,versicolor +6.4,3.2,4.5,1.5,versicolor +6.9,3.1,4.9,1.5,versicolor +5.5,2.3,4.0,1.3,versicolor +6.5,2.8,4.6,1.5,versicolor +5.7,2.8,4.5,1.3,versicolor +6.3,3.3,4.7,1.6,versicolor +4.9,2.4,3.3,1.0,versicolor +6.6,2.9,4.6,1.3,versicolor +5.2,2.7,3.9,1.4,versicolor +5.0,2.0,3.5,1.0,versicolor +5.9,3.0,4.2,1.5,versicolor +6.0,2.2,4.0,1.0,versicolor +6.1,2.9,4.7,1.4,versicolor +5.6,2.9,3.6,1.3,versicolor +6.7,3.1,4.4,1.4,versicolor +5.6,3.0,4.5,1.5,versicolor +5.8,2.7,4.1,1.0,versicolor +6.2,2.2,4.5,1.5,versicolor +5.6,2.5,3.9,1.1,versicolor +5.9,3.2,4.8,1.8,versicolor +6.1,2.8,4.0,1.3,versicolor +6.3,2.5,4.9,1.5,versicolor +6.1,2.8,4.7,1.2,versicolor +6.4,2.9,4.3,1.3,versicolor +6.6,3.0,4.4,1.4,versicolor +6.8,2.8,4.8,1.4,versicolor +6.7,3.0,5.0,1.7,versicolor +6.0,2.9,4.5,1.5,versicolor +5.7,2.6,3.5,1.0,versicolor +5.5,2.4,3.8,1.1,versicolor +5.5,2.4,3.7,1.0,versicolor +5.8,2.7,3.9,1.2,versicolor +6.0,2.7,5.1,1.6,versicolor +5.4,3.0,4.5,1.5,versicolor +6.0,3.4,4.5,1.6,versicolor +6.7,3.1,4.7,1.5,versicolor +6.3,2.3,4.4,1.3,versicolor +5.6,3.0,4.1,1.3,versicolor +5.5,2.5,4.0,1.3,versicolor +5.5,2.6,4.4,1.2,versicolor +6.1,3.0,4.6,1.4,versicolor +5.8,2.6,4.0,1.2,versicolor +5.0,2.3,3.3,1.0,versicolor +5.6,2.7,4.2,1.3,versicolor +5.7,3.0,4.2,1.2,versicolor +5.7,2.9,4.2,1.3,versicolor +6.2,2.9,4.3,1.3,versicolor +5.1,2.5,3.0,1.1,versicolor +5.7,2.8,4.1,1.3,versicolor +6.3,3.3,6.0,2.5,virginica +5.8,2.7,5.1,1.9,virginica +7.1,3.0,5.9,2.1,virginica +6.3,2.9,5.6,1.8,virginica +6.5,3.0,5.8,2.2,virginica +7.6,3.0,6.6,2.1,virginica +4.9,2.5,4.5,1.7,virginica +7.3,2.9,6.3,1.8,virginica +6.7,2.5,5.8,1.8,virginica +7.2,3.6,6.1,2.5,virginica +6.5,3.2,5.1,2.0,virginica +6.4,2.7,5.3,1.9,virginica +6.8,3.0,5.5,2.1,virginica +5.7,2.5,5.0,2.0,virginica +5.8,2.8,5.1,2.4,virginica +6.4,3.2,5.3,2.3,virginica +6.5,3.0,5.5,1.8,virginica +7.7,3.8,6.7,2.2,virginica +7.7,2.6,6.9,2.3,virginica +6.0,2.2,5.0,1.5,virginica +6.9,3.2,5.7,2.3,virginica +5.6,2.8,4.9,2.0,virginica +7.7,2.8,6.7,2.0,virginica +6.3,2.7,4.9,1.8,virginica +6.7,3.3,5.7,2.1,virginica +7.2,3.2,6.0,1.8,virginica +6.2,2.8,4.8,1.8,virginica +6.1,3.0,4.9,1.8,virginica +6.4,2.8,5.6,2.1,virginica +7.2,3.0,5.8,1.6,virginica +7.4,2.8,6.1,1.9,virginica +7.9,3.8,6.4,2.0,virginica +6.4,2.8,5.6,2.2,virginica +6.3,2.8,5.1,1.5,virginica +6.1,2.6,5.6,1.4,virginica +7.7,3.0,6.1,2.3,virginica +6.3,3.4,5.6,2.4,virginica +6.4,3.1,5.5,1.8,virginica +6.0,3.0,4.8,1.8,virginica +6.9,3.1,5.4,2.1,virginica +6.7,3.1,5.6,2.4,virginica +6.9,3.1,5.1,2.3,virginica +5.8,2.7,5.1,1.9,virginica +6.8,3.2,5.9,2.3,virginica +6.7,3.3,5.7,2.5,virginica +6.7,3.0,5.2,2.3,virginica +6.3,2.5,5.0,1.9,virginica +6.5,3.0,5.2,2.0,virginica +6.2,3.4,5.4,2.3,virginica +5.9,3.0,5.1,1.8,virginica diff --git a/data/iris.txt b/data/iris.txt new file mode 100644 index 0000000..5d88503 --- /dev/null +++ b/data/iris.txt @@ -0,0 +1,69 @@ +1. Title: Iris Plants Database + Updated Sept 21 by C.Blake - Added discrepency information + +2. Sources: + (a) Creator: R.A. Fisher + (b) Donor: Michael Marshall (MARSHALL%PLU@io.arc.nasa.gov) + (c) Date: July, 1988 + +3. Past Usage: + - Publications: too many to mention!!! Here are a few. + 1. Fisher,R.A. "The use of multiple measurements in taxonomic problems" + Annual Eugenics, 7, Part II, 179-188 (1936); also in "Contributions + to Mathematical Statistics" (John Wiley, NY, 1950). + 2. Duda,R.O., & Hart,P.E. (1973) Pattern Classification and Scene Analysis. + (Q327.D83) John Wiley & Sons. ISBN 0-471-22361-1. See page 218. + 3. Dasarathy, B.V. (1980) "Nosing Around the Neighborhood: A New System + Structure and Classification Rule for Recognition in Partially Exposed + Environments". IEEE Transactions on Pattern Analysis and Machine + Intelligence, Vol. PAMI-2, No. 1, 67-71. + -- Results: + -- very low misclassification rates (0% for the setosa class) + 4. Gates, G.W. (1972) "The Reduced Nearest Neighbor Rule". IEEE + Transactions on Information Theory, May 1972, 431-433. + -- Results: + -- very low misclassification rates again + 5. See also: 1988 MLC Proceedings, 54-64. Cheeseman et al's AUTOCLASS II + conceptual clustering system finds 3 classes in the data. + +4. Relevant Information: + --- This is perhaps the best known database to be found in the pattern + recognition literature. Fisher's paper is a classic in the field + and is referenced frequently to this day. (See Duda & Hart, for + example.) The data set contains 3 classes of 50 instances each, + where each class refers to a type of iris plant. One class is + linearly separable from the other 2; the latter are NOT linearly + separable from each other. + --- Predicted attribute: class of iris plant. + --- This is an exceedingly simple domain. + --- This data differs from the data presented in Fishers article + (identified by Steve Chadwick, spchadwick@espeedaz.net ) + The 35th sample should be: 4.9,3.1,1.5,0.2,"Iris-setosa" + where the error is in the fourth feature. + The 38th sample: 4.9,3.6,1.4,0.1,"Iris-setosa" + where the errors are in the second and third features. + +5. Number of Instances: 150 (50 in each of three classes) + +6. Number of Attributes: 4 numeric, predictive attributes and the class + +7. Attribute Information: + 1. sepal length in cm + 2. sepal width in cm + 3. petal length in cm + 4. petal width in cm + 5. class: + -- Iris Setosa + -- Iris Versicolour + -- Iris Virginica + +8. Missing Attribute Values: None + +Summary Statistics: + Min Max Mean SD Class Correlation + sepal length: 4.3 7.9 5.84 0.83 0.7826 + sepal width: 2.0 4.4 3.05 0.43 -0.4194 + petal length: 1.0 6.9 3.76 1.76 0.9490 (high!) + petal width: 0.1 2.5 1.20 0.76 0.9565 (high!) + +9. Class Distribution: 33.3% for each of 3 classes. diff --git a/data/orings.csv b/data/orings.csv new file mode 100644 index 0000000..200a27c --- /dev/null +++ b/data/orings.csv @@ -0,0 +1,23 @@ +6, 0, 66, 50, 1 +6, 1, 70, 50, 2 +6, 0, 69, 50, 3 +6, 0, 68, 50, 4 +6, 0, 67, 50, 5 +6, 0, 72, 50, 6 +6, 0, 73, 100, 7 +6, 0, 70, 100, 8 +6, 1, 57, 200, 9 +6, 1, 63, 200, 10 +6, 1, 70, 200, 11 +6, 0, 78, 200, 12 +6, 0, 67, 200, 13 +6, 2, 53, 200, 14 +6, 0, 67, 200, 15 +6, 0, 75, 200, 16 +6, 0, 70, 200, 17 +6, 0, 81, 200, 18 +6, 0, 76, 200, 19 +6, 0, 79, 200, 20 +6, 0, 75, 200, 21 +6, 0, 76, 200, 22 +6, 1, 58, 200, 23 diff --git a/data/orings.txt b/data/orings.txt new file mode 100644 index 0000000..8cc6395 --- /dev/null +++ b/data/orings.txt @@ -0,0 +1,78 @@ +Source: http://www1.ics.uci.edu/pub/machine-learning-databases/space-shuttle/ + +1. Title: Challenger Space Shuttle O-Ring Data (2 databases) + +2. Sources: + -- David Draper (draper@math.ucla.edu) + University of California, Los Angeles + -- Donor: David Draper (draper@math.ucla.edu) + -- Date: 5 August 1993 + +3. Past Usage: + + 1. Draper,~D. (1993). Assessment and propagation of model uncertainty. + In {\it Proceedings of the Fourth International Workshop on Artificial + Intelligence and Statistics} (pp. 497--509). Ft. Lauderdale, FL: + Unpublished. + -- Discrete model uncertainty analysis + -- Analysis suggests that obvious different extrapolations of the + data exist at 31 degrees Fahrenheit (i.e., freezing), which sharply + discredits the assumption of no temperature effect. + 2. Dalal,~S.~R., Fowlkes,~E.~B., \& Hoadley,~B. (1989). Risk analysis of + the space shuttle: pre-Challenger prediction of failure. {\it Journal + of the American Statisticians Association}, {\it 84}, 945--957. + 3. Lavine,~M. (1991). Problems in extrapolation illustrated with space + shuttle O-ring data. {\it Journal of the American Statisticians + Association}, {\it 86}, 919--922. + 4. Martz~H.~F., \& Zimmer,~W.~J. (1992). The risk of catastrophic failure + of the solid rocket boosters on the space shuttle. {\it American + Statistics}, {\it 46}, 42--47. + +4. Number of instances: 23 in each of two files + +5. Relevant Information: + + There are two databases: (both use the same set of 5 attributes) + 1. Primary o-ring erosion and/or blowby + 2. Primary o-ring erosion only + The two databases are identical except for the 2nd attribute of the + 21st instance (confirmed by David Draper on 8/5/93). + + Edited from (Draper, 1993): + The motivation for collecting this database was the explosion of the + USA Space Shuttle Challenger on 28 January, 1986. An investigation + ensued into the reliability of the shuttle's propulsion system. The + explosion was eventually traced to the failure of one of the three field + joints on one of the two solid booster rockets. Each of these six field + joints includes two O-rings, designated as primary and secondary, which + fail when phenomena called erosion and blowby both occur. + The night before the launch a decision had to be made regarding + launch safety. The discussion among engineers and managers leading to + this decision included concern that the probability of failure of the + O-rings depended on the temperature t at launch, which was forecase to + be 31 degrees F. There are strong engineering reasons based on the + composition of O-rings to support the judgment that failure + probability may rise monotonically as temperature drops. One other + variable, the pressure s at which safety testing for field join leaks + was performed, was available, but its relevance to the failure process + was unclear. + Draper's paper includes a menacing figure graphing the number of field + joints experiencing stress vs. liftoff temperature for the 23 shuttle + flights previous to the Challenger disaster. No previous liftoff + temperature was under 53 degrees F. Although tremendous extrapolation + must be done from the given data to assess risk at 31 degrees F, it + is obvious even to the layman "to foresee the unacceptably high risk + created by launching at 31 degrees F." For more information, see + Draper (1993) or the other previous analyses. + The task is to predict the number of O-rings that will experience + thermal distress for a given flight when the launch temperature is + below freezing. + +6. Number of Attributes: 5 + 1. Number of O-rings at risk on a given flight + 2. Number experiencing thermal distress + 3. Launch temperature (degrees F) + 4. Leak-check pressure (psi) + 5. Temporal order of flight + +7. Attribute Information: all values are positive integers diff --git a/data/restaurant.csv b/data/restaurant.csv new file mode 100644 index 0000000..f31f70c --- /dev/null +++ b/data/restaurant.csv @@ -0,0 +1,12 @@ +Yes, No, No, Yes, Some, $$$, No, Yes, French, 0-10, Yes +Yes, No, No, Yes, Full, $, No, No, Thai, 30-60, No +No, Yes, No, No, Some, $, No, No, Burger, 0-10, Yes +Yes, No, Yes, Yes, Full, $, No, No, Thai, 10-30, Yes +Yes, No, Yes, No, Full, $$$, No, Yes, French, >60, No +No, Yes, No, Yes, Some, $$, Yes, Yes, Italian, 0-10, Yes +No, Yes, No, No, None, $, Yes, No, Burger, 0-10, No +No, No, No, Yes, Some, $$, Yes, Yes, Thai, 0-10, Yes +No, Yes, Yes, No, Full, $, Yes, No, Burger, >60, No +Yes, Yes, Yes, Yes, Full, $$$, No, Yes, Italian, 10-30, No +No, No, No, No, None, $, No, No, Thai, 0-10, No +Yes, Yes, Yes, Yes, Full, $, No, No, Burger, 30-60, Yes diff --git a/data/zoo.csv b/data/zoo.csv new file mode 100644 index 0000000..214be04 --- /dev/null +++ b/data/zoo.csv @@ -0,0 +1,101 @@ +aardvark,1,0,0,1,0,0,1,1,1,1,0,0,4,0,0,1,mammal +antelope,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,mammal +bass,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,fish +bear,1,0,0,1,0,0,1,1,1,1,0,0,4,0,0,1,mammal +boar,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,mammal +buffalo,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,mammal +calf,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,1,mammal +carp,0,0,1,0,0,1,0,1,1,0,0,1,0,1,1,0,fish +catfish,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,fish +cavy,1,0,0,1,0,0,0,1,1,1,0,0,4,0,1,0,mammal +cheetah,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,mammal +chicken,0,1,1,0,1,0,0,0,1,1,0,0,2,1,1,0,bird +chub,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,fish +clam,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,shellfish +crab,0,0,1,0,0,1,1,0,0,0,0,0,4,0,0,0,shellfish +crayfish,0,0,1,0,0,1,1,0,0,0,0,0,6,0,0,0,shellfish +crow,0,1,1,0,1,0,1,0,1,1,0,0,2,1,0,0,bird +deer,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,mammal +dogfish,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,1,fish +dolphin,0,0,0,1,0,1,1,1,1,1,0,1,0,1,0,1,mammal +dove,0,1,1,0,1,0,0,0,1,1,0,0,2,1,1,0,bird +duck,0,1,1,0,1,1,0,0,1,1,0,0,2,1,0,0,bird +elephant,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,mammal +flamingo,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,1,bird +flea,0,0,1,0,0,0,0,0,0,1,0,0,6,0,0,0,insect +frog,0,0,1,0,0,1,1,1,1,1,0,0,4,0,0,0,amphibian +frog,0,0,1,0,0,1,1,1,1,1,1,0,4,0,0,0,amphibian +fruitbat,1,0,0,1,1,0,0,1,1,1,0,0,2,1,0,0,mammal +giraffe,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,mammal +girl,1,0,0,1,0,0,1,1,1,1,0,0,2,0,1,1,mammal +gnat,0,0,1,0,1,0,0,0,0,1,0,0,6,0,0,0,insect +goat,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,1,mammal +gorilla,1,0,0,1,0,0,0,1,1,1,0,0,2,0,0,1,mammal +gull,0,1,1,0,1,1,1,0,1,1,0,0,2,1,0,0,bird +haddock,0,0,1,0,0,1,0,1,1,0,0,1,0,1,0,0,fish +hamster,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,0,mammal +hare,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,0,mammal +hawk,0,1,1,0,1,0,1,0,1,1,0,0,2,1,0,0,bird +herring,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,fish +honeybee,1,0,1,0,1,0,0,0,0,1,1,0,6,0,1,0,insect +housefly,1,0,1,0,1,0,0,0,0,1,0,0,6,0,0,0,insect +kiwi,0,1,1,0,0,0,1,0,1,1,0,0,2,1,0,0,bird +ladybird,0,0,1,0,1,0,1,0,0,1,0,0,6,0,0,0,insect +lark,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,0,bird +leopard,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,mammal +lion,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,mammal +lobster,0,0,1,0,0,1,1,0,0,0,0,0,6,0,0,0,shellfish +lynx,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,mammal +mink,1,0,0,1,0,1,1,1,1,1,0,0,4,1,0,1,mammal +mole,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,0,mammal +mongoose,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,mammal +moth,1,0,1,0,1,0,0,0,0,1,0,0,6,0,0,0,insect +newt,0,0,1,0,0,1,1,1,1,1,0,0,4,1,0,0,amphibian +octopus,0,0,1,0,0,1,1,0,0,0,0,0,8,0,0,1,shellfish +opossum,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,0,mammal +oryx,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,1,mammal +ostrich,0,1,1,0,0,0,0,0,1,1,0,0,2,1,0,1,bird +parakeet,0,1,1,0,1,0,0,0,1,1,0,0,2,1,1,0,bird +penguin,0,1,1,0,0,1,1,0,1,1,0,0,2,1,0,1,bird +pheasant,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,0,bird +pike,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,1,fish +piranha,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,0,fish +pitviper,0,0,1,0,0,0,1,1,1,1,1,0,0,1,0,0,reptile +platypus,1,0,1,1,0,1,1,0,1,1,0,0,4,1,0,1,mammal +polecat,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,mammal +pony,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,1,mammal +porpoise,0,0,0,1,0,1,1,1,1,1,0,1,0,1,0,1,mammal +puma,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,mammal +pussycat,1,0,0,1,0,0,1,1,1,1,0,0,4,1,1,1,mammal +raccoon,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,mammal +reindeer,1,0,0,1,0,0,0,1,1,1,0,0,4,1,1,1,mammal +rhea,0,1,1,0,0,0,1,0,1,1,0,0,2,1,0,1,bird +scorpion,0,0,0,0,0,0,1,0,0,1,1,0,8,1,0,0,shellfish +seahorse,0,0,1,0,0,1,0,1,1,0,0,1,0,1,0,0,fish +seal,1,0,0,1,0,1,1,1,1,1,0,1,0,0,0,1,mammal +sealion,1,0,0,1,0,1,1,1,1,1,0,1,2,1,0,1,mammal +seasnake,0,0,0,0,0,1,1,1,1,0,1,0,0,1,0,0,reptile +seawasp,0,0,1,0,0,1,1,0,0,0,1,0,0,0,0,0,shellfish +skimmer,0,1,1,0,1,1,1,0,1,1,0,0,2,1,0,0,bird +skua,0,1,1,0,1,1,1,0,1,1,0,0,2,1,0,0,bird +slowworm,0,0,1,0,0,0,1,1,1,1,0,0,0,1,0,0,reptile +slug,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,shellfish +sole,0,0,1,0,0,1,0,1,1,0,0,1,0,1,0,0,fish +sparrow,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,0,bird +squirrel,1,0,0,1,0,0,0,1,1,1,0,0,2,1,0,0,mammal +starfish,0,0,1,0,0,1,1,0,0,0,0,0,5,0,0,0,shellfish +stingray,0,0,1,0,0,1,1,1,1,0,1,1,0,1,0,1,fish +swan,0,1,1,0,1,1,0,0,1,1,0,0,2,1,0,1,bird +termite,0,0,1,0,0,0,0,0,0,1,0,0,6,0,0,0,insect +toad,0,0,1,0,0,1,0,1,1,1,0,0,4,0,0,0,amphibian +tortoise,0,0,1,0,0,0,0,0,1,1,0,0,4,1,0,1,reptile +tuatara,0,0,1,0,0,0,1,1,1,1,0,0,4,1,0,0,reptile +tuna,0,0,1,0,0,1,1,1,1,0,0,1,0,1,0,1,fish +vampire,1,0,0,1,1,0,0,1,1,1,0,0,2,1,0,0,mammal +vole,1,0,0,1,0,0,0,1,1,1,0,0,4,1,0,0,mammal +vulture,0,1,1,0,1,0,1,0,1,1,0,0,2,1,0,1,bird +wallaby,1,0,0,1,0,0,0,1,1,1,0,0,2,1,0,1,mammal +wasp,1,0,1,0,1,0,0,0,0,1,1,0,6,0,0,0,insect +wolf,1,0,0,1,0,0,1,1,1,1,0,0,4,1,0,1,mammal +worm,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,shellfish +wren,0,1,1,0,1,0,0,0,1,1,0,0,2,1,0,0,bird diff --git a/data/zoo.txt b/data/zoo.txt new file mode 100644 index 0000000..257fde6 --- /dev/null +++ b/data/zoo.txt @@ -0,0 +1,68 @@ +Source: http://www1.ics.uci.edu/pub/machine-learning-databases/zoo/', +1. Title: Zoo database + +2. Source Information + -- Creator: Richard Forsyth + -- Donor: Richard S. Forsyth + 8 Grosvenor Avenue + Mapperley Park + Nottingham NG3 5DX + 0602-621676 + -- Date: 5/15/1990 + +3. Past Usage: + -- None known other than what is shown in Forsyth's PC/BEAGLE User's Guide. + +4. Relevant Information: + -- A simple database containing 17 Boolean-valued attributes. The "type" + attribute appears to be the class attribute. Here is a breakdown of + which animals are in which type: (I find it unusual that there are + 2 instances of "frog" and one of "girl"!) + + Class# Set of animals: + ====== =============================================================== + 1 (41) aardvark, antelope, bear, boar, buffalo, calf, + cavy, cheetah, deer, dolphin, elephant, + fruitbat, giraffe, girl, goat, gorilla, hamster, + hare, leopard, lion, lynx, mink, mole, mongoose, + opossum, oryx, platypus, polecat, pony, + porpoise, puma, pussycat, raccoon, reindeer, + seal, sealion, squirrel, vampire, vole, wallaby,wolf + 2 (20) chicken, crow, dove, duck, flamingo, gull, hawk, + kiwi, lark, ostrich, parakeet, penguin, pheasant, + rhea, skimmer, skua, sparrow, swan, vulture, wren + 3 (5) pitviper, seasnake, slowworm, tortoise, tuatara + 4 (13) bass, carp, catfish, chub, dogfish, haddock, + herring, pike, piranha, seahorse, sole, stingray, tuna + 5 (4) frog, frog, newt, toad + 6 (8) flea, gnat, honeybee, housefly, ladybird, moth, termite, wasp + 7 (10) clam, crab, crayfish, lobster, octopus, + scorpion, seawasp, slug, starfish, worm + +5. Number of Instances: 101 + +6. Number of Attributes: 18 (animal name, 15 Boolean attributes, 2 numerics) + +7. Attribute Information: (name of attribute and type of value domain) + 1. animal name: Unique for each instance + 2. hair Boolean + 3. feathers Boolean + 4. eggs Boolean + 5. milk Boolean + 6. airborne Boolean + 7. aquatic Boolean + 8. predator Boolean + 9. toothed Boolean + 10. backbone Boolean + 11. breathes Boolean + 12. venomous Boolean + 13. fins Boolean + 14. legs Numeric (set of values: {0,2,4,5,6,8}) + 15. tail Boolean + 16. domestic Boolean + 17. catsize Boolean + 18. type Numeric (integer values in range [1,7]) + +8. Missing Attribute Values: None + +9. Class Distribution: Given above diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/__pycache__/__init__.cpython-311.pyc b/utils/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..a00f864 Binary files /dev/null and b/utils/__pycache__/__init__.cpython-311.pyc differ diff --git a/utils/__pycache__/__init__.cpython-38.pyc b/utils/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..e2c0c1c Binary files /dev/null and b/utils/__pycache__/__init__.cpython-38.pyc differ diff --git a/utils/__pycache__/dataset4learners.cpython-38.pyc b/utils/__pycache__/dataset4learners.cpython-38.pyc new file mode 100644 index 0000000..ae6b455 Binary files /dev/null and b/utils/__pycache__/dataset4learners.cpython-38.pyc differ diff --git a/utils/__pycache__/utils.cpython-311.pyc b/utils/__pycache__/utils.cpython-311.pyc new file mode 100644 index 0000000..6529e16 Binary files /dev/null and b/utils/__pycache__/utils.cpython-311.pyc differ diff --git a/utils/__pycache__/utils.cpython-38.pyc b/utils/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000..8d973e2 Binary files /dev/null and b/utils/__pycache__/utils.cpython-38.pyc differ diff --git a/utils/dataset4learners.py b/utils/dataset4learners.py new file mode 100644 index 0000000..c17d97e --- /dev/null +++ b/utils/dataset4learners.py @@ -0,0 +1,183 @@ +import sys +sys.path.insert(1, '../') +from utils.utils import * +from collections import defaultdict + + +### the ABC of data set +class DataSet: + """ + A data set for a machine learning problem. It has the following fields: + + d.examples A list of examples. Each one is a list of attribute values. + d.attrs A list of integers to index into an example, so example[attr] + gives a value. Normally the same as range(len(d.examples[0])). + d.attr_names Optional list of mnemonic names for corresponding attrs. + d.target The attribute that a learning algorithm will try to predict. + By default the final attribute. + d.inputs The list of attrs without the target. + d.values A list of lists: each sublist is the set of possible + values for the corresponding attribute. If initially None, + it is computed from the known examples by self.set_problem. + If not None, an erroneous value raises ValueError. + d.distance A function from a pair of examples to a non-negative number. + Should be symmetric, etc. Defaults to mean_boolean_error + since that can handle any field types. + d.name Name of the data set (for output display only). + d.source URL or other source where the data came from. + d.exclude A list of attribute indexes to exclude from d.inputs. Elements + of this list can either be integers (attrs) or attr_names. + + Normally, you call the constructor and you're done; then you just + access fields like d.examples and d.target and d.inputs. + """ + + def __init__(self, examples=None, attrs=None, attr_names=None, target=-1, inputs=None, + values=None, distance=mean_boolean_error, name='', source='', exclude=()): + """ + Accepts any of DataSet's fields. Examples can also be a + string or file from which to parse examples using parse_csv. + Optional parameter: exclude, as documented in .set_problem(). + >>> DataSet(examples='1, 2, 3') + + """ + self.name = name + self.source = source + self.values = values + self.distance = distance + self.got_values_flag = bool(values) + + # initialize .examples from string or list or data directory + if isinstance(examples, str): + self.examples = parse_csv(examples) + elif examples is None: + self.examples = parse_csv(open_data(name + '.csv').read()) + else: + self.examples = examples + + # attrs are the indices of examples, unless otherwise stated. + if self.examples is not None and attrs is None: + attrs = list(range(len(self.examples[0]))) + + self.attrs = attrs + + # initialize .attr_names from string, list, or by default + if isinstance(attr_names, str): + self.attr_names = attr_names.split() + else: + self.attr_names = attr_names or attrs + self.set_problem(target, inputs=inputs, exclude=exclude) + + def set_problem(self, target, inputs=None, exclude=()): + """ + Set (or change) the target and/or inputs. + This way, one DataSet can be used multiple ways. inputs, if specified, + is a list of attributes, or specify exclude as a list of attributes + to not use in inputs. Attributes can be -n .. n, or an attr_name. + Also computes the list of possible values, if that wasn't done yet. + """ + self.target = self.attr_num(target) + exclude = list(map(self.attr_num, exclude)) + if inputs: + self.inputs = remove_all(self.target, inputs) + else: + self.inputs = [a for a in self.attrs if a != self.target and a not in exclude] + if not self.values: + self.update_values() + self.check_me() + + def check_me(self): + """Check that my fields make sense.""" + assert len(self.attr_names) == len(self.attrs) + assert self.target in self.attrs + assert self.target not in self.inputs + assert set(self.inputs).issubset(set(self.attrs)) + if self.got_values_flag: + # only check if values are provided while initializing DataSet + list(map(self.check_example, self.examples)) + + def add_example(self, example): + """Add an example to the list of examples, checking it first.""" + self.check_example(example) + self.examples.append(example) + + def check_example(self, example): + """Raise ValueError if example has any invalid values.""" + if self.values: + for a in self.attrs: + if example[a] not in self.values[a]: + raise ValueError('Bad value {} for attribute {} in {}' + .format(example[a], self.attr_names[a], example)) + + def attr_num(self, attr): + """Returns the number used for attr, which can be a name, or -n .. n-1.""" + if isinstance(attr, str): + return self.attr_names.index(attr) + elif attr < 0: + return len(self.attrs) + attr + else: + return attr + + def update_values(self): + self.values = list(map(unique, zip(*self.examples))) + + def sanitize(self, example): + """Return a copy of example, with non-input attributes replaced by None.""" + return [attr_i if i in self.inputs else None for i, attr_i in enumerate(example)][:-1] + + def classes_to_numbers(self, classes=None): + """Converts class names to numbers.""" + if not classes: + # if classes were not given, extract them from values + classes = sorted(self.values[self.target]) + for item in self.examples: + item[self.target] = classes.index(item[self.target]) + + def remove_examples(self, value=''): + """Remove examples that contain given value.""" + self.examples = [x for x in self.examples if value not in x] + self.update_values() + + def split_values_by_classes(self): + """Split values into buckets according to their class.""" + buckets = defaultdict(lambda: []) + target_names = self.values[self.target] + + for v in self.examples: + item = [a for a in v if a not in target_names] # remove target from item + buckets[v[self.target]].append(item) # add item to bucket of its class + + return buckets + + def find_means_and_deviations(self): + """ + Finds the means and standard deviations of self.dataset. + means : a dictionary for each class/target. Holds a list of the means + of the features for the class. + deviations: a dictionary for each class/target. Holds a list of the sample + standard deviations of the features for the class. + """ + target_names = self.values[self.target] + feature_numbers = len(self.inputs) + + item_buckets = self.split_values_by_classes() + + means = defaultdict(lambda: [0] * feature_numbers) + deviations = defaultdict(lambda: [0] * feature_numbers) + + for t in target_names: + # find all the item feature values for item in class t + features = [[] for _ in range(feature_numbers)] + for item in item_buckets[t]: + for i in range(feature_numbers): + features[i].append(item[i]) + + # calculate means and deviations fo the class + for i in range(feature_numbers): + means[t][i] = mean(features[i]) + deviations[t][i] = stdev(features[i]) + + return means, deviations + + def __repr__(self): + return ''.format(self.name, len(self.examples), len(self.attrs)) diff --git a/utils/utils.py b/utils/utils.py new file mode 100644 index 0000000..f8edda4 --- /dev/null +++ b/utils/utils.py @@ -0,0 +1,198 @@ +### system related operations +import os +from inspect import getsource +from IPython.display import HTML +from IPython.display import display + + +def psource(*functions): + """Print the source code for the given function(s).""" + source_code = '\n\n'.join(getsource(fn) for fn in functions) + try: + from pygments.formatters import HtmlFormatter + from pygments.lexers import PythonLexer + from pygments import highlight + + display(HTML(highlight(source_code, PythonLexer(), HtmlFormatter(full=True)))) + + except ImportError: + print(source_code) + + +def parse_csv(input, delim=','): + r""" + Input is a string consisting of lines, each line has comma-delimited + fields. Convert this into a list of lists. Blank lines are skipped. + Fields that look like numbers are converted to numbers. + The delim defaults to ',' but '\t' and None are also reasonable values. + >>> parse_csv('1, 2, 3 \n 0, 2, na') + [[1, 2, 3], [0, 2, 'na']] + """ + lines = [line for line in input.splitlines() if line.strip()] + return [list(map(num_or_str, line.split(delim))) for line in lines] + + +def open_data(name, mode='r'): + data_root = os.path.dirname(__file__) + data_file = os.path.join(data_root, *[os.pardir, 'data', name]) + + return open(data_file, mode=mode) + + +### math and data structure +import numpy as np +import math +from statistics import mean, stdev +import random + + +def normalize(dist): + """Multiply each number by a constant such that the sum is 1.0""" + if isinstance(dist, dict): + total = sum(dist.values()) + for key in dist: + dist[key] = dist[key] / total + assert 0 <= dist[key] <= 1 # probabilities must be between 0 and 1 + return dist + total = sum(dist) + return [(n / total) for n in dist] + + +def random_weights(min_value, max_value, num_weights): + return [random.uniform(min_value, max_value) for _ in range(num_weights)] + + +def sigmoid(x): + """Return activation value of x with sigmoid function.""" + return 1 / (1 + np.exp(-x)) + + +def sigmoid_derivative(value): + return value * (1 - value) + + +def remove_all(item, seq): + """Return a copy of seq (or string) with all occurrences of item removed.""" + if isinstance(seq, str): + return seq.replace(item, '') + elif isinstance(seq, set): + rest = seq.copy() + rest.remove(item) + return rest + else: + return [x for x in seq if x != item] + + +def unique(seq): + """Remove duplicate elements from seq. Assumes hashable elements.""" + return list(set(seq)) + + +def num_or_str(x): # TODO: rename as `atom` + """The argument is a string; convert to a number if + possible, or strip it.""" + try: + return int(x) + except ValueError: + try: + return float(x) + except ValueError: + return str(x).strip() + + +def euclidean_distance(x, y): + return np.sqrt(sum((_x - _y) ** 2 for _x, _y in zip(x, y))) + + +def manhattan_distance(x, y): + return sum(abs(_x - _y) for _x, _y in zip(x, y)) + + +def hamming_distance(x, y): + return sum(_x != _y for _x, _y in zip(x, y)) + + +def rms_error(x, y): + return np.sqrt(ms_error(x, y)) + + +def ms_error(x, y): + return mean((x - y) ** 2 for x, y in zip(x, y)) + + +def mean_error(x, y): + return mean(abs(x - y) for x, y in zip(x, y)) + + +def mean_boolean_error(x, y): + return mean(_x != _y for _x, _y in zip(x, y)) + + +identity = lambda x: x + + +def argmin_random_tie(seq, key=identity): + """Return a minimum element of seq; break ties at random.""" + return min(shuffled(seq), key=key) + + +def argmax_random_tie(seq, key=identity): + """Return an element with highest fn(seq[i]) score; break ties at random.""" + return max(shuffled(seq), key=key) + + +def shuffled(iterable): + """Randomly shuffle a copy of iterable.""" + items = list(iterable) + random.shuffle(items) + return items + + +### learner related operations +class Activation: + + def function(self, x): + return NotImplementedError + + def derivative(self, x): + return NotImplementedError + + def __call__(self, x): + return self.function(x) + + +class Sigmoid(Activation): + + def function(self, x): + return 1 / (1 + np.exp(-x)) + + def derivative(self, value): + return value * (1 - value) + + +def err_ratio(learner, dataset, examples=None): + """ + Return the proportion of the examples that are NOT correctly predicted. + verbose - 0: No output; 1: Output wrong; 2 (or greater): Output correct + """ + examples = examples or dataset.examples + if len(examples) == 0: + return 0.0 + right = 0 + for example in examples: + desired = example[dataset.target] + output = learner.predict(dataset.sanitize(example)) + if output == desired: + right += 1 + return 1 - (right / len(examples)) + + +def grade_learner(learner, tests): + """ + Grades the given learner based on how many tests it passes. + tests is a list with each element in the form: (values, output). + """ + # for X, y in tests: + # print(learner.predict(X), y) + return mean([int(learner.predict(X) == y) for X, y in tests]) + \ No newline at end of file