chapter 7 finished

ch07-autograd/autograd.ipynb

@@ -0,0 +1,390 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: pygraphviz in /Users/tgbaggio/opt/anaconda3/lib/python3.8/site-packages (1.11)\r\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install pygraphviz"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from utils import Scalar, draw_graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"152pt\" height=\"154pt\"\n",
+       " viewBox=\"0.00 0.00 152.29 154.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 150)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-150 148.295,-150 148.295,4 -4,4\"/>\n",
+       "<!-- 4416457600forward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4416457600forward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"0,-0.5 0,-54.5 63.2949,-54.5 63.2949,-0.5 0,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"0,-36.5 63.2949,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 2.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"0,-18.5 63.2949,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4416458128forward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4416458128forward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M52,-91.5C52,-91.5 91.2949,-91.5 91.2949,-91.5 97.2949,-91.5 103.295,-97.5 103.295,-103.5 103.295,-103.5 103.295,-133.5 103.295,-133.5 103.295,-139.5 97.2949,-145.5 91.2949,-145.5 91.2949,-145.5 52,-145.5 52,-145.5 46,-145.5 40,-139.5 40,-133.5 40,-133.5 40,-103.5 40,-103.5 40,-97.5 46,-91.5 52,-91.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"71.6475\" y=\"-133.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"40,-127.5 103.295,-127.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"71.6475\" y=\"-115.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 3.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"40,-109.5 103.295,-109.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"71.6475\" y=\"-97.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4416457600forward&#45;&gt;4416458128forward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4416457600forward&#45;&gt;4416458128forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M43.4324,-54.7216C47.2979,-63.3221 51.6588,-73.0253 55.7758,-82.1855\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"52.6549,-83.7795 59.9467,-91.4659 59.0397,-80.9099 52.6549,-83.7795\"/>\n",
+       "</g>\n",
+       "<!-- 4416457264forward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4416457264forward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"81,-0.5 81,-54.5 144.295,-54.5 144.295,-0.5 81,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81,-36.5 144.295,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81,-18.5 144.295,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4416457264forward&#45;&gt;4416458128forward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4416457264forward&#45;&gt;4416458128forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M100.568,-54.7216C96.6058,-63.3221 92.1358,-73.0253 87.916,-82.1855\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"84.646,-80.9188 83.6407,-91.4659 91.0038,-83.8477 84.646,-80.9188\"/>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x107233850>"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 简单的计算图\n",
+    "a = Scalar(1.0, label='a')\n",
+    "b = Scalar(2.0, label='b')\n",
+    "c = a + b\n",
+    "draw_graph(c)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"233pt\" height=\"245pt\"\n",
+       " viewBox=\"0.00 0.00 233.29 245.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 241)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-241 229.295,-241 229.295,4 -4,4\"/>\n",
+       "<!-- 4416480256forward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4416480256forward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"0,-0.5 0,-54.5 63.2949,-54.5 63.2949,-0.5 0,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"0,-36.5 63.2949,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 2.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"0,-18.5 63.2949,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4416457264forward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4416457264forward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M52,-91.5C52,-91.5 91.2949,-91.5 91.2949,-91.5 97.2949,-91.5 103.295,-97.5 103.295,-103.5 103.295,-103.5 103.295,-133.5 103.295,-133.5 103.295,-139.5 97.2949,-145.5 91.2949,-145.5 91.2949,-145.5 52,-145.5 52,-145.5 46,-145.5 40,-139.5 40,-133.5 40,-133.5 40,-103.5 40,-103.5 40,-97.5 46,-91.5 52,-91.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"71.6475\" y=\"-133.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"40,-127.5 103.295,-127.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"71.6475\" y=\"-115.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 3.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"40,-109.5 103.295,-109.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"71.6475\" y=\"-97.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4416480256forward&#45;&gt;4416457264forward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4416480256forward&#45;&gt;4416457264forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M43.4324,-54.7216C47.2979,-63.3221 51.6588,-73.0253 55.7758,-82.1855\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"52.6549,-83.7795 59.9467,-91.4659 59.0397,-80.9099 52.6549,-83.7795\"/>\n",
+       "</g>\n",
+       "<!-- 4416480304forward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4416480304forward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M89.5,-182.5C89.5,-182.5 133.795,-182.5 133.795,-182.5 139.795,-182.5 145.795,-188.5 145.795,-194.5 145.795,-194.5 145.795,-224.5 145.795,-224.5 145.795,-230.5 139.795,-236.5 133.795,-236.5 133.795,-236.5 89.5,-236.5 89.5,-236.5 83.5,-236.5 77.5,-230.5 77.5,-224.5 77.5,-224.5 77.5,-194.5 77.5,-194.5 77.5,-188.5 83.5,-182.5 89.5,-182.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"111.647\" y=\"-224.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.5,-218.5 145.795,-218.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"111.647\" y=\"-206.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 12.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.5,-200.5 145.795,-200.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"111.647\" y=\"-188.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4416457264forward&#45;&gt;4416480304forward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4416457264forward&#45;&gt;4416480304forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M83.4324,-145.722C87.2979,-154.322 91.6588,-164.025 95.7758,-173.185\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"92.6549,-174.78 99.9467,-182.466 99.0397,-171.91 92.6549,-174.78\"/>\n",
+       "</g>\n",
+       "<!-- 4416480352forward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4416480352forward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"81,-0.5 81,-54.5 144.295,-54.5 144.295,-0.5 81,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81,-36.5 144.295,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81,-18.5 144.295,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4416480352forward&#45;&gt;4416457264forward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4416480352forward&#45;&gt;4416457264forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M100.568,-54.7216C96.6058,-63.3221 92.1358,-73.0253 87.916,-82.1855\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"84.646,-80.9188 83.6407,-91.4659 91.0038,-83.8477 84.646,-80.9188\"/>\n",
+       "</g>\n",
+       "<!-- 4416458128forward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4416458128forward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M133,-91.5C133,-91.5 172.295,-91.5 172.295,-91.5 178.295,-91.5 184.295,-97.5 184.295,-103.5 184.295,-103.5 184.295,-133.5 184.295,-133.5 184.295,-139.5 178.295,-145.5 172.295,-145.5 172.295,-145.5 133,-145.5 133,-145.5 127,-145.5 121,-139.5 121,-133.5 121,-133.5 121,-103.5 121,-103.5 121,-97.5 127,-91.5 133,-91.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.647\" y=\"-133.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"121,-127.5 184.295,-127.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.647\" y=\"-115.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"121,-109.5 184.295,-109.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.647\" y=\"-97.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4416480352forward&#45;&gt;4416458128forward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4416480352forward&#45;&gt;4416458128forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M124.432,-54.7216C128.298,-63.3221 132.659,-73.0253 136.776,-82.1855\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"133.655,-83.7795 140.947,-91.4659 140.04,-80.9099 133.655,-83.7795\"/>\n",
+       "</g>\n",
+       "<!-- 4416480400forward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4416480400forward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"162,-0.5 162,-54.5 225.295,-54.5 225.295,-0.5 162,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"193.647\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"162,-36.5 225.295,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"193.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"162,-18.5 225.295,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"193.647\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">c</text>\n",
+       "</g>\n",
+       "<!-- 4416480400forward&#45;&gt;4416458128forward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4416480400forward&#45;&gt;4416458128forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M181.568,-54.7216C177.606,-63.3221 173.136,-73.0253 168.916,-82.1855\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"165.646,-80.9188 164.641,-91.4659 172.004,-83.8477 165.646,-80.9188\"/>\n",
+       "</g>\n",
+       "<!-- 4416458128forward&#45;&gt;4416480304forward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4416458128forward&#45;&gt;4416480304forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M140.568,-145.722C136.606,-154.322 132.136,-164.025 127.916,-173.185\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"124.646,-171.919 123.641,-182.466 131.004,-174.848 124.646,-171.919\"/>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x1073e2610>"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 稍微复杂的计算图\n",
+    "a = Scalar(1.0, label='a')\n",
+    "b = Scalar(2.0, label='b')\n",
+    "c = Scalar(4.0, label='c')\n",
+    "d = a + b\n",
+    "e = a * c\n",
+    "f = d * e\n",
+    "backward_process = f.backward(draw_graph)\n",
+    "draw_graph(f)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"235pt\" height=\"273pt\"\n",
+       " viewBox=\"0.00 0.00 235.29 273.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 269)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-269 231.295,-269 231.295,4 -4,4\"/>\n",
+       "<!-- 4416480256backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4416480256backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"0,-0.5 0,-54.5 63.2949,-54.5 63.2949,-0.5 0,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"0,-36.5 63.2949,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 2.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"0,-18.5 63.2949,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4416480304backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4416480304backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M92.5,-210.5C92.5,-210.5 136.795,-210.5 136.795,-210.5 142.795,-210.5 148.795,-216.5 148.795,-222.5 148.795,-222.5 148.795,-252.5 148.795,-252.5 148.795,-258.5 142.795,-264.5 136.795,-264.5 136.795,-264.5 92.5,-264.5 92.5,-264.5 86.5,-264.5 80.5,-258.5 80.5,-252.5 80.5,-252.5 80.5,-222.5 80.5,-222.5 80.5,-216.5 86.5,-210.5 92.5,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"114.647\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"80.5,-246.5 148.795,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"114.647\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 12.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"80.5,-228.5 148.795,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"114.647\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4416457264backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4416457264backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M51,-105.5C51,-105.5 90.2949,-105.5 90.2949,-105.5 96.2949,-105.5 102.295,-111.5 102.295,-117.5 102.295,-117.5 102.295,-147.5 102.295,-147.5 102.295,-153.5 96.2949,-159.5 90.2949,-159.5 90.2949,-159.5 51,-159.5 51,-159.5 45,-159.5 39,-153.5 39,-147.5 39,-147.5 39,-117.5 39,-117.5 39,-111.5 45,-105.5 51,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"70.6475\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"39,-141.5 102.295,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"70.6475\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 3.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"39,-123.5 102.295,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"70.6475\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4416480304backward&#45;&gt;4416457264backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4416480304backward&#45;&gt;4416457264backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M103.431,-210.242C98.1036,-197.772 91.6561,-182.679 85.8933,-169.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"88.991,-167.531 81.8439,-159.71 82.5537,-170.281 88.991,-167.531\"/>\n",
+       "<text text-anchor=\"middle\" x=\"109.647\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4416458128backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4416458128backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M132,-105.5C132,-105.5 171.295,-105.5 171.295,-105.5 177.295,-105.5 183.295,-111.5 183.295,-117.5 183.295,-117.5 183.295,-147.5 183.295,-147.5 183.295,-153.5 177.295,-159.5 171.295,-159.5 171.295,-159.5 132,-159.5 132,-159.5 126,-159.5 120,-153.5 120,-147.5 120,-147.5 120,-117.5 120,-117.5 120,-111.5 126,-105.5 132,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"151.647\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 3.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"120,-141.5 183.295,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"151.647\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"120,-123.5 183.295,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"151.647\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4416480304backward&#45;&gt;4416458128backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4416480304backward&#45;&gt;4416458128backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M124.08,-210.242C128.559,-197.772 133.981,-182.679 138.827,-169.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"142.145,-170.304 142.232,-159.71 135.558,-167.938 142.145,-170.304\"/>\n",
+       "<text text-anchor=\"middle\" x=\"149.647\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 3.00</text>\n",
+       "</g>\n",
+       "<!-- 4416457264backward&#45;&gt;4416480256backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4416457264backward&#45;&gt;4416480256backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M44.0267,-105.081C39.8297,-99.5174 36.0792,-93.3848 33.6475,-87 30.9867,-80.014 29.69,-72.2124 29.1931,-64.6408\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"32.6902,-64.4725 28.9582,-54.5567 25.6921,-64.6356 32.6902,-64.4725\"/>\n",
+       "<text text-anchor=\"middle\" x=\"47.6475\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4416480352backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4416480352backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"81.4434,-0.5 81.4434,-54.5 145.852,-54.5 145.852,-0.5 81.4434,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"113.647\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 16.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81.4434,-36.5 145.852,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"113.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81.4434,-18.5 145.852,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"113.647\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4416457264backward&#45;&gt;4416480352backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4416457264backward&#45;&gt;4416480352backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M81.6094,-105.242C86.8153,-92.7723 93.1163,-77.6793 98.7481,-64.1891\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"102.083,-65.2863 102.706,-54.7098 95.6231,-62.5895 102.083,-65.2863\"/>\n",
+       "<text text-anchor=\"middle\" x=\"109.647\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4416480400backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4416480400backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"164,-0.5 164,-54.5 227.295,-54.5 227.295,-0.5 164,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"195.647\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 3.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"164,-36.5 227.295,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"195.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"164,-18.5 227.295,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"195.647\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">c</text>\n",
+       "</g>\n",
+       "<!-- 4416458128backward&#45;&gt;4416480352backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4416458128backward&#45;&gt;4416480352backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M141.96,-105.242C137.36,-92.7723 131.791,-77.6793 126.814,-64.1891\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"130.062,-62.8802 123.317,-54.7098 123.495,-65.3032 130.062,-62.8802\"/>\n",
+       "<text text-anchor=\"middle\" x=\"153.147\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 12.00</text>\n",
+       "</g>\n",
+       "<!-- 4416458128backward&#45;&gt;4416480400backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4416458128backward&#45;&gt;4416480400backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M165.115,-105.349C168.042,-99.3715 171.04,-93.0085 173.647,-87 176.836,-79.6512 180.016,-71.6757 182.926,-64.0461\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"186.219,-65.2325 186.448,-54.6401 179.664,-62.7775 186.219,-65.2325\"/>\n",
+       "<text text-anchor=\"middle\" x=\"193.647\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 3.00</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x1073dcfa0>"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "draw_graph(f, 'backward')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 将反向传播的过程展示出来\n",
+    "for index, pic in enumerate(backward_process):\n",
+    "    pic.view(str(index))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

ch07-autograd/dropout.ipynb

@@ -0,0 +1,917 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from utils import Scalar, draw_graph\n",
+    "from linear_model import Linear, mse"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"308pt\" height=\"483pt\"\n",
+       " viewBox=\"0.00 0.00 308.46 483.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 479)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-479 304.462,-479 304.462,4 -4,4\"/>\n",
+       "<!-- 4398170656backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4398170656backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"187.814\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y1= 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4398171184backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4398171184backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-105.5 156.167,-159.5 219.462,-159.5 219.462,-105.5 156.167,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;5.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-141.5 219.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-123.5 219.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4398171232backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4398171232backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M247.167,-210.5C247.167,-210.5 286.462,-210.5 286.462,-210.5 292.462,-210.5 298.462,-216.5 298.462,-222.5 298.462,-222.5 298.462,-252.5 298.462,-252.5 298.462,-258.5 292.462,-264.5 286.462,-264.5 286.462,-264.5 247.167,-264.5 247.167,-264.5 241.167,-264.5 235.167,-258.5 235.167,-252.5 235.167,-252.5 235.167,-222.5 235.167,-222.5 235.167,-216.5 241.167,-210.5 247.167,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"235.167,-246.5 298.462,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"235.167,-228.5 298.462,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4398171232backward&#45;&gt;4398171184backward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4398171232backward&#45;&gt;4398171184backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M246.675,-210.242C236.832,-197.409 224.858,-181.798 214.286,-168.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"216.78,-165.515 207.917,-159.71 211.226,-169.775 216.78,-165.515\"/>\n",
+       "<text text-anchor=\"middle\" x=\"246.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4398170992backward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4398170992backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M249.167,-105.5C249.167,-105.5 288.462,-105.5 288.462,-105.5 294.462,-105.5 300.462,-111.5 300.462,-117.5 300.462,-117.5 300.462,-147.5 300.462,-147.5 300.462,-153.5 294.462,-159.5 288.462,-159.5 288.462,-159.5 249.167,-159.5 249.167,-159.5 243.167,-159.5 237.167,-153.5 237.167,-147.5 237.167,-147.5 237.167,-117.5 237.167,-117.5 237.167,-111.5 243.167,-105.5 249.167,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"237.167,-141.5 300.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"237.167,-123.5 300.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4398171232backward&#45;&gt;4398170992backward -->\n",
+       "<g id=\"edge14\" class=\"edge\"><title>4398171232backward&#45;&gt;4398170992backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M267.324,-210.242C267.561,-198.014 267.848,-183.264 268.106,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"271.61,-169.776 268.305,-159.71 264.612,-169.64 271.61,-169.776\"/>\n",
+       "<text text-anchor=\"middle\" x=\"282.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4398170752backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4398170752backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"29.8142\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"29.8142\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y2= 4.50</text>\n",
+       "</g>\n",
+       "<!-- 4398170800backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4398170800backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"268.814\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x1= 1.50</text>\n",
+       "</g>\n",
+       "<!-- 4398170848backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4398170848backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"106.814\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x2= 2.00</text>\n",
+       "</g>\n",
+       "<!-- 4398171376backward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4398171376backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M89.1667,-210.5C89.1667,-210.5 128.462,-210.5 128.462,-210.5 134.462,-210.5 140.462,-216.5 140.462,-222.5 140.462,-222.5 140.462,-252.5 140.462,-252.5 140.462,-258.5 134.462,-264.5 128.462,-264.5 128.462,-264.5 89.1667,-264.5 89.1667,-264.5 83.1667,-264.5 77.1667,-258.5 77.1667,-252.5 77.1667,-252.5 77.1667,-222.5 77.1667,-222.5 77.1667,-216.5 83.1667,-210.5 89.1667,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.1667,-246.5 140.462,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.1667,-228.5 140.462,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4398171376backward&#45;&gt;4398171184backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4398171376backward&#45;&gt;4398171184backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M128.954,-210.242C138.796,-197.409 150.77,-181.798 161.342,-168.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"164.403,-169.775 167.711,-159.71 158.848,-165.515 164.403,-169.775\"/>\n",
+       "<text text-anchor=\"middle\" x=\"168.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.50</text>\n",
+       "</g>\n",
+       "<!-- 4398170512backward -->\n",
+       "<g id=\"node11\" class=\"node\"><title>4398170512backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M87.1667,-105.5C87.1667,-105.5 126.462,-105.5 126.462,-105.5 132.462,-105.5 138.462,-111.5 138.462,-117.5 138.462,-117.5 138.462,-147.5 138.462,-147.5 138.462,-153.5 132.462,-159.5 126.462,-159.5 126.462,-159.5 87.1667,-159.5 87.1667,-159.5 81.1667,-159.5 75.1667,-153.5 75.1667,-147.5 75.1667,-147.5 75.1667,-117.5 75.1667,-117.5 75.1667,-111.5 81.1667,-105.5 87.1667,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"75.1667,-141.5 138.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"75.1667,-123.5 138.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4398171376backward&#45;&gt;4398170512backward -->\n",
+       "<g id=\"edge13\" class=\"edge\"><title>4398171376backward&#45;&gt;4398170512backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M108.304,-210.242C108.067,-198.014 107.78,-183.264 107.522,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"111.017,-169.64 107.323,-159.71 104.018,-169.776 111.017,-169.64\"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.50</text>\n",
+       "</g>\n",
+       "<!-- 4398171424backward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4398171424backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M87.1667,-315.5C87.1667,-315.5 126.462,-315.5 126.462,-315.5 132.462,-315.5 138.462,-321.5 138.462,-327.5 138.462,-327.5 138.462,-357.5 138.462,-357.5 138.462,-363.5 132.462,-369.5 126.462,-369.5 126.462,-369.5 87.1667,-369.5 87.1667,-369.5 81.1667,-369.5 75.1667,-363.5 75.1667,-357.5 75.1667,-357.5 75.1667,-327.5 75.1667,-327.5 75.1667,-321.5 81.1667,-315.5 87.1667,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"75.1667,-351.5 138.462,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"75.1667,-333.5 138.462,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4398171424backward&#45;&gt;4398170752backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4398171424backward&#45;&gt;4398170752backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M87.1846,-315.242C72.7481,-295.931 53.6087,-270.329 41.4997,-254.131\"/>\n",
+       "</g>\n",
+       "<!-- 4398171424backward&#45;&gt;4398171376backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4398171424backward&#45;&gt;4398171376backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M107.324,-315.242C107.561,-303.014 107.848,-288.264 108.106,-274.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"111.61,-274.776 108.305,-264.71 104.612,-274.64 111.61,-274.776\"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.395\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.50</text>\n",
+       "</g>\n",
+       "<!-- 4398170992backward&#45;&gt;4398170800backward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4398170992backward&#45;&gt;4398170800backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M268.814,-105.242C268.814,-86.5272 268.814,-61.9035 268.814,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4398171040backward -->\n",
+       "<g id=\"node12\" class=\"node\"><title>4398171040backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"155.195,-0.5 155.195,-54.5 220.433,-54.5 220.433,-0.5 155.195,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;10.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"155.195,-36.5 220.433,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"155.195,-18.5 220.433,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4398170992backward&#45;&gt;4398171040backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4398170992backward&#45;&gt;4398171040backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M248.165,-105.242C237.978,-92.2881 225.564,-76.503 214.65,-62.6249\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"217.359,-60.4068 208.426,-54.7098 211.856,-64.7339 217.359,-60.4068\"/>\n",
+       "<text text-anchor=\"middle\" x=\"247.395\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.50</text>\n",
+       "</g>\n",
+       "<!-- 4398171520backward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4398171520backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M124.667,-420.5C124.667,-420.5 168.962,-420.5 168.962,-420.5 174.962,-420.5 180.962,-426.5 180.962,-432.5 180.962,-432.5 180.962,-462.5 180.962,-462.5 180.962,-468.5 174.962,-474.5 168.962,-474.5 168.962,-474.5 124.667,-474.5 124.667,-474.5 118.667,-474.5 112.667,-468.5 112.667,-462.5 112.667,-462.5 112.667,-432.5 112.667,-432.5 112.667,-426.5 118.667,-420.5 124.667,-420.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"146.814\" y=\"-462.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"112.667,-456.5 180.962,-456.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"146.814\" y=\"-444.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 10.62</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"112.667,-438.5 180.962,-438.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"146.814\" y=\"-426.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4398171520backward&#45;&gt;4398171424backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4398171520backward&#45;&gt;4398171424backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M136.617,-420.242C131.774,-407.772 125.913,-392.679 120.674,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"123.876,-377.765 116.993,-369.71 117.35,-380.299 123.876,-377.765\"/>\n",
+       "<text text-anchor=\"middle\" x=\"143.814\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.50</text>\n",
+       "</g>\n",
+       "<!-- 4398171088backward -->\n",
+       "<g id=\"node13\" class=\"node\"><title>4398171088backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M168.167,-315.5C168.167,-315.5 207.462,-315.5 207.462,-315.5 213.462,-315.5 219.462,-321.5 219.462,-327.5 219.462,-327.5 219.462,-357.5 219.462,-357.5 219.462,-363.5 213.462,-369.5 207.462,-369.5 207.462,-369.5 168.167,-369.5 168.167,-369.5 162.167,-369.5 156.167,-363.5 156.167,-357.5 156.167,-357.5 156.167,-327.5 156.167,-327.5 156.167,-321.5 162.167,-315.5 168.167,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"156.167,-351.5 219.462,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"156.167,-333.5 219.462,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4398171520backward&#45;&gt;4398171088backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4398171520backward&#45;&gt;4398171088backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M157.266,-420.242C162.23,-407.772 168.238,-392.679 173.608,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"176.935,-380.295 177.381,-369.71 170.431,-377.706 176.935,-380.295\"/>\n",
+       "<text text-anchor=\"middle\" x=\"182.814\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4398170512backward&#45;&gt;4398170848backward -->\n",
+       "<g id=\"edge11\" class=\"edge\"><title>4398170512backward&#45;&gt;4398170848backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M106.814,-105.242C106.814,-86.5272 106.814,-61.9035 106.814,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4398170512backward&#45;&gt;4398171040backward -->\n",
+       "<g id=\"edge12\" class=\"edge\"><title>4398170512backward&#45;&gt;4398171040backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M127.463,-105.242C137.651,-92.2881 150.064,-76.503 160.978,-62.6249\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"163.772,-64.7339 167.203,-54.7098 158.27,-60.4068 163.772,-64.7339\"/>\n",
+       "<text text-anchor=\"middle\" x=\"167.395\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;9.00</text>\n",
+       "</g>\n",
+       "<!-- 4398171088backward&#45;&gt;4398170656backward -->\n",
+       "<g id=\"edge10\" class=\"edge\"><title>4398171088backward&#45;&gt;4398170656backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M187.814,-315.242C187.814,-296.527 187.814,-271.903 187.814,-255.657\"/>\n",
+       "</g>\n",
+       "<!-- 4398171088backward&#45;&gt;4398171232backward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4398171088backward&#45;&gt;4398171232backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M207.954,-315.242C217.796,-302.409 229.77,-286.798 240.342,-273.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"243.403,-274.775 246.711,-264.71 237.848,-270.515 243.403,-274.775\"/>\n",
+       "<text text-anchor=\"middle\" x=\"246.395\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x10626c6a0>"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x1 = Scalar(1.5, label='x1', requires_grad=False)\n",
+    "y1 = Scalar(1.0, label='y1', requires_grad=False)\n",
+    "x2 = Scalar(2.0, label='x2', requires_grad=False)\n",
+    "y2 = Scalar(4.5, label='y2', requires_grad=False)\n",
+    "# 反向传播\n",
+    "model = Linear()\n",
+    "loss = mse([model.error(x1, y1), model.error(x2, y2)])\n",
+    "loss.backward()\n",
+    "draw_graph(loss, 'backward')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"307pt\" height=\"483pt\"\n",
+       " viewBox=\"0.00 0.00 306.63 483.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 479)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-479 302.628,-479 302.628,4 -4,4\"/>\n",
+       "<!-- 4387998736backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4387998736backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M170.167,-210.5C170.167,-210.5 209.462,-210.5 209.462,-210.5 215.462,-210.5 221.462,-216.5 221.462,-222.5 221.462,-222.5 221.462,-252.5 221.462,-252.5 221.462,-258.5 215.462,-264.5 209.462,-264.5 209.462,-264.5 170.167,-264.5 170.167,-264.5 164.167,-264.5 158.167,-258.5 158.167,-252.5 158.167,-252.5 158.167,-222.5 158.167,-222.5 158.167,-216.5 164.167,-210.5 170.167,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.814\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"158.167,-246.5 221.462,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"158.167,-228.5 221.462,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.814\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4387998640backward -->\n",
+       "<g id=\"node11\" class=\"node\"><title>4387998640backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"200.167,-105.5 200.167,-159.5 263.462,-159.5 263.462,-105.5 200.167,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"231.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"200.167,-141.5 263.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"231.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"200.167,-123.5 263.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"231.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4387998736backward&#45;&gt;4387998640backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4387998736backward&#45;&gt;4387998640backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M200.521,-210.242C205.606,-197.772 211.76,-182.679 217.261,-169.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"220.592,-170.291 221.127,-159.71 214.11,-167.648 220.592,-170.291\"/>\n",
+       "<text text-anchor=\"middle\" x=\"227.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4387997680backward -->\n",
+       "<g id=\"node13\" class=\"node\"><title>4387997680backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"119.167,-105.5 119.167,-159.5 182.462,-159.5 182.462,-105.5 119.167,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"150.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;5.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"119.167,-141.5 182.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"150.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"119.167,-123.5 182.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"150.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4387998736backward&#45;&gt;4387997680backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4387998736backward&#45;&gt;4387997680backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M179.872,-210.242C175.15,-197.772 169.435,-182.679 164.328,-169.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"167.553,-167.822 160.738,-159.71 161.006,-170.301 167.553,-167.822\"/>\n",
+       "<text text-anchor=\"middle\" x=\"186.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4398170656backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4398170656backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"268.814\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y1= 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4387998784backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4387998784backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"231.814\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"231.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x1= 0.00</text>\n",
+       "</g>\n",
+       "<!-- 4387996768backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4387996768backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M130.667,-420.5C130.667,-420.5 174.962,-420.5 174.962,-420.5 180.962,-420.5 186.962,-426.5 186.962,-432.5 186.962,-432.5 186.962,-462.5 186.962,-462.5 186.962,-468.5 180.962,-474.5 174.962,-474.5 174.962,-474.5 130.667,-474.5 130.667,-474.5 124.667,-474.5 118.667,-468.5 118.667,-462.5 118.667,-462.5 118.667,-432.5 118.667,-432.5 118.667,-426.5 124.667,-420.5 130.667,-420.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.814\" y=\"-462.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"118.667,-456.5 186.962,-456.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.814\" y=\"-444.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 10.62</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"118.667,-438.5 186.962,-438.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.814\" y=\"-426.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4387995856backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4387995856backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M89.1667,-315.5C89.1667,-315.5 128.462,-315.5 128.462,-315.5 134.462,-315.5 140.462,-321.5 140.462,-327.5 140.462,-327.5 140.462,-357.5 140.462,-357.5 140.462,-363.5 134.462,-369.5 128.462,-369.5 128.462,-369.5 89.1667,-369.5 89.1667,-369.5 83.1667,-369.5 77.1667,-363.5 77.1667,-357.5 77.1667,-357.5 77.1667,-327.5 77.1667,-327.5 77.1667,-321.5 83.1667,-315.5 89.1667,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.1667,-351.5 140.462,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.1667,-333.5 140.462,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4387996768backward&#45;&gt;4387995856backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4387996768backward&#45;&gt;4387995856backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M141.597,-420.242C136.27,-407.772 129.823,-392.679 124.06,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"127.158,-377.531 120.011,-369.71 120.72,-380.281 127.158,-377.531\"/>\n",
+       "<text text-anchor=\"middle\" x=\"147.814\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.50</text>\n",
+       "</g>\n",
+       "<!-- 4387998448backward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4387998448backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M170.167,-315.5C170.167,-315.5 209.462,-315.5 209.462,-315.5 215.462,-315.5 221.462,-321.5 221.462,-327.5 221.462,-327.5 221.462,-357.5 221.462,-357.5 221.462,-363.5 215.462,-369.5 209.462,-369.5 209.462,-369.5 170.167,-369.5 170.167,-369.5 164.167,-369.5 158.167,-363.5 158.167,-357.5 158.167,-357.5 158.167,-327.5 158.167,-327.5 158.167,-321.5 164.167,-315.5 170.167,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.814\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"158.167,-351.5 221.462,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.814\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"158.167,-333.5 221.462,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.814\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4387996768backward&#45;&gt;4387998448backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4387996768backward&#45;&gt;4387998448backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M162.247,-420.242C166.726,-407.772 172.148,-392.679 176.994,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"180.312,-380.304 180.399,-369.71 173.724,-377.938 180.312,-380.304\"/>\n",
+       "<text text-anchor=\"middle\" x=\"186.814\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4398170752backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4398170752backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"29.8142\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"29.8142\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y2= 4.50</text>\n",
+       "</g>\n",
+       "<!-- 4387995856backward&#45;&gt;4398170752backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4387995856backward&#45;&gt;4398170752backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M88.6747,-315.242C73.8633,-295.931 54.2268,-270.329 41.8032,-254.131\"/>\n",
+       "</g>\n",
+       "<!-- 4387998544backward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4387998544backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M89.1667,-210.5C89.1667,-210.5 128.462,-210.5 128.462,-210.5 134.462,-210.5 140.462,-216.5 140.462,-222.5 140.462,-222.5 140.462,-252.5 140.462,-252.5 140.462,-258.5 134.462,-264.5 128.462,-264.5 128.462,-264.5 89.1667,-264.5 89.1667,-264.5 83.1667,-264.5 77.1667,-258.5 77.1667,-252.5 77.1667,-252.5 77.1667,-222.5 77.1667,-222.5 77.1667,-216.5 83.1667,-210.5 89.1667,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.1667,-246.5 140.462,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.1667,-228.5 140.462,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4387995856backward&#45;&gt;4387998544backward -->\n",
+       "<g id=\"edge14\" class=\"edge\"><title>4387995856backward&#45;&gt;4387998544backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M108.814,-315.242C108.814,-303.014 108.814,-288.264 108.814,-274.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"112.314,-274.71 108.814,-264.71 105.314,-274.71 112.314,-274.71\"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.395\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.50</text>\n",
+       "</g>\n",
+       "<!-- 4398170848backward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4398170848backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"69.8142\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"69.8142\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x2= 2.00</text>\n",
+       "</g>\n",
+       "<!-- 4387998448backward&#45;&gt;4387998736backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4387998448backward&#45;&gt;4387998736backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M189.814,-315.242C189.814,-303.014 189.814,-288.264 189.814,-274.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"193.314,-274.71 189.814,-264.71 186.314,-274.71 193.314,-274.71\"/>\n",
+       "<text text-anchor=\"middle\" x=\"204.395\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4387998448backward&#45;&gt;4398170656backward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4387998448backward&#45;&gt;4398170656backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M209.954,-315.242C224.765,-295.931 244.402,-270.329 256.825,-254.131\"/>\n",
+       "</g>\n",
+       "<!-- 4387997968backward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4387997968backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"119.167,-0.5 119.167,-54.5 182.462,-54.5 182.462,-0.5 119.167,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"150.814\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;9.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"119.167,-36.5 182.462,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"150.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"119.167,-18.5 182.462,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"150.814\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4387998688backward -->\n",
+       "<g id=\"node12\" class=\"node\"><title>4387998688backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M50.1667,-105.5C50.1667,-105.5 89.4616,-105.5 89.4616,-105.5 95.4616,-105.5 101.462,-111.5 101.462,-117.5 101.462,-117.5 101.462,-147.5 101.462,-147.5 101.462,-153.5 95.4616,-159.5 89.4616,-159.5 89.4616,-159.5 50.1667,-159.5 50.1667,-159.5 44.1667,-159.5 38.1667,-153.5 38.1667,-147.5 38.1667,-147.5 38.1667,-117.5 38.1667,-117.5 38.1667,-111.5 44.1667,-105.5 50.1667,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"69.8142\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"38.1667,-141.5 101.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"69.8142\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"38.1667,-123.5 101.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"69.8142\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4387998544backward&#45;&gt;4387998688backward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4387998544backward&#45;&gt;4387998688backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M82.0634,-210.101C77.8492,-204.535 74.0862,-198.397 71.6521,-192 68.9937,-185.014 67.7114,-177.212 67.2338,-169.641\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"70.7314,-169.483 67.0276,-159.556 63.7328,-169.626 70.7314,-169.483\"/>\n",
+       "<text text-anchor=\"middle\" x=\"86.3952\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.50</text>\n",
+       "</g>\n",
+       "<!-- 4387998544backward&#45;&gt;4387997680backward -->\n",
+       "<g id=\"edge13\" class=\"edge\"><title>4387998544backward&#45;&gt;4387997680backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M119.521,-210.242C124.606,-197.772 130.76,-182.679 136.261,-169.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"139.592,-170.291 140.127,-159.71 133.11,-167.648 139.592,-170.291\"/>\n",
+       "<text text-anchor=\"middle\" x=\"147.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.50</text>\n",
+       "</g>\n",
+       "<!-- 4387998640backward&#45;&gt;4387998784backward -->\n",
+       "<g id=\"edge12\" class=\"edge\"><title>4387998640backward&#45;&gt;4387998784backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M231.814,-105.242C231.814,-86.5272 231.814,-61.9035 231.814,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4387998640backward&#45;&gt;4387997968backward -->\n",
+       "<g id=\"edge10\" class=\"edge\"><title>4387998640backward&#45;&gt;4387997968backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M211.165,-105.242C200.978,-92.2881 188.564,-76.503 177.65,-62.6249\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"180.359,-60.4068 171.426,-54.7098 174.856,-64.7339 180.359,-60.4068\"/>\n",
+       "<text text-anchor=\"middle\" x=\"209.814\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0.00</text>\n",
+       "</g>\n",
+       "<!-- 4387998688backward&#45;&gt;4398170848backward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4387998688backward&#45;&gt;4398170848backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M69.8142,-105.242C69.8142,-86.5272 69.8142,-61.9035 69.8142,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4387998688backward&#45;&gt;4387997968backward -->\n",
+       "<g id=\"edge11\" class=\"edge\"><title>4387998688backward&#45;&gt;4387997968backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M90.4635,-105.242C100.651,-92.2881 113.064,-76.503 123.978,-62.6249\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"126.772,-64.7339 130.203,-54.7098 121.27,-60.4068 126.772,-64.7339\"/>\n",
+       "<text text-anchor=\"middle\" x=\"130.395\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;9.00</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x10626c310>"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 将变量x1设置成0\n",
+    "x1 = Scalar(0.0, label='x1', requires_grad=False)\n",
+    "# 反向传播\n",
+    "model = Linear()\n",
+    "loss = mse([model.error(x1, y1), model.error(x2, y2)])\n",
+    "loss.backward()\n",
+    "draw_graph(loss, 'backward')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"308pt\" height=\"483pt\"\n",
+       " viewBox=\"0.00 0.00 308.46 483.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 479)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-479 304.462,-479 304.462,4 -4,4\"/>\n",
+       "<!-- 4398102032backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4398102032backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"193.647\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"193.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x1= 1.50</text>\n",
+       "</g>\n",
+       "<!-- 4398102080backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4398102080backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"81,-0.5 81,-54.5 144.295,-54.5 144.295,-0.5 81,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;8.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81,-36.5 144.295,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81,-18.5 144.295,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4398313536backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4398313536backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M92,-315.5C92,-315.5 131.295,-315.5 131.295,-315.5 137.295,-315.5 143.295,-321.5 143.295,-327.5 143.295,-327.5 143.295,-357.5 143.295,-357.5 143.295,-363.5 137.295,-369.5 131.295,-369.5 131.295,-369.5 92,-369.5 92,-369.5 86,-369.5 80,-363.5 80,-357.5 80,-357.5 80,-327.5 80,-327.5 80,-321.5 86,-315.5 92,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"111.647\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"80,-351.5 143.295,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"111.647\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"80,-333.5 143.295,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"111.647\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4398313680backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4398313680backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M14,-210.5C14,-210.5 53.2949,-210.5 53.2949,-210.5 59.2949,-210.5 65.2949,-216.5 65.2949,-222.5 65.2949,-222.5 65.2949,-252.5 65.2949,-252.5 65.2949,-258.5 59.2949,-264.5 53.2949,-264.5 53.2949,-264.5 14,-264.5 14,-264.5 8,-264.5 2,-258.5 2,-252.5 2,-252.5 2,-222.5 2,-222.5 2,-216.5 8,-210.5 14,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"33.6475\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"2,-246.5 65.2949,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"33.6475\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"2,-228.5 65.2949,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"33.6475\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4398313536backward&#45;&gt;4398313680backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4398313536backward&#45;&gt;4398313680backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M91.7629,-315.242C82.0447,-302.409 70.2225,-286.798 59.7847,-273.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"62.323,-270.569 53.4957,-264.71 56.7426,-274.795 62.323,-270.569\"/>\n",
+       "<text text-anchor=\"middle\" x=\"91.2285\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4398104288backward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4398104288backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"112.647\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y2= 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4398313536backward&#45;&gt;4398104288backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4398313536backward&#45;&gt;4398104288backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M111.902,-315.242C112.084,-296.527 112.323,-271.903 112.481,-255.657\"/>\n",
+       "</g>\n",
+       "<!-- 4398313584backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4398313584backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M136,-420.5C136,-420.5 175.295,-420.5 175.295,-420.5 181.295,-420.5 187.295,-426.5 187.295,-432.5 187.295,-432.5 187.295,-462.5 187.295,-462.5 187.295,-468.5 181.295,-474.5 175.295,-474.5 175.295,-474.5 136,-474.5 136,-474.5 130,-474.5 124,-468.5 124,-462.5 124,-462.5 124,-432.5 124,-432.5 124,-426.5 130,-420.5 136,-420.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"155.647\" y=\"-462.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"124,-456.5 187.295,-456.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"155.647\" y=\"-444.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 8.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"124,-438.5 187.295,-438.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"155.647\" y=\"-426.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4398313584backward&#45;&gt;4398313536backward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4398313584backward&#45;&gt;4398313536backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M144.431,-420.242C139.104,-407.772 132.656,-392.679 126.893,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"129.991,-377.531 122.844,-369.71 123.554,-380.281 129.991,-377.531\"/>\n",
+       "<text text-anchor=\"middle\" x=\"149.647\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4398102320backward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4398102320backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"161,-315.5 161,-369.5 224.295,-369.5 224.295,-315.5 161,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"192.647\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"161,-351.5 224.295,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"192.647\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"161,-333.5 224.295,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"192.647\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4398313584backward&#45;&gt;4398102320backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4398313584backward&#45;&gt;4398102320backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M165.08,-420.242C169.559,-407.772 174.981,-392.679 179.827,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"183.145,-380.304 183.232,-369.71 176.558,-377.938 183.145,-380.304\"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.647\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0.00</text>\n",
+       "</g>\n",
+       "<!-- 4398103184backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4398103184backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"160,-210.5 160,-264.5 223.295,-264.5 223.295,-210.5 160,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"191.647\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"160,-246.5 223.295,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"191.647\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"160,-228.5 223.295,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"191.647\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4398102800backward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4398102800backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"81,-105.5 81,-159.5 144.295,-159.5 144.295,-105.5 81,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81,-141.5 144.295,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81,-123.5 144.295,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4398103184backward&#45;&gt;4398102800backward -->\n",
+       "<g id=\"edge10\" class=\"edge\"><title>4398103184backward&#45;&gt;4398102800backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M171.508,-210.242C161.665,-197.409 149.691,-181.798 139.12,-168.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"141.613,-165.515 132.75,-159.71 136.059,-169.775 141.613,-165.515\"/>\n",
+       "<text text-anchor=\"middle\" x=\"170.647\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0.00</text>\n",
+       "</g>\n",
+       "<!-- 4398103376backward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4398103376backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"162,-105.5 162,-159.5 225.295,-159.5 225.295,-105.5 162,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"193.647\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"162,-141.5 225.295,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"193.647\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"162,-123.5 225.295,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"193.647\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4398103184backward&#45;&gt;4398103376backward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4398103184backward&#45;&gt;4398103376backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M192.157,-210.242C192.395,-198.014 192.681,-183.264 192.939,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"196.444,-169.776 193.139,-159.71 189.445,-169.64 196.444,-169.776\"/>\n",
+       "<text text-anchor=\"middle\" x=\"206.647\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0.00</text>\n",
+       "</g>\n",
+       "<!-- 4398313680backward&#45;&gt;4398102800backward -->\n",
+       "<g id=\"edge14\" class=\"edge\"><title>4398313680backward&#45;&gt;4398102800backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M53.7869,-210.242C63.6298,-197.409 75.6035,-181.798 86.1752,-168.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"89.236,-169.775 92.5448,-159.71 83.6816,-165.515 89.236,-169.775\"/>\n",
+       "<text text-anchor=\"middle\" x=\"92.2285\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4398103472backward -->\n",
+       "<g id=\"node11\" class=\"node\"><title>4398103472backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M12,-105.5C12,-105.5 51.2949,-105.5 51.2949,-105.5 57.2949,-105.5 63.2949,-111.5 63.2949,-117.5 63.2949,-117.5 63.2949,-147.5 63.2949,-147.5 63.2949,-153.5 57.2949,-159.5 51.2949,-159.5 51.2949,-159.5 12,-159.5 12,-159.5 6,-159.5 0,-153.5 0,-147.5 0,-147.5 0,-117.5 0,-117.5 0,-111.5 6,-105.5 12,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"0,-141.5 63.2949,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"0,-123.5 63.2949,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4398313680backward&#45;&gt;4398103472backward -->\n",
+       "<g id=\"edge12\" class=\"edge\"><title>4398313680backward&#45;&gt;4398103472backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M33.1376,-210.242C32.9002,-198.014 32.6138,-183.264 32.3557,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"35.85,-169.64 32.1564,-159.71 28.8513,-169.776 35.85,-169.64\"/>\n",
+       "<text text-anchor=\"middle\" x=\"48.2285\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4398102320backward&#45;&gt;4398103184backward -->\n",
+       "<g id=\"edge11\" class=\"edge\"><title>4398102320backward&#45;&gt;4398103184backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M192.393,-315.242C192.274,-303.014 192.131,-288.264 192.002,-274.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"195.499,-274.675 191.902,-264.71 188.499,-274.743 195.499,-274.675\"/>\n",
+       "<text text-anchor=\"middle\" x=\"205.647\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0.00</text>\n",
+       "</g>\n",
+       "<!-- 4398104528backward -->\n",
+       "<g id=\"node12\" class=\"node\"><title>4398104528backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"270.647\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"270.647\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y1= 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4398102320backward&#45;&gt;4398104528backward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4398102320backward&#45;&gt;4398104528backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M212.532,-315.242C227.156,-295.931 246.544,-270.329 258.81,-254.131\"/>\n",
+       "</g>\n",
+       "<!-- 4398103376backward&#45;&gt;4398102032backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4398103376backward&#45;&gt;4398102032backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M193.647,-105.242C193.647,-86.5272 193.647,-61.9035 193.647,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4398103376backward&#45;&gt;4398102080backward -->\n",
+       "<g id=\"edge13\" class=\"edge\"><title>4398103376backward&#45;&gt;4398102080backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M172.998,-105.242C162.811,-92.2881 150.397,-76.503 139.484,-62.6249\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"142.192,-60.4068 133.259,-54.7098 136.689,-64.7339 142.192,-60.4068\"/>\n",
+       "<text text-anchor=\"middle\" x=\"171.647\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0.00</text>\n",
+       "</g>\n",
+       "<!-- 4398103472backward&#45;&gt;4398102080backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4398103472backward&#45;&gt;4398102080backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M52.2968,-105.242C62.484,-92.2881 74.8975,-76.503 85.8114,-62.6249\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"88.6054,-64.7339 92.0359,-54.7098 83.103,-60.4068 88.6054,-64.7339\"/>\n",
+       "<text text-anchor=\"middle\" x=\"91.2285\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;8.00</text>\n",
+       "</g>\n",
+       "<!-- 4398102512backward -->\n",
+       "<g id=\"node13\" class=\"node\"><title>4398102512backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"31.6475\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x2= 2.00</text>\n",
+       "</g>\n",
+       "<!-- 4398103472backward&#45;&gt;4398102512backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4398103472backward&#45;&gt;4398102512backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M31.6475,-105.242C31.6475,-86.5272 31.6475,-61.9035 31.6475,-45.6575\"/>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x10628f1f0>"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x1 = Scalar(1.5, label='x1', requires_grad=False)\n",
+    "y1 = Scalar(1.0, label='y1', requires_grad=False)\n",
+    "x2 = Scalar(2.0, label='x2', requires_grad=False)\n",
+    "y2 = Scalar(4.0, label='y2', requires_grad=False)\n",
+    "# 反向传播\n",
+    "model = Linear()\n",
+    "l = model.error(x1, y1)\n",
+    "# 将变量x1，y1的损失设置成0\n",
+    "l.value = 0.0\n",
+    "loss = mse([l, model.error(x2, y2)])\n",
+    "loss.backward()\n",
+    "draw_graph(loss, 'backward')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"308pt\" height=\"483pt\"\n",
+       " viewBox=\"0.00 0.00 308.46 483.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 479)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-479 304.462,-479 304.462,4 -4,4\"/>\n",
+       "<!-- 4398315552backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4398315552backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-0.5 156.167,-54.5 219.462,-54.5 219.462,-0.5 156.167,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;9.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-36.5 219.462,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-18.5 219.462,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4398315648backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4398315648backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M88.1667,-315.5C88.1667,-315.5 127.462,-315.5 127.462,-315.5 133.462,-315.5 139.462,-321.5 139.462,-327.5 139.462,-327.5 139.462,-357.5 139.462,-357.5 139.462,-363.5 133.462,-369.5 127.462,-369.5 127.462,-369.5 88.1667,-369.5 88.1667,-369.5 82.1667,-369.5 76.1667,-363.5 76.1667,-357.5 76.1667,-357.5 76.1667,-327.5 76.1667,-327.5 76.1667,-321.5 82.1667,-315.5 88.1667,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"107.814\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"76.1667,-351.5 139.462,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"107.814\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"76.1667,-333.5 139.462,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"107.814\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4398314784backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4398314784backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"29.8142\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"29.8142\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y1= 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4398315648backward&#45;&gt;4398314784backward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4398315648backward&#45;&gt;4398314784backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M87.9297,-315.242C73.3057,-295.931 53.9177,-270.329 41.6514,-254.131\"/>\n",
+       "</g>\n",
+       "<!-- 4398314832backward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4398314832backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M89.1667,-210.5C89.1667,-210.5 128.462,-210.5 128.462,-210.5 134.462,-210.5 140.462,-216.5 140.462,-222.5 140.462,-222.5 140.462,-252.5 140.462,-252.5 140.462,-258.5 134.462,-264.5 128.462,-264.5 128.462,-264.5 89.1667,-264.5 89.1667,-264.5 83.1667,-264.5 77.1667,-258.5 77.1667,-252.5 77.1667,-252.5 77.1667,-222.5 77.1667,-222.5 77.1667,-216.5 83.1667,-210.5 89.1667,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.1667,-246.5 140.462,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.1667,-228.5 140.462,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4398315648backward&#45;&gt;4398314832backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4398315648backward&#45;&gt;4398314832backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M108.069,-315.242C108.188,-303.014 108.331,-288.264 108.46,-274.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"111.962,-274.743 108.56,-264.71 104.963,-274.675 111.962,-274.743\"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.395\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4398315264backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4398315264backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"187.814\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y2= 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4398315792backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4398315792backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M169.167,-315.5C169.167,-315.5 208.462,-315.5 208.462,-315.5 214.462,-315.5 220.462,-321.5 220.462,-327.5 220.462,-327.5 220.462,-357.5 220.462,-357.5 220.462,-363.5 214.462,-369.5 208.462,-369.5 208.462,-369.5 169.167,-369.5 169.167,-369.5 163.167,-369.5 157.167,-363.5 157.167,-357.5 157.167,-357.5 157.167,-327.5 157.167,-327.5 157.167,-321.5 163.167,-315.5 169.167,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.814\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"157.167,-351.5 220.462,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.814\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"157.167,-333.5 220.462,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.814\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4398315792backward&#45;&gt;4398315264backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4398315792backward&#45;&gt;4398315264backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M188.559,-315.242C188.378,-296.527 188.138,-271.903 187.981,-255.657\"/>\n",
+       "</g>\n",
+       "<!-- 4398314400backward -->\n",
+       "<g id=\"node11\" class=\"node\"><title>4398314400backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M247.167,-210.5C247.167,-210.5 286.462,-210.5 286.462,-210.5 292.462,-210.5 298.462,-216.5 298.462,-222.5 298.462,-222.5 298.462,-252.5 298.462,-252.5 298.462,-258.5 292.462,-264.5 286.462,-264.5 286.462,-264.5 247.167,-264.5 247.167,-264.5 241.167,-264.5 235.167,-258.5 235.167,-252.5 235.167,-252.5 235.167,-222.5 235.167,-222.5 235.167,-216.5 241.167,-210.5 247.167,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"235.167,-246.5 298.462,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"235.167,-228.5 298.462,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4398315792backward&#45;&gt;4398314400backward -->\n",
+       "<g id=\"edge12\" class=\"edge\"><title>4398315792backward&#45;&gt;4398314400backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M208.699,-315.242C218.417,-302.409 230.239,-286.798 240.677,-273.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"243.719,-274.795 246.966,-264.71 238.139,-270.569 243.719,-274.795\"/>\n",
+       "<text text-anchor=\"middle\" x=\"246.395\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4398315840backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4398315840backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M132.167,-420.5C132.167,-420.5 171.462,-420.5 171.462,-420.5 177.462,-420.5 183.462,-426.5 183.462,-432.5 183.462,-432.5 183.462,-462.5 183.462,-462.5 183.462,-468.5 177.462,-474.5 171.462,-474.5 171.462,-474.5 132.167,-474.5 132.167,-474.5 126.167,-474.5 120.167,-468.5 120.167,-462.5 120.167,-462.5 120.167,-432.5 120.167,-432.5 120.167,-426.5 126.167,-420.5 132.167,-420.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"151.814\" y=\"-462.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"120.167,-456.5 183.462,-456.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"151.814\" y=\"-444.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"120.167,-438.5 183.462,-438.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"151.814\" y=\"-426.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4398315840backward&#45;&gt;4398315648backward -->\n",
+       "<g id=\"edge13\" class=\"edge\"><title>4398315840backward&#45;&gt;4398315648backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M140.597,-420.242C135.27,-407.772 128.823,-392.679 123.06,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"126.158,-377.531 119.011,-369.71 119.72,-380.281 126.158,-377.531\"/>\n",
+       "<text text-anchor=\"middle\" x=\"146.814\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4398315840backward&#45;&gt;4398315792backward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4398315840backward&#45;&gt;4398315792backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M161.247,-420.242C165.726,-407.772 171.148,-392.679 175.994,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"179.312,-380.304 179.399,-369.71 172.724,-377.938 179.312,-380.304\"/>\n",
+       "<text text-anchor=\"middle\" x=\"185.814\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4398314928backward -->\n",
+       "<g id=\"node12\" class=\"node\"><title>4398314928backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-105.5 156.167,-159.5 219.462,-159.5 219.462,-105.5 156.167,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;5.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-141.5 219.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-123.5 219.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4398314832backward&#45;&gt;4398314928backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4398314832backward&#45;&gt;4398314928backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M128.954,-210.242C138.796,-197.409 150.77,-181.798 161.342,-168.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"164.403,-169.775 167.711,-159.71 158.848,-165.515 164.403,-169.775\"/>\n",
+       "<text text-anchor=\"middle\" x=\"168.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4398315456backward -->\n",
+       "<g id=\"node13\" class=\"node\"><title>4398315456backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M87.1667,-105.5C87.1667,-105.5 126.462,-105.5 126.462,-105.5 132.462,-105.5 138.462,-111.5 138.462,-117.5 138.462,-117.5 138.462,-147.5 138.462,-147.5 138.462,-153.5 132.462,-159.5 126.462,-159.5 126.462,-159.5 87.1667,-159.5 87.1667,-159.5 81.1667,-159.5 75.1667,-153.5 75.1667,-147.5 75.1667,-147.5 75.1667,-117.5 75.1667,-117.5 75.1667,-111.5 81.1667,-105.5 87.1667,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"75.1667,-141.5 138.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"75.1667,-123.5 138.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4398314832backward&#45;&gt;4398315456backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4398314832backward&#45;&gt;4398315456backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M108.304,-210.242C108.067,-198.014 107.78,-183.264 107.522,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"111.017,-169.64 107.323,-159.71 104.018,-169.776 111.017,-169.64\"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4398315360backward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4398315360backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"106.814\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x1= 1.50</text>\n",
+       "</g>\n",
+       "<!-- 4398315888backward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4398315888backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M249.167,-105.5C249.167,-105.5 288.462,-105.5 288.462,-105.5 294.462,-105.5 300.462,-111.5 300.462,-117.5 300.462,-117.5 300.462,-147.5 300.462,-147.5 300.462,-153.5 294.462,-159.5 288.462,-159.5 288.462,-159.5 249.167,-159.5 249.167,-159.5 243.167,-159.5 237.167,-153.5 237.167,-147.5 237.167,-147.5 237.167,-117.5 237.167,-117.5 237.167,-111.5 243.167,-105.5 249.167,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"237.167,-141.5 300.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"237.167,-123.5 300.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4398315888backward&#45;&gt;4398315552backward -->\n",
+       "<g id=\"edge14\" class=\"edge\"><title>4398315888backward&#45;&gt;4398315552backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M248.165,-105.242C237.978,-92.2881 225.564,-76.503 214.65,-62.6249\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"217.359,-60.4068 208.426,-54.7098 211.856,-64.7339 217.359,-60.4068\"/>\n",
+       "<text text-anchor=\"middle\" x=\"247.395\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;8.00</text>\n",
+       "</g>\n",
+       "<!-- 4398315408backward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4398315408backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"268.814\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x2= 2.00</text>\n",
+       "</g>\n",
+       "<!-- 4398315888backward&#45;&gt;4398315408backward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4398315888backward&#45;&gt;4398315408backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M268.814,-105.242C268.814,-86.5272 268.814,-61.9035 268.814,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4398314400backward&#45;&gt;4398315888backward -->\n",
+       "<g id=\"edge11\" class=\"edge\"><title>4398314400backward&#45;&gt;4398315888backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M267.324,-210.242C267.561,-198.014 267.848,-183.264 268.106,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"271.61,-169.776 268.305,-159.71 264.612,-169.64 271.61,-169.776\"/>\n",
+       "<text text-anchor=\"middle\" x=\"282.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4398314400backward&#45;&gt;4398314928backward -->\n",
+       "<g id=\"edge10\" class=\"edge\"><title>4398314400backward&#45;&gt;4398314928backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M246.675,-210.242C236.832,-197.409 224.858,-181.798 214.286,-168.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"216.78,-165.515 207.917,-159.71 211.226,-169.775 216.78,-165.515\"/>\n",
+       "<text text-anchor=\"middle\" x=\"246.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4398315456backward&#45;&gt;4398315552backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4398315456backward&#45;&gt;4398315552backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M127.463,-105.242C137.651,-92.2881 150.064,-76.503 160.978,-62.6249\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"163.772,-64.7339 167.203,-54.7098 158.27,-60.4068 163.772,-64.7339\"/>\n",
+       "<text text-anchor=\"middle\" x=\"167.395\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.50</text>\n",
+       "</g>\n",
+       "<!-- 4398315456backward&#45;&gt;4398315360backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4398315456backward&#45;&gt;4398315360backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M106.814,-105.242C106.814,-86.5272 106.814,-61.9035 106.814,-45.6575\"/>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x10628f310>"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x1 = Scalar(1.5, label='x1', requires_grad=False)\n",
+    "y1 = Scalar(1.0, label='y1', requires_grad=False)\n",
+    "x2 = Scalar(2.0, label='x2', requires_grad=False)\n",
+    "y2 = Scalar(4.0, label='y2', requires_grad=False)\n",
+    "# 反向传播\n",
+    "model = Linear()\n",
+    "loss = mse([model.error(x1, y1), model.error(x2, y2)])\n",
+    "loss.value = 0\n",
+    "loss.backward()\n",
+    "draw_graph(loss, 'backward')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

ch07-autograd/gpu.ipynb

@@ -0,0 +1,255 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "id": "O2CjOgU3Ryet"
+   },
+   "outputs": [],
+   "source": [
+    "import torch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "1yv3LclNR3FH",
+    "outputId": "1b2b1fec-9f85-4a60-b3a0-d0ac93ef93be"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 检查是否有GPU\n",
+    "torch.cuda.is_available()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "C2lR7FW1R50T",
+    "outputId": "a1e8c64c-f84a-4039-a219-d9c36367036d"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# GPU的个数\n",
+    "torch.cuda.device_count()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "uST41cZWCFG4",
+    "outputId": "2d6789c8-281b-415a-d4b6-6b0eb70f0738"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "False\n",
+      "True\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 默认情况下，创建的张量存放在内存，使用CPU进行计算\n",
+    "x = torch.randn(2, 3)\n",
+    "print(x.is_cuda)\n",
+    "# 可以使用张量提供的函数，将数据移到GPU\n",
+    "# 当有n个GPU时，相应的设备id是cuda:0, ... cude:n-1\n",
+    "print(x.to('cuda:0').is_cuda)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "S1WER-1XSQP8",
+    "outputId": "442f1e34-5129-414f-c6b6-e218a5ea0da9"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "True\n",
+      "False\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 在创建张量时，通过指定device来将张量移到GPU\n",
+    "y = torch.randn(2, 3, device='cuda:0')\n",
+    "print(y.is_cuda)\n",
+    "print(y.to('cpu').is_cuda)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 214
+    },
+    "id": "Oy9Ov40TSTUN",
+    "outputId": "823b4cba-e105-4e95-d716-bf568bed34a9"
+   },
+   "outputs": [
+    {
+     "ename": "RuntimeError",
+     "evalue": "ignored",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-6-175ea4fe0736>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# 不支持跨计算核心运算\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mRuntimeError\u001b[0m: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu!"
+     ]
+    }
+   ],
+   "source": [
+    "# 不支持跨计算核心运算\n",
+    "x + y"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "id": "LdEB0ngaSY5t"
+   },
+   "outputs": [],
+   "source": [
+    "import time\n",
+    "\n",
+    "def measure_compute_time(device_id, dimension):\n",
+    "    \"\"\"\n",
+    "    展示GPU在串行计算和并行计算时的表现\n",
+    "    当dimension比较大时，GPU主要进行并行计算\n",
+    "    \"\"\"\n",
+    "    start_time = time.time()\n",
+    "    x = torch.ones((dimension, dimension), device=device_id)\n",
+    "    # for循环时串行计算\n",
+    "    for _ in range(10 ** 5):\n",
+    "        # x是矩阵，x + x是并行计算\n",
+    "        x + x\n",
+    "    elapsed_time = time.time() - start_time\n",
+    "    print(f'For device {device_id}, compute time = {elapsed_time: .4f}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "OP8VdqhmE_qa",
+    "outputId": "118345d3-aa01-4942-a9bf-eadc01fa1c8d"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "For device cpu, compute time =  0.5259\n",
+      "For device cuda:0, compute time =  1.0495\n"
+     ]
+    }
+   ],
+   "source": [
+    "measure_compute_time('cpu', 1)\n",
+    "measure_compute_time('cuda:0', 1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "-6glK0TfFyTO",
+    "outputId": "4a5cec04-00fb-4b63-b4f3-ad89b3393e14"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "For device cpu, compute time =  27.2803\n",
+      "For device cuda:0, compute time =  1.1041\n"
+     ]
+    }
+   ],
+   "source": [
+    "measure_compute_time('cpu', 1000)\n",
+    "measure_compute_time('cuda:0', 1000)"
+   ]
+  }
+ ],
+ "metadata": {
+  "accelerator": "GPU",
+  "colab": {
+   "gpuType": "V100",
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

ch07-autograd/gradient_accumulation.ipynb

@@ -0,0 +1,666 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from utils import Scalar, draw_graph\n",
+    "from linear_model import Linear, mse"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"308pt\" height=\"483pt\"\n",
+       " viewBox=\"0.00 0.00 308.46 483.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 479)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-479 304.462,-479 304.462,4 -4,4\"/>\n",
+       "<!-- 4390527520backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4390527520backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"187.814\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y1= 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4390528048backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4390528048backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-105.5 156.167,-159.5 219.462,-159.5 219.462,-105.5 156.167,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;5.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-141.5 219.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-123.5 219.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4390528096backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4390528096backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M247.167,-210.5C247.167,-210.5 286.462,-210.5 286.462,-210.5 292.462,-210.5 298.462,-216.5 298.462,-222.5 298.462,-222.5 298.462,-252.5 298.462,-252.5 298.462,-258.5 292.462,-264.5 286.462,-264.5 286.462,-264.5 247.167,-264.5 247.167,-264.5 241.167,-264.5 235.167,-258.5 235.167,-252.5 235.167,-252.5 235.167,-222.5 235.167,-222.5 235.167,-216.5 241.167,-210.5 247.167,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"235.167,-246.5 298.462,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"235.167,-228.5 298.462,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4390528096backward&#45;&gt;4390528048backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4390528096backward&#45;&gt;4390528048backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M246.675,-210.242C236.832,-197.409 224.858,-181.798 214.286,-168.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"216.78,-165.515 207.917,-159.71 211.226,-169.775 216.78,-165.515\"/>\n",
+       "<text text-anchor=\"middle\" x=\"246.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4390527856backward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4390527856backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M249.167,-105.5C249.167,-105.5 288.462,-105.5 288.462,-105.5 294.462,-105.5 300.462,-111.5 300.462,-117.5 300.462,-117.5 300.462,-147.5 300.462,-147.5 300.462,-153.5 294.462,-159.5 288.462,-159.5 288.462,-159.5 249.167,-159.5 249.167,-159.5 243.167,-159.5 237.167,-153.5 237.167,-147.5 237.167,-147.5 237.167,-117.5 237.167,-117.5 237.167,-111.5 243.167,-105.5 249.167,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"237.167,-141.5 300.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"237.167,-123.5 300.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4390528096backward&#45;&gt;4390527856backward -->\n",
+       "<g id=\"edge14\" class=\"edge\"><title>4390528096backward&#45;&gt;4390527856backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M267.324,-210.242C267.561,-198.014 267.848,-183.264 268.106,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"271.61,-169.776 268.305,-159.71 264.612,-169.64 271.61,-169.776\"/>\n",
+       "<text text-anchor=\"middle\" x=\"282.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4390527616backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4390527616backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"29.8142\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"29.8142\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y2= 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4390527664backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4390527664backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"268.814\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x1= 1.50</text>\n",
+       "</g>\n",
+       "<!-- 4390527712backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4390527712backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"106.814\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x2= 2.00</text>\n",
+       "</g>\n",
+       "<!-- 4390528240backward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4390528240backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M89.1667,-210.5C89.1667,-210.5 128.462,-210.5 128.462,-210.5 134.462,-210.5 140.462,-216.5 140.462,-222.5 140.462,-222.5 140.462,-252.5 140.462,-252.5 140.462,-258.5 134.462,-264.5 128.462,-264.5 128.462,-264.5 89.1667,-264.5 89.1667,-264.5 83.1667,-264.5 77.1667,-258.5 77.1667,-252.5 77.1667,-252.5 77.1667,-222.5 77.1667,-222.5 77.1667,-216.5 83.1667,-210.5 89.1667,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.1667,-246.5 140.462,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.1667,-228.5 140.462,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4390528240backward&#45;&gt;4390528048backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4390528240backward&#45;&gt;4390528048backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M128.954,-210.242C138.796,-197.409 150.77,-181.798 161.342,-168.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"164.403,-169.775 167.711,-159.71 158.848,-165.515 164.403,-169.775\"/>\n",
+       "<text text-anchor=\"middle\" x=\"168.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4390527376backward -->\n",
+       "<g id=\"node11\" class=\"node\"><title>4390527376backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M87.1667,-105.5C87.1667,-105.5 126.462,-105.5 126.462,-105.5 132.462,-105.5 138.462,-111.5 138.462,-117.5 138.462,-117.5 138.462,-147.5 138.462,-147.5 138.462,-153.5 132.462,-159.5 126.462,-159.5 126.462,-159.5 87.1667,-159.5 87.1667,-159.5 81.1667,-159.5 75.1667,-153.5 75.1667,-147.5 75.1667,-147.5 75.1667,-117.5 75.1667,-117.5 75.1667,-111.5 81.1667,-105.5 87.1667,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"75.1667,-141.5 138.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"75.1667,-123.5 138.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4390528240backward&#45;&gt;4390527376backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4390528240backward&#45;&gt;4390527376backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M108.304,-210.242C108.067,-198.014 107.78,-183.264 107.522,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"111.017,-169.64 107.323,-159.71 104.018,-169.776 111.017,-169.64\"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4390528288backward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4390528288backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M87.1667,-315.5C87.1667,-315.5 126.462,-315.5 126.462,-315.5 132.462,-315.5 138.462,-321.5 138.462,-327.5 138.462,-327.5 138.462,-357.5 138.462,-357.5 138.462,-363.5 132.462,-369.5 126.462,-369.5 126.462,-369.5 87.1667,-369.5 87.1667,-369.5 81.1667,-369.5 75.1667,-363.5 75.1667,-357.5 75.1667,-357.5 75.1667,-327.5 75.1667,-327.5 75.1667,-321.5 81.1667,-315.5 87.1667,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"75.1667,-351.5 138.462,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"75.1667,-333.5 138.462,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4390528288backward&#45;&gt;4390527616backward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4390528288backward&#45;&gt;4390527616backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M87.1846,-315.242C72.7481,-295.931 53.6087,-270.329 41.4997,-254.131\"/>\n",
+       "</g>\n",
+       "<!-- 4390528288backward&#45;&gt;4390528240backward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4390528288backward&#45;&gt;4390528240backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M107.324,-315.242C107.561,-303.014 107.848,-288.264 108.106,-274.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"111.61,-274.776 108.305,-264.71 104.612,-274.64 111.61,-274.776\"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.395\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4390527856backward&#45;&gt;4390527664backward -->\n",
+       "<g id=\"edge11\" class=\"edge\"><title>4390527856backward&#45;&gt;4390527664backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M268.814,-105.242C268.814,-86.5272 268.814,-61.9035 268.814,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4390527904backward -->\n",
+       "<g id=\"node12\" class=\"node\"><title>4390527904backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-0.5 156.167,-54.5 219.462,-54.5 219.462,-0.5 156.167,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;9.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-36.5 219.462,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-18.5 219.462,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4390527856backward&#45;&gt;4390527904backward -->\n",
+       "<g id=\"edge12\" class=\"edge\"><title>4390527856backward&#45;&gt;4390527904backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M248.165,-105.242C237.978,-92.2881 225.564,-76.503 214.65,-62.6249\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"217.359,-60.4068 208.426,-54.7098 211.856,-64.7339 217.359,-60.4068\"/>\n",
+       "<text text-anchor=\"middle\" x=\"247.395\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.50</text>\n",
+       "</g>\n",
+       "<!-- 4390528384backward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4390528384backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M127.167,-420.5C127.167,-420.5 166.462,-420.5 166.462,-420.5 172.462,-420.5 178.462,-426.5 178.462,-432.5 178.462,-432.5 178.462,-462.5 178.462,-462.5 178.462,-468.5 172.462,-474.5 166.462,-474.5 166.462,-474.5 127.167,-474.5 127.167,-474.5 121.167,-474.5 115.167,-468.5 115.167,-462.5 115.167,-462.5 115.167,-432.5 115.167,-432.5 115.167,-426.5 121.167,-420.5 127.167,-420.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"146.814\" y=\"-462.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"115.167,-456.5 178.462,-456.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"146.814\" y=\"-444.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 8.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"115.167,-438.5 178.462,-438.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"146.814\" y=\"-426.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4390528384backward&#45;&gt;4390528288backward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4390528384backward&#45;&gt;4390528288backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M136.617,-420.242C131.774,-407.772 125.913,-392.679 120.674,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"123.876,-377.765 116.993,-369.71 117.35,-380.299 123.876,-377.765\"/>\n",
+       "<text text-anchor=\"middle\" x=\"143.814\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4390527952backward -->\n",
+       "<g id=\"node13\" class=\"node\"><title>4390527952backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M168.167,-315.5C168.167,-315.5 207.462,-315.5 207.462,-315.5 213.462,-315.5 219.462,-321.5 219.462,-327.5 219.462,-327.5 219.462,-357.5 219.462,-357.5 219.462,-363.5 213.462,-369.5 207.462,-369.5 207.462,-369.5 168.167,-369.5 168.167,-369.5 162.167,-369.5 156.167,-363.5 156.167,-357.5 156.167,-357.5 156.167,-327.5 156.167,-327.5 156.167,-321.5 162.167,-315.5 168.167,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"156.167,-351.5 219.462,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"156.167,-333.5 219.462,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4390528384backward&#45;&gt;4390527952backward -->\n",
+       "<g id=\"edge10\" class=\"edge\"><title>4390528384backward&#45;&gt;4390527952backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M157.266,-420.242C162.23,-407.772 168.238,-392.679 173.608,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"176.935,-380.295 177.381,-369.71 170.431,-377.706 176.935,-380.295\"/>\n",
+       "<text text-anchor=\"middle\" x=\"182.814\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4390527376backward&#45;&gt;4390527712backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4390527376backward&#45;&gt;4390527712backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M106.814,-105.242C106.814,-86.5272 106.814,-61.9035 106.814,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4390527376backward&#45;&gt;4390527904backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4390527376backward&#45;&gt;4390527904backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M127.463,-105.242C137.651,-92.2881 150.064,-76.503 160.978,-62.6249\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"163.772,-64.7339 167.203,-54.7098 158.27,-60.4068 163.772,-64.7339\"/>\n",
+       "<text text-anchor=\"middle\" x=\"167.395\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;8.00</text>\n",
+       "</g>\n",
+       "<!-- 4390527952backward&#45;&gt;4390527520backward -->\n",
+       "<g id=\"edge13\" class=\"edge\"><title>4390527952backward&#45;&gt;4390527520backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M187.814,-315.242C187.814,-296.527 187.814,-271.903 187.814,-255.657\"/>\n",
+       "</g>\n",
+       "<!-- 4390527952backward&#45;&gt;4390528096backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4390527952backward&#45;&gt;4390528096backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M207.954,-315.242C217.796,-302.409 229.77,-286.798 240.342,-273.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"243.403,-274.775 246.711,-264.71 237.848,-270.515 243.403,-274.775\"/>\n",
+       "<text text-anchor=\"middle\" x=\"246.395\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x105b226a0>"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x1 = Scalar(1.5, label='x1', requires_grad=False)\n",
+    "y1 = Scalar(1.0, label='y1', requires_grad=False)\n",
+    "x2 = Scalar(2.0, label='x2', requires_grad=False)\n",
+    "y2 = Scalar(4.0, label='y2', requires_grad=False)\n",
+    "# 反向传播\n",
+    "model = Linear()\n",
+    "loss = mse([model.error(x1, y1), model.error(x2, y2)])\n",
+    "loss.backward()\n",
+    "draw_graph(loss, 'backward')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"252pt\" height=\"588pt\"\n",
+       " viewBox=\"0.00 0.00 251.56 588.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 584)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-584 247.557,-584 247.557,4 -4,4\"/>\n",
+       "<!-- 4390527520backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4390527520backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"105.909\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"105.909\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y1= 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4380367456backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4380367456backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"35.909\" cy=\"-447.5\" rx=\"35.8183\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"35.909\" y=\"-444.5\" font-family=\"Times,serif\" font-size=\"10.00\">input= 0.50</text>\n",
+       "</g>\n",
+       "<!-- 4390527664backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4390527664backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"91.909\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"91.909\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x1= 1.50</text>\n",
+       "</g>\n",
+       "<!-- 4390456000backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4390456000backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M111.262,-105.5C111.262,-105.5 150.557,-105.5 150.557,-105.5 156.557,-105.5 162.557,-111.5 162.557,-117.5 162.557,-117.5 162.557,-147.5 162.557,-147.5 162.557,-153.5 156.557,-159.5 150.557,-159.5 150.557,-159.5 111.262,-159.5 111.262,-159.5 105.262,-159.5 99.2616,-153.5 99.2616,-147.5 99.2616,-147.5 99.2616,-117.5 99.2616,-117.5 99.2616,-111.5 105.262,-105.5 111.262,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"130.909\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"99.2616,-141.5 162.557,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"130.909\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"99.2616,-123.5 162.557,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"130.909\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4390456000backward&#45;&gt;4390527664backward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4390456000backward&#45;&gt;4390527664backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M120.967,-105.242C113.835,-86.4079 104.438,-61.5896 98.2884,-45.348\"/>\n",
+       "</g>\n",
+       "<!-- 4390454752backward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4390454752backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"139.262,-0.5 139.262,-54.5 202.557,-54.5 202.557,-0.5 139.262,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"170.909\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"139.262,-36.5 202.557,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"170.909\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"139.262,-18.5 202.557,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"170.909\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4390456000backward&#45;&gt;4390454752backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4390456000backward&#45;&gt;4390454752backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M141.106,-105.242C145.949,-92.7723 151.81,-77.6793 157.049,-64.1891\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"160.373,-65.2986 160.73,-54.7098 153.848,-62.7645 160.373,-65.2986\"/>\n",
+       "<text text-anchor=\"middle\" x=\"168.49\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.50</text>\n",
+       "</g>\n",
+       "<!-- 4380367648backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4380367648backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M102.262,-420.5C102.262,-420.5 141.557,-420.5 141.557,-420.5 147.557,-420.5 153.557,-426.5 153.557,-432.5 153.557,-432.5 153.557,-462.5 153.557,-462.5 153.557,-468.5 147.557,-474.5 141.557,-474.5 141.557,-474.5 102.262,-474.5 102.262,-474.5 96.2616,-474.5 90.2616,-468.5 90.2616,-462.5 90.2616,-462.5 90.2616,-432.5 90.2616,-432.5 90.2616,-426.5 96.2616,-420.5 102.262,-420.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"121.909\" y=\"-462.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 0.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"90.2616,-456.5 153.557,-456.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"121.909\" y=\"-444.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"90.2616,-438.5 153.557,-438.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"121.909\" y=\"-426.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4380365104backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4380365104backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M102.262,-315.5C102.262,-315.5 141.557,-315.5 141.557,-315.5 147.557,-315.5 153.557,-321.5 153.557,-327.5 153.557,-327.5 153.557,-357.5 153.557,-357.5 153.557,-363.5 147.557,-369.5 141.557,-369.5 141.557,-369.5 102.262,-369.5 102.262,-369.5 96.2616,-369.5 90.2616,-363.5 90.2616,-357.5 90.2616,-357.5 90.2616,-327.5 90.2616,-327.5 90.2616,-321.5 96.2616,-315.5 102.262,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"121.909\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"90.2616,-351.5 153.557,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"121.909\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"90.2616,-333.5 153.557,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"121.909\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4380367648backward&#45;&gt;4380365104backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4380367648backward&#45;&gt;4380365104backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M121.909,-420.242C121.909,-408.014 121.909,-393.264 121.909,-379.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"125.409,-379.71 121.909,-369.71 118.409,-379.71 125.409,-379.71\"/>\n",
+       "<text text-anchor=\"middle\" x=\"135.909\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4380365104backward&#45;&gt;4390527520backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4380365104backward&#45;&gt;4390527520backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M117.83,-315.242C114.923,-296.527 111.098,-271.903 108.574,-255.657\"/>\n",
+       "</g>\n",
+       "<!-- 4390457296backward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4390457296backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M165.262,-210.5C165.262,-210.5 204.557,-210.5 204.557,-210.5 210.557,-210.5 216.557,-216.5 216.557,-222.5 216.557,-222.5 216.557,-252.5 216.557,-252.5 216.557,-258.5 210.557,-264.5 204.557,-264.5 204.557,-264.5 165.262,-264.5 165.262,-264.5 159.262,-264.5 153.262,-258.5 153.262,-252.5 153.262,-252.5 153.262,-222.5 153.262,-222.5 153.262,-216.5 159.262,-210.5 165.262,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"184.909\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"153.262,-246.5 216.557,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"184.909\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"153.262,-228.5 216.557,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"184.909\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4380365104backward&#45;&gt;4390457296backward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4380365104backward&#45;&gt;4390457296backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M137.97,-315.242C145.745,-302.53 155.188,-287.092 163.559,-273.405\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"166.646,-275.067 168.878,-264.71 160.674,-271.414 166.646,-275.067\"/>\n",
+       "<text text-anchor=\"middle\" x=\"172.49\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4390456624backward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4390456624backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"180.262,-105.5 180.262,-159.5 243.557,-159.5 243.557,-105.5 180.262,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"211.909\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"180.262,-141.5 243.557,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"211.909\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"180.262,-123.5 243.557,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"211.909\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4390457296backward&#45;&gt;4390456000backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4390457296backward&#45;&gt;4390456000backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M171.143,-210.242C164.542,-197.651 156.539,-182.386 149.414,-168.797\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"152.393,-166.941 144.65,-159.71 146.194,-170.192 152.393,-166.941\"/>\n",
+       "<text text-anchor=\"middle\" x=\"176.49\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4390457296backward&#45;&gt;4390456624backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4390457296backward&#45;&gt;4390456624backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M191.792,-210.242C195.029,-197.893 198.941,-182.972 202.451,-169.582\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"205.888,-170.27 205.039,-159.71 199.117,-168.495 205.888,-170.27\"/>\n",
+       "<text text-anchor=\"middle\" x=\"215.49\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4380367840backward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4380367840backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M45.2616,-525.5C45.2616,-525.5 84.5565,-525.5 84.5565,-525.5 90.5565,-525.5 96.5565,-531.5 96.5565,-537.5 96.5565,-537.5 96.5565,-567.5 96.5565,-567.5 96.5565,-573.5 90.5565,-579.5 84.5565,-579.5 84.5565,-579.5 45.2616,-579.5 45.2616,-579.5 39.2616,-579.5 33.2616,-573.5 33.2616,-567.5 33.2616,-567.5 33.2616,-537.5 33.2616,-537.5 33.2616,-531.5 39.2616,-525.5 45.2616,-525.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"64.909\" y=\"-567.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"33.2616,-561.5 96.5565,-561.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"64.909\" y=\"-549.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"33.2616,-543.5 96.5565,-543.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"64.909\" y=\"-531.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4380367840backward&#45;&gt;4380367456backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4380367840backward&#45;&gt;4380367456backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M57.5161,-525.242C52.2132,-506.408 45.2255,-481.59 40.6527,-465.348\"/>\n",
+       "</g>\n",
+       "<!-- 4380367840backward&#45;&gt;4380367648backward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4380367840backward&#45;&gt;4380367648backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M79.44,-525.242C86.4079,-512.651 94.8556,-497.386 102.376,-483.797\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"105.625,-485.154 107.405,-474.71 99.5002,-481.765 105.625,-485.154\"/>\n",
+       "<text text-anchor=\"middle\" x=\"110.909\" y=\"-495.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0.50</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x105171640>"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 梯度累积\n",
+    "model = Linear()\n",
+    "# 使用x1，y1传播一次\n",
+    "# 系数0.5是因为平均数权重调整\n",
+    "loss = 0.5 * mse([model.error(x1, y1)])\n",
+    "loss.backward()\n",
+    "draw_graph(loss, 'backward')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"267pt\" height=\"588pt\"\n",
+       " viewBox=\"0.00 0.00 266.56 588.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 584)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-584 262.557,-584 262.557,4 -4,4\"/>\n",
+       "<!-- 4390527616backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4390527616backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"107.909\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"107.909\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y2= 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4390455952backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4390455952backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M126.262,-105.5C126.262,-105.5 165.557,-105.5 165.557,-105.5 171.557,-105.5 177.557,-111.5 177.557,-117.5 177.557,-117.5 177.557,-147.5 177.557,-147.5 177.557,-153.5 171.557,-159.5 165.557,-159.5 165.557,-159.5 126.262,-159.5 126.262,-159.5 120.262,-159.5 114.262,-153.5 114.262,-147.5 114.262,-147.5 114.262,-117.5 114.262,-117.5 114.262,-111.5 120.262,-105.5 126.262,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"145.909\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"114.262,-141.5 177.557,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"145.909\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"114.262,-123.5 177.557,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"145.909\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4390527712backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4390527712backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"106.909\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.909\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x2= 2.00</text>\n",
+       "</g>\n",
+       "<!-- 4390455952backward&#45;&gt;4390527712backward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4390455952backward&#45;&gt;4390527712backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M135.967,-105.242C128.835,-86.4079 119.438,-61.5896 113.288,-45.348\"/>\n",
+       "</g>\n",
+       "<!-- 4390454752backward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4390454752backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"154.262,-0.5 154.262,-54.5 217.557,-54.5 217.557,-0.5 154.262,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"185.909\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;9.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"154.262,-36.5 217.557,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"185.909\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"154.262,-18.5 217.557,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"185.909\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4390455952backward&#45;&gt;4390454752backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4390455952backward&#45;&gt;4390454752backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M156.106,-105.242C160.949,-92.7723 166.81,-77.6793 172.049,-64.1891\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"175.373,-65.2986 175.73,-54.7098 168.848,-62.7645 175.373,-65.2986\"/>\n",
+       "<text text-anchor=\"middle\" x=\"182.49\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;8.00</text>\n",
+       "</g>\n",
+       "<!-- 4390457008backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4390457008backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M167.262,-210.5C167.262,-210.5 206.557,-210.5 206.557,-210.5 212.557,-210.5 218.557,-216.5 218.557,-222.5 218.557,-222.5 218.557,-252.5 218.557,-252.5 218.557,-258.5 212.557,-264.5 206.557,-264.5 206.557,-264.5 167.262,-264.5 167.262,-264.5 161.262,-264.5 155.262,-258.5 155.262,-252.5 155.262,-252.5 155.262,-222.5 155.262,-222.5 155.262,-216.5 161.262,-210.5 167.262,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"186.909\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"155.262,-246.5 218.557,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"186.909\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"155.262,-228.5 218.557,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"186.909\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4390457008backward&#45;&gt;4390455952backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4390457008backward&#45;&gt;4390455952backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M176.457,-210.242C171.493,-197.772 165.485,-182.679 160.115,-169.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"163.292,-167.706 156.342,-159.71 156.789,-170.295 163.292,-167.706\"/>\n",
+       "<text text-anchor=\"middle\" x=\"182.49\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4390456624backward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4390456624backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"195.262,-105.5 195.262,-159.5 258.557,-159.5 258.557,-105.5 195.262,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"226.909\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;5.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"195.262,-141.5 258.557,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"226.909\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"195.262,-123.5 258.557,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"226.909\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4390457008backward&#45;&gt;4390456624backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4390457008backward&#45;&gt;4390456624backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M197.106,-210.242C201.949,-197.772 207.81,-182.679 213.049,-169.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"216.373,-170.299 216.73,-159.71 209.848,-167.765 216.373,-170.299\"/>\n",
+       "<text text-anchor=\"middle\" x=\"223.49\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4390456528backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4390456528backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M104.262,-315.5C104.262,-315.5 143.557,-315.5 143.557,-315.5 149.557,-315.5 155.557,-321.5 155.557,-327.5 155.557,-327.5 155.557,-357.5 155.557,-357.5 155.557,-363.5 149.557,-369.5 143.557,-369.5 143.557,-369.5 104.262,-369.5 104.262,-369.5 98.2616,-369.5 92.2616,-363.5 92.2616,-357.5 92.2616,-357.5 92.2616,-327.5 92.2616,-327.5 92.2616,-321.5 98.2616,-315.5 104.262,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.909\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"92.2616,-351.5 155.557,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.909\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"92.2616,-333.5 155.557,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.909\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4390456528backward&#45;&gt;4390527616backward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4390456528backward&#45;&gt;4390527616backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M119.83,-315.242C116.923,-296.527 113.098,-271.903 110.574,-255.657\"/>\n",
+       "</g>\n",
+       "<!-- 4390456528backward&#45;&gt;4390457008backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4390456528backward&#45;&gt;4390457008backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M139.97,-315.242C147.745,-302.53 157.188,-287.092 165.559,-273.405\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"168.646,-275.067 170.878,-264.71 162.674,-271.414 168.646,-275.067\"/>\n",
+       "<text text-anchor=\"middle\" x=\"173.49\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4390457104backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4390457104backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M46.2616,-525.5C46.2616,-525.5 85.5565,-525.5 85.5565,-525.5 91.5565,-525.5 97.5565,-531.5 97.5565,-537.5 97.5565,-537.5 97.5565,-567.5 97.5565,-567.5 97.5565,-573.5 91.5565,-579.5 85.5565,-579.5 85.5565,-579.5 46.2616,-579.5 46.2616,-579.5 40.2616,-579.5 34.2616,-573.5 34.2616,-567.5 34.2616,-567.5 34.2616,-537.5 34.2616,-537.5 34.2616,-531.5 40.2616,-525.5 46.2616,-525.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"65.909\" y=\"-567.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"34.2616,-561.5 97.5565,-561.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"65.909\" y=\"-549.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 8.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"34.2616,-543.5 97.5565,-543.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"65.909\" y=\"-531.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4390456720backward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4390456720backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"35.909\" cy=\"-447.5\" rx=\"35.8183\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"35.909\" y=\"-444.5\" font-family=\"Times,serif\" font-size=\"10.00\">input= 0.50</text>\n",
+       "</g>\n",
+       "<!-- 4390457104backward&#45;&gt;4390456720backward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4390457104backward&#45;&gt;4390456720backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M58.2612,-525.242C52.7754,-506.408 45.5468,-481.59 40.8162,-465.348\"/>\n",
+       "</g>\n",
+       "<!-- 4390456768backward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4390456768backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M101.762,-420.5C101.762,-420.5 146.057,-420.5 146.057,-420.5 152.057,-420.5 158.057,-426.5 158.057,-432.5 158.057,-432.5 158.057,-462.5 158.057,-462.5 158.057,-468.5 152.057,-474.5 146.057,-474.5 146.057,-474.5 101.762,-474.5 101.762,-474.5 95.7616,-474.5 89.7616,-468.5 89.7616,-462.5 89.7616,-462.5 89.7616,-432.5 89.7616,-432.5 89.7616,-426.5 95.7616,-420.5 101.762,-420.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.909\" y=\"-462.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 0.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"89.7616,-456.5 158.057,-456.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.909\" y=\"-444.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 16.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"89.7616,-438.5 158.057,-438.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.909\" y=\"-426.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4390457104backward&#45;&gt;4390456768backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4390457104backward&#45;&gt;4390456768backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M80.695,-525.242C87.785,-512.651 96.381,-497.386 104.033,-483.797\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"107.293,-485.141 109.15,-474.71 101.194,-481.706 107.293,-485.141\"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.909\" y=\"-495.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0.50</text>\n",
+       "</g>\n",
+       "<!-- 4390456768backward&#45;&gt;4390456528backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4390456768backward&#45;&gt;4390456528backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M123.909,-420.242C123.909,-408.014 123.909,-393.264 123.909,-379.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"127.409,-379.71 123.909,-369.71 120.409,-379.71 127.409,-379.71\"/>\n",
+       "<text text-anchor=\"middle\" x=\"137.909\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.00</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x105b108e0>"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 使用x2，y2传播一次\n",
+    "loss = 0.5 * mse([model.error(x2, y2)])\n",
+    "loss.backward()\n",
+    "draw_graph(loss, 'backward')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "\n",
+    "# 固定随机种子，使得运行结果可以稳定复现\n",
+    "torch.manual_seed(1024)\n",
+    "# 产生训练用的数据\n",
+    "x_origin = torch.linspace(100, 300, 200)\n",
+    "# 将变量X归一化，否则梯度下降法很容易不稳定\n",
+    "x = (x_origin - torch.mean(x_origin)) / torch.std(x_origin)\n",
+    "epsilon = torch.randn(x.shape)\n",
+    "y = 10 * x + 5 + epsilon"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 4, Result: y = 3.12 * x + -1.99\n",
+      "Step 8, Result: y = 3.48 * x + -2.28\n",
+      "Step 12, Result: y = 3.22 * x + -1.97\n",
+      "Step 16, Result: y = 2.85 * x + -1.22\n",
+      "Step 20, Result: y = 2.68 * x + -0.23\n",
+      "Step 24, Result: y = 2.92 * x + 1.08\n",
+      "Step 28, Result: y = 3.74 * x + 2.61\n",
+      "Step 32, Result: y = 5.07 * x + 4.15\n",
+      "Step 36, Result: y = 6.73 * x + 5.52\n",
+      "Step 40, Result: y = 8.22 * x + 6.48\n",
+      "Step 44, Result: y = 9.36 * x + 5.75\n",
+      "Step 48, Result: y = 9.75 * x + 5.42\n",
+      "Step 52, Result: y = 9.88 * x + 5.28\n",
+      "Step 56, Result: y = 9.89 * x + 5.26\n",
+      "Step 60, Result: y = 9.89 * x + 5.20\n",
+      "Step 64, Result: y = 9.88 * x + 5.18\n",
+      "Step 68, Result: y = 9.88 * x + 5.17\n",
+      "Step 72, Result: y = 9.84 * x + 5.14\n",
+      "Step 76, Result: y = 9.86 * x + 5.15\n",
+      "Step 80, Result: y = 9.94 * x + 5.21\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 生成模型\n",
+    "model = Linear()\n",
+    "# 定义每批次用到的数据量\n",
+    "batch_size = 20\n",
+    "# 定义每批次梯度累积的次数\n",
+    "gradient_accumulation_iter = 4\n",
+    "micro_size = int(batch_size / gradient_accumulation_iter)\n",
+    "learning_rate = 0.1\n",
+    "\n",
+    "for t in range(20 * gradient_accumulation_iter):\n",
+    "    # 选取当前批次的数据，用于训练模型\n",
+    "    ix = (t * micro_size) % len(x)\n",
+    "    xx = x[ix: ix + micro_size]\n",
+    "    yy = y[ix: ix + micro_size]\n",
+    "    # 计算当前批次数据的损失\n",
+    "    loss = mse([model.error(_x, _y) for _x, _y in zip(xx, yy)])\n",
+    "    loss *= 1 / gradient_accumulation_iter\n",
+    "    # 计算损失函数的梯度\n",
+    "    loss.backward()\n",
+    "    if (t + 1) % gradient_accumulation_iter == 0:\n",
+    "        # 迭代更新模型参数的估计值\n",
+    "        model.a -= learning_rate * model.a.grad\n",
+    "        model.b -= learning_rate * model.b.grad\n",
+    "        # 将使用完的梯度清零\n",
+    "        model.a.grad = 0.0\n",
+    "        model.b.grad = 0.0\n",
+    "        print(f'Step {t + 1}, Result: {model.string()}')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

ch07-autograd/linear_model.py

@@ -0,0 +1,52 @@
+# -*- coding: UTF-8 -*-
+"""
+此脚本用于定义线性回归模型
+"""
+
+
+from utils import Scalar
+
+
+def mse(errors):
+    """
+    计算均方误差
+    """
+    n = len(errors)
+    wrt = {}
+    value = 0.0
+    requires_grad = False
+    for item in errors:
+        value += item.value ** 2 / n
+        wrt[item] = 2 / n * item.value
+        requires_grad = requires_grad or item.requires_grad
+    output = Scalar(value, errors, 'mse')
+    output.requires_grad=requires_grad
+    output.grad_wrt = wrt
+    return output
+
+
+class Linear:
+    def __init__(self):
+        """
+        定义线性回归模型的参数：a, b
+        """
+        self.a = Scalar(0.0, label='a')
+        self.b = Scalar(0.0, label='b')
+
+    def forward(self, x):
+        """
+        根据当前的参数估计值，得到模型的预测结果
+        """
+        return self.a * x + self.b
+    
+    def error(self, x, y):
+        """
+        当前数据的模型误差
+        """
+        return y - self.forward(x)
+
+    def string(self):
+        """
+        输出当前模型的结果
+        """
+        return f'y = {self.a.value:.2f} * x + {self.b.value:.2f}'

ch07-autograd/optim_process.ipynb

@@ -0,0 +1,752 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from utils import Scalar, draw_graph\n",
+    "from linear_model import Linear, mse"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"309pt\" height=\"427pt\"\n",
+       " viewBox=\"0.00 0.00 309.46 427.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 423)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-423 305.462,-423 305.462,4 -4,4\"/>\n",
+       "<!-- 4520854528forward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4520854528forward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M174,-91.5C174,-91.5 213.295,-91.5 213.295,-91.5 219.295,-91.5 225.295,-97.5 225.295,-103.5 225.295,-103.5 225.295,-133.5 225.295,-133.5 225.295,-139.5 219.295,-145.5 213.295,-145.5 213.295,-145.5 174,-145.5 174,-145.5 168,-145.5 162,-139.5 162,-133.5 162,-133.5 162,-103.5 162,-103.5 162,-97.5 168,-91.5 174,-91.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"193.647\" y=\"-133.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"162,-127.5 225.295,-127.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"193.647\" y=\"-115.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"162,-109.5 225.295,-109.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"193.647\" y=\"-97.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4520854576forward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4520854576forward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M173,-182.5C173,-182.5 212.295,-182.5 212.295,-182.5 218.295,-182.5 224.295,-188.5 224.295,-194.5 224.295,-194.5 224.295,-224.5 224.295,-224.5 224.295,-230.5 218.295,-236.5 212.295,-236.5 212.295,-236.5 173,-236.5 173,-236.5 167,-236.5 161,-230.5 161,-224.5 161,-224.5 161,-194.5 161,-194.5 161,-188.5 167,-182.5 173,-182.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"192.647\" y=\"-224.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"161,-218.5 224.295,-218.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"192.647\" y=\"-206.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"161,-200.5 224.295,-200.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"192.647\" y=\"-188.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4520854528forward&#45;&gt;4520854576forward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4520854528forward&#45;&gt;4520854576forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M193.353,-145.722C193.259,-154.051 193.154,-163.414 193.054,-172.316\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"189.553,-172.427 192.94,-182.466 196.552,-172.506 189.553,-172.427\"/>\n",
+       "</g>\n",
+       "<!-- 4520854624forward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4520854624forward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M173,-273.5C173,-273.5 212.295,-273.5 212.295,-273.5 218.295,-273.5 224.295,-279.5 224.295,-285.5 224.295,-285.5 224.295,-315.5 224.295,-315.5 224.295,-321.5 218.295,-327.5 212.295,-327.5 212.295,-327.5 173,-327.5 173,-327.5 167,-327.5 161,-321.5 161,-315.5 161,-315.5 161,-285.5 161,-285.5 161,-279.5 167,-273.5 173,-273.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"192.647\" y=\"-315.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"161,-309.5 224.295,-309.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"192.647\" y=\"-297.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"161,-291.5 224.295,-291.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"192.647\" y=\"-279.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4520854576forward&#45;&gt;4520854624forward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4520854576forward&#45;&gt;4520854624forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M192.647,-236.722C192.647,-245.051 192.647,-254.414 192.647,-263.316\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"189.148,-263.466 192.647,-273.466 196.148,-263.466 189.148,-263.466\"/>\n",
+       "</g>\n",
+       "<!-- 4520854096forward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4520854096forward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"81,-0.5 81,-54.5 144.295,-54.5 144.295,-0.5 81,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81,-36.5 144.295,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81,-18.5 144.295,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4520854096forward&#45;&gt;4520854528forward -->\n",
+       "<g id=\"edge11\" class=\"edge\"><title>4520854096forward&#45;&gt;4520854528forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M136.512,-54.7216C144.834,-63.8653 154.29,-74.2553 163.081,-83.9142\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"160.634,-86.426 169.953,-91.4659 165.811,-81.7144 160.634,-86.426\"/>\n",
+       "</g>\n",
+       "<!-- 4520853952forward -->\n",
+       "<g id=\"node12\" class=\"node\"><title>4520853952forward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M12,-91.5C12,-91.5 51.2949,-91.5 51.2949,-91.5 57.2949,-91.5 63.2949,-97.5 63.2949,-103.5 63.2949,-103.5 63.2949,-133.5 63.2949,-133.5 63.2949,-139.5 57.2949,-145.5 51.2949,-145.5 51.2949,-145.5 12,-145.5 12,-145.5 6,-145.5 0,-139.5 0,-133.5 0,-133.5 0,-103.5 0,-103.5 0,-97.5 6,-91.5 12,-91.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-133.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"0,-127.5 63.2949,-127.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-115.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"0,-109.5 63.2949,-109.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-97.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4520854096forward&#45;&gt;4520853952forward -->\n",
+       "<g id=\"edge13\" class=\"edge\"><title>4520854096forward&#45;&gt;4520853952forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M88.7829,-54.7216C80.461,-63.8653 71.005,-74.2553 62.2143,-83.9142\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"59.4839,-81.7144 55.3415,-91.4659 64.6608,-86.426 59.4839,-81.7144\"/>\n",
+       "</g>\n",
+       "<!-- 4520854672forward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4520854672forward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M132,-364.5C132,-364.5 171.295,-364.5 171.295,-364.5 177.295,-364.5 183.295,-370.5 183.295,-376.5 183.295,-376.5 183.295,-406.5 183.295,-406.5 183.295,-412.5 177.295,-418.5 171.295,-418.5 171.295,-418.5 132,-418.5 132,-418.5 126,-418.5 120,-412.5 120,-406.5 120,-406.5 120,-376.5 120,-376.5 120,-370.5 126,-364.5 132,-364.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"151.647\" y=\"-406.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"120,-400.5 183.295,-400.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"151.647\" y=\"-388.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 8.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"120,-382.5 183.295,-382.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"151.647\" y=\"-370.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4520854624forward&#45;&gt;4520854672forward -->\n",
+       "<g id=\"edge10\" class=\"edge\"><title>4520854624forward&#45;&gt;4520854672forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M180.568,-327.722C176.606,-336.322 172.136,-346.025 167.916,-355.185\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"164.646,-353.919 163.641,-364.466 171.004,-356.848 164.646,-353.919\"/>\n",
+       "</g>\n",
+       "<!-- 4520854144forward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4520854144forward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"81,-91.5 81,-145.5 144.295,-145.5 144.295,-91.5 81,-91.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-133.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81,-127.5 144.295,-127.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-115.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"81,-109.5 144.295,-109.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-97.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4520854144forward&#45;&gt;4520854576forward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4520854144forward&#45;&gt;4520854576forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M136.217,-145.722C144.437,-154.865 153.776,-165.255 162.458,-174.914\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"159.958,-177.369 169.246,-182.466 165.164,-172.689 159.958,-177.369\"/>\n",
+       "</g>\n",
+       "<!-- 4520854384forward -->\n",
+       "<g id=\"node11\" class=\"node\"><title>4520854384forward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M14,-182.5C14,-182.5 53.2949,-182.5 53.2949,-182.5 59.2949,-182.5 65.2949,-188.5 65.2949,-194.5 65.2949,-194.5 65.2949,-224.5 65.2949,-224.5 65.2949,-230.5 59.2949,-236.5 53.2949,-236.5 53.2949,-236.5 14,-236.5 14,-236.5 8,-236.5 2,-230.5 2,-224.5 2,-224.5 2,-194.5 2,-194.5 2,-188.5 8,-182.5 14,-182.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"33.6475\" y=\"-224.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"2,-218.5 65.2949,-218.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"33.6475\" y=\"-206.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"2,-200.5 65.2949,-200.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"33.6475\" y=\"-188.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4520854144forward&#45;&gt;4520854384forward -->\n",
+       "<g id=\"edge14\" class=\"edge\"><title>4520854144forward&#45;&gt;4520854384forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M89.3721,-145.722C81.2558,-154.865 72.0332,-165.255 63.4596,-174.914\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"60.7773,-172.664 56.7564,-182.466 66.0124,-177.311 60.7773,-172.664\"/>\n",
+       "</g>\n",
+       "<!-- 4520854192forward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4520854192forward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"193.647\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"193.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x2= 2.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854192forward&#45;&gt;4520854528forward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4520854192forward&#45;&gt;4520854528forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M193.647,-45.6601C193.647,-55.7626 193.647,-68.9344 193.647,-81.2361\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"190.148,-81.2784 193.647,-91.2784 197.148,-81.2785 190.148,-81.2784\"/>\n",
+       "</g>\n",
+       "<!-- 4520854240forward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4520854240forward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"112.647\" cy=\"-209.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"112.647\" y=\"-206.5\" font-family=\"Times,serif\" font-size=\"10.00\">y1= 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854480forward -->\n",
+       "<g id=\"node13\" class=\"node\"><title>4520854480forward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M92,-273.5C92,-273.5 131.295,-273.5 131.295,-273.5 137.295,-273.5 143.295,-279.5 143.295,-285.5 143.295,-285.5 143.295,-315.5 143.295,-315.5 143.295,-321.5 137.295,-327.5 131.295,-327.5 131.295,-327.5 92,-327.5 92,-327.5 86,-327.5 80,-321.5 80,-315.5 80,-315.5 80,-285.5 80,-285.5 80,-279.5 86,-273.5 92,-273.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"111.647\" y=\"-315.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=None</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"80,-309.5 143.295,-309.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"111.647\" y=\"-297.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"80,-291.5 143.295,-291.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"111.647\" y=\"-279.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4520854240forward&#45;&gt;4520854480forward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4520854240forward&#45;&gt;4520854480forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M112.455,-227.66C112.341,-237.763 112.193,-250.934 112.055,-263.236\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"108.555,-263.24 111.942,-273.278 115.554,-263.318 108.555,-263.24\"/>\n",
+       "</g>\n",
+       "<!-- 4520854288forward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4520854288forward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"271.647\" cy=\"-209.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"271.647\" y=\"-206.5\" font-family=\"Times,serif\" font-size=\"10.00\">y2= 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854288forward&#45;&gt;4520854624forward -->\n",
+       "<g id=\"edge12\" class=\"edge\"><title>4520854288forward&#45;&gt;4520854624forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M258.246,-225.598C248.44,-236.645 234.751,-252.067 222.412,-265.968\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"219.751,-263.694 215.73,-273.496 224.986,-268.341 219.751,-263.694\"/>\n",
+       "</g>\n",
+       "<!-- 4520853856forward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4520853856forward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" cx=\"31.6475\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x1= 1.50</text>\n",
+       "</g>\n",
+       "<!-- 4520853856forward&#45;&gt;4520853952forward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4520853856forward&#45;&gt;4520853952forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M31.6475,-45.6601C31.6475,-55.7626 31.6475,-68.9344 31.6475,-81.2361\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"28.1476,-81.2784 31.6475,-91.2784 35.1476,-81.2785 28.1476,-81.2784\"/>\n",
+       "</g>\n",
+       "<!-- 4520854384forward&#45;&gt;4520854480forward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4520854384forward&#45;&gt;4520854480forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M56.6282,-236.722C64.6418,-245.865 73.7476,-256.255 82.2127,-265.914\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"79.6078,-268.252 88.831,-273.466 84.8721,-263.638 79.6078,-268.252\"/>\n",
+       "</g>\n",
+       "<!-- 4520853952forward&#45;&gt;4520854384forward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4520853952forward&#45;&gt;4520854384forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M32.2367,-145.722C32.4239,-154.051 32.6343,-163.414 32.8343,-172.316\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"29.3385,-172.547 33.0624,-182.466 36.3368,-172.39 29.3385,-172.547\"/>\n",
+       "</g>\n",
+       "<!-- 4520854480forward&#45;&gt;4520854672forward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4520854480forward&#45;&gt;4520854672forward</title>\n",
+       "<path fill=\"none\" stroke=\"black\" d=\"M123.432,-327.722C127.298,-336.322 131.659,-346.025 135.776,-355.185\"/>\n",
+       "<polygon fill=\"black\" stroke=\"black\" points=\"132.655,-356.78 139.947,-364.466 139.04,-353.91 132.655,-356.78\"/>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x10d76c520>"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 计算图膨胀\n",
+    "model = Linear()\n",
+    "x1 = Scalar(1.5, label='x1', requires_grad=False)\n",
+    "y1 = Scalar(1.0, label='y1', requires_grad=False)\n",
+    "x2 = Scalar(2.0, label='x2', requires_grad=False)\n",
+    "y2 = Scalar(4.0, label='y2', requires_grad=False)\n",
+    "\n",
+    "loss = mse([model.error(x1, y1), model.error(x2, y2)])\n",
+    "draw_graph(loss)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"307pt\" height=\"483pt\"\n",
+       " viewBox=\"0.00 0.00 307.29 483.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 479)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-479 303.295,-479 303.295,4 -4,4\"/>\n",
+       "<!-- 4520854528backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4520854528backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M28,-105.5C28,-105.5 67.2949,-105.5 67.2949,-105.5 73.2949,-105.5 79.2949,-111.5 79.2949,-117.5 79.2949,-117.5 79.2949,-147.5 79.2949,-147.5 79.2949,-153.5 73.2949,-159.5 67.2949,-159.5 67.2949,-159.5 28,-159.5 28,-159.5 22,-159.5 16,-153.5 16,-147.5 16,-147.5 16,-117.5 16,-117.5 16,-111.5 22,-105.5 28,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"47.6475\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"16,-141.5 79.2949,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"47.6475\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"16,-123.5 79.2949,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"47.6475\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4520854096backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4520854096backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"127,-0.5 127,-54.5 190.295,-54.5 190.295,-0.5 127,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"158.647\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;9.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"127,-36.5 190.295,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"158.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"127,-18.5 190.295,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"158.647\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4520854528backward&#45;&gt;4520854096backward -->\n",
+       "<g id=\"edge11\" class=\"edge\"><title>4520854528backward&#45;&gt;4520854096backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M75.9447,-105.242C90.1659,-92.0459 107.553,-75.912 122.709,-61.8487\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"125.452,-64.0774 130.402,-54.7098 120.691,-58.9462 125.452,-64.0774\"/>\n",
+       "<text text-anchor=\"middle\" x=\"125.229\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;8.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854192backward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4520854192backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"47.6475\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"47.6475\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x2= 2.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854528backward&#45;&gt;4520854192backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4520854528backward&#45;&gt;4520854192backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M47.6475,-105.242C47.6475,-86.5272 47.6475,-61.9035 47.6475,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4520854576backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4520854576backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M12,-210.5C12,-210.5 51.2949,-210.5 51.2949,-210.5 57.2949,-210.5 63.2949,-216.5 63.2949,-222.5 63.2949,-222.5 63.2949,-252.5 63.2949,-252.5 63.2949,-258.5 57.2949,-264.5 51.2949,-264.5 51.2949,-264.5 12,-264.5 12,-264.5 6,-264.5 0,-258.5 0,-252.5 0,-252.5 0,-222.5 0,-222.5 0,-216.5 6,-210.5 12,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"0,-246.5 63.2949,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"0,-228.5 63.2949,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4520854576backward&#45;&gt;4520854528backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4520854576backward&#45;&gt;4520854528backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M35.7263,-210.242C37.6258,-198.014 39.9171,-183.264 41.9812,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"45.4995,-170.129 43.576,-159.71 38.5824,-169.054 45.4995,-170.129\"/>\n",
+       "<text text-anchor=\"middle\" x=\"55.2285\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854144backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4520854144backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"134,-105.5 134,-159.5 197.295,-159.5 197.295,-105.5 134,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"165.647\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;5.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"134,-141.5 197.295,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"165.647\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"134,-123.5 197.295,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"165.647\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4520854576backward&#45;&gt;4520854144backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4520854576backward&#45;&gt;4520854144backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M63.4036,-212.09C82.0416,-197.764 105.766,-179.528 125.669,-164.23\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"127.859,-166.961 133.655,-158.091 123.593,-161.411 127.859,-166.961\"/>\n",
+       "<text text-anchor=\"middle\" x=\"122.229\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854624backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4520854624backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M89,-315.5C89,-315.5 128.295,-315.5 128.295,-315.5 134.295,-315.5 140.295,-321.5 140.295,-327.5 140.295,-327.5 140.295,-357.5 140.295,-357.5 140.295,-363.5 134.295,-369.5 128.295,-369.5 128.295,-369.5 89,-369.5 89,-369.5 83,-369.5 77,-363.5 77,-357.5 77,-357.5 77,-327.5 77,-327.5 77,-321.5 83,-315.5 89,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.647\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77,-351.5 140.295,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.647\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77,-333.5 140.295,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.647\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4520854624backward&#45;&gt;4520854576backward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4520854624backward&#45;&gt;4520854576backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M89.0179,-315.242C79.4242,-302.409 67.7536,-286.798 57.4496,-273.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"60.0321,-270.623 51.2412,-264.71 54.4255,-274.815 60.0321,-270.623\"/>\n",
+       "<text text-anchor=\"middle\" x=\"90.2285\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854288backward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4520854288backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"110.647\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"110.647\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y2= 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854624backward&#45;&gt;4520854288backward -->\n",
+       "<g id=\"edge12\" class=\"edge\"><title>4520854624backward&#45;&gt;4520854288backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M109.157,-315.242C109.521,-296.527 109.999,-271.903 110.314,-255.657\"/>\n",
+       "</g>\n",
+       "<!-- 4520854672backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4520854672backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M133,-420.5C133,-420.5 172.295,-420.5 172.295,-420.5 178.295,-420.5 184.295,-426.5 184.295,-432.5 184.295,-432.5 184.295,-462.5 184.295,-462.5 184.295,-468.5 178.295,-474.5 172.295,-474.5 172.295,-474.5 133,-474.5 133,-474.5 127,-474.5 121,-468.5 121,-462.5 121,-462.5 121,-432.5 121,-432.5 121,-426.5 127,-420.5 133,-420.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.647\" y=\"-462.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"121,-456.5 184.295,-456.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.647\" y=\"-444.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 8.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"121,-438.5 184.295,-438.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.647\" y=\"-426.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4520854672backward&#45;&gt;4520854624backward -->\n",
+       "<g id=\"edge10\" class=\"edge\"><title>4520854672backward&#45;&gt;4520854624backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M141.431,-420.242C136.104,-407.772 129.656,-392.679 123.893,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"126.991,-377.531 119.844,-369.71 120.554,-380.281 126.991,-377.531\"/>\n",
+       "<text text-anchor=\"middle\" x=\"147.647\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854480backward -->\n",
+       "<g id=\"node13\" class=\"node\"><title>4520854480backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M170,-315.5C170,-315.5 209.295,-315.5 209.295,-315.5 215.295,-315.5 221.295,-321.5 221.295,-327.5 221.295,-327.5 221.295,-357.5 221.295,-357.5 221.295,-363.5 215.295,-369.5 209.295,-369.5 209.295,-369.5 170,-369.5 170,-369.5 164,-369.5 158,-363.5 158,-357.5 158,-357.5 158,-327.5 158,-327.5 158,-321.5 164,-315.5 170,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.647\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"158,-351.5 221.295,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.647\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"158,-333.5 221.295,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.647\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4520854672backward&#45;&gt;4520854480backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4520854672backward&#45;&gt;4520854480backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M162.08,-420.242C166.559,-407.772 171.981,-392.679 176.827,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"180.145,-380.304 180.232,-369.71 173.558,-377.938 180.145,-380.304\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.647\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854240backward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4520854240backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"188.647\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.647\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y1= 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4520853856backward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4520853856backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"257.647\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"257.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x1= 1.50</text>\n",
+       "</g>\n",
+       "<!-- 4520854384backward -->\n",
+       "<g id=\"node11\" class=\"node\"><title>4520854384backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M248,-210.5C248,-210.5 287.295,-210.5 287.295,-210.5 293.295,-210.5 299.295,-216.5 299.295,-222.5 299.295,-222.5 299.295,-252.5 299.295,-252.5 299.295,-258.5 293.295,-264.5 287.295,-264.5 287.295,-264.5 248,-264.5 248,-264.5 242,-264.5 236,-258.5 236,-252.5 236,-252.5 236,-222.5 236,-222.5 236,-216.5 242,-210.5 248,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"267.647\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"236,-246.5 299.295,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"267.647\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"236,-228.5 299.295,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"267.647\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4520854384backward&#45;&gt;4520854144backward -->\n",
+       "<g id=\"edge14\" class=\"edge\"><title>4520854384backward&#45;&gt;4520854144backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M241.645,-210.242C228.576,-197.046 212.599,-180.912 198.672,-166.849\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"201.126,-164.353 191.603,-159.71 196.152,-169.278 201.126,-164.353\"/>\n",
+       "<text text-anchor=\"middle\" x=\"237.229\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4520853952backward -->\n",
+       "<g id=\"node12\" class=\"node\"><title>4520853952backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M238,-105.5C238,-105.5 277.295,-105.5 277.295,-105.5 283.295,-105.5 289.295,-111.5 289.295,-117.5 289.295,-117.5 289.295,-147.5 289.295,-147.5 289.295,-153.5 283.295,-159.5 277.295,-159.5 277.295,-159.5 238,-159.5 238,-159.5 232,-159.5 226,-153.5 226,-147.5 226,-147.5 226,-117.5 226,-117.5 226,-111.5 232,-105.5 238,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"257.647\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"226,-141.5 289.295,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"257.647\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"226,-123.5 289.295,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"257.647\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4520854384backward&#45;&gt;4520853952backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4520854384backward&#45;&gt;4520853952backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M265.098,-210.242C263.911,-198.014 262.479,-183.264 261.189,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"264.642,-169.325 260.192,-159.71 257.675,-170.001 264.642,-169.325\"/>\n",
+       "<text text-anchor=\"middle\" x=\"277.229\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4520853952backward&#45;&gt;4520854096backward -->\n",
+       "<g id=\"edge13\" class=\"edge\"><title>4520853952backward&#45;&gt;4520854096backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M232.409,-105.242C219.842,-92.167 204.502,-76.2078 191.074,-62.2363\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"193.293,-59.4941 183.839,-54.7098 188.246,-64.3449 193.293,-59.4941\"/>\n",
+       "<text text-anchor=\"middle\" x=\"228.229\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.50</text>\n",
+       "</g>\n",
+       "<!-- 4520853952backward&#45;&gt;4520853856backward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4520853952backward&#45;&gt;4520853856backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M257.647,-105.242C257.647,-86.5272 257.647,-61.9035 257.647,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4520854480backward&#45;&gt;4520854240backward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4520854480backward&#45;&gt;4520854240backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M189.393,-315.242C189.211,-296.527 188.972,-271.903 188.814,-255.657\"/>\n",
+       "</g>\n",
+       "<!-- 4520854480backward&#45;&gt;4520854384backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4520854480backward&#45;&gt;4520854384backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M209.532,-315.242C219.25,-302.409 231.072,-286.798 241.51,-273.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"244.552,-274.795 247.799,-264.71 238.972,-270.569 244.552,-274.795\"/>\n",
+       "<text text-anchor=\"middle\" x=\"247.229\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x10d75bf40>"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 第一次触发方向传播\n",
+    "loss.backward()\n",
+    "draw_graph(loss, 'backward')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"307pt\" height=\"483pt\"\n",
+       " viewBox=\"0.00 0.00 307.29 483.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 479)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-479 303.295,-479 303.295,4 -4,4\"/>\n",
+       "<!-- 4520854528backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4520854528backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M27,-105.5C27,-105.5 66.2949,-105.5 66.2949,-105.5 72.2949,-105.5 78.2949,-111.5 78.2949,-117.5 78.2949,-117.5 78.2949,-147.5 78.2949,-147.5 78.2949,-153.5 72.2949,-159.5 66.2949,-159.5 66.2949,-159.5 27,-159.5 27,-159.5 21,-159.5 15,-153.5 15,-147.5 15,-147.5 15,-117.5 15,-117.5 15,-111.5 21,-105.5 27,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"46.6475\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;8.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"15,-141.5 78.2949,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"46.6475\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"15,-123.5 78.2949,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"46.6475\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4520854096backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4520854096backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"124.028,-0.5 124.028,-54.5 189.267,-54.5 189.267,-0.5 124.028,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"156.647\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;19.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"124.028,-36.5 189.267,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"156.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"124.028,-18.5 189.267,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"156.647\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4520854528backward&#45;&gt;4520854096backward -->\n",
+       "<g id=\"edge11\" class=\"edge\"><title>4520854528backward&#45;&gt;4520854096backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M74.6897,-105.242C88.7829,-92.0459 106.013,-75.912 121.032,-61.8487\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"123.749,-64.0997 128.656,-54.7098 118.965,-58.99 123.749,-64.0997\"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.229\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;8.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854192backward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4520854192backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"46.6475\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"46.6475\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x2= 2.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854528backward&#45;&gt;4520854192backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4520854528backward&#45;&gt;4520854192backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M46.6475,-105.242C46.6475,-86.5272 46.6475,-61.9035 46.6475,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4520854576backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4520854576backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M12,-210.5C12,-210.5 51.2949,-210.5 51.2949,-210.5 57.2949,-210.5 63.2949,-216.5 63.2949,-222.5 63.2949,-222.5 63.2949,-252.5 63.2949,-252.5 63.2949,-258.5 57.2949,-264.5 51.2949,-264.5 51.2949,-264.5 12,-264.5 12,-264.5 6,-264.5 0,-258.5 0,-252.5 0,-252.5 0,-222.5 0,-222.5 0,-216.5 6,-210.5 12,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;8.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"0,-246.5 63.2949,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"0,-228.5 63.2949,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4520854576backward&#45;&gt;4520854528backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4520854576backward&#45;&gt;4520854528backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M35.4714,-210.242C37.2521,-198.014 39.4002,-183.264 41.3353,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"44.8528,-170.11 42.8305,-159.71 37.9258,-169.101 44.8528,-170.11\"/>\n",
+       "<text text-anchor=\"middle\" x=\"55.2285\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854144backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4520854144backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"132.028,-105.5 132.028,-159.5 197.267,-159.5 197.267,-105.5 132.028,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"164.647\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;10.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"132.028,-141.5 197.267,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"164.647\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"132.028,-123.5 197.267,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"164.647\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4520854576backward&#45;&gt;4520854144backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4520854576backward&#45;&gt;4520854144backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M63.5043,-211.829C81.5254,-197.873 104.256,-180.269 123.601,-165.287\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"125.898,-167.936 131.662,-159.045 121.612,-162.401 125.898,-167.936\"/>\n",
+       "<text text-anchor=\"middle\" x=\"121.229\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854624backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4520854624backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M89,-315.5C89,-315.5 128.295,-315.5 128.295,-315.5 134.295,-315.5 140.295,-321.5 140.295,-327.5 140.295,-327.5 140.295,-357.5 140.295,-357.5 140.295,-363.5 134.295,-369.5 128.295,-369.5 128.295,-369.5 89,-369.5 89,-369.5 83,-369.5 77,-363.5 77,-357.5 77,-357.5 77,-327.5 77,-327.5 77,-321.5 83,-315.5 89,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.647\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 8.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77,-351.5 140.295,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.647\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77,-333.5 140.295,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.647\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4520854624backward&#45;&gt;4520854576backward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4520854624backward&#45;&gt;4520854576backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M89.0179,-315.242C79.4242,-302.409 67.7536,-286.798 57.4496,-273.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"60.0321,-270.623 51.2412,-264.71 54.4255,-274.815 60.0321,-270.623\"/>\n",
+       "<text text-anchor=\"middle\" x=\"90.2285\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854288backward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4520854288backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"110.647\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"110.647\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y2= 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854624backward&#45;&gt;4520854288backward -->\n",
+       "<g id=\"edge12\" class=\"edge\"><title>4520854624backward&#45;&gt;4520854288backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M109.157,-315.242C109.521,-296.527 109.999,-271.903 110.314,-255.657\"/>\n",
+       "</g>\n",
+       "<!-- 4520854672backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4520854672backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M133,-420.5C133,-420.5 172.295,-420.5 172.295,-420.5 178.295,-420.5 184.295,-426.5 184.295,-432.5 184.295,-432.5 184.295,-462.5 184.295,-462.5 184.295,-468.5 178.295,-474.5 172.295,-474.5 172.295,-474.5 133,-474.5 133,-474.5 127,-474.5 121,-468.5 121,-462.5 121,-462.5 121,-432.5 121,-432.5 121,-426.5 127,-420.5 133,-420.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.647\" y=\"-462.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 2.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"121,-456.5 184.295,-456.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.647\" y=\"-444.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 8.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"121,-438.5 184.295,-438.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.647\" y=\"-426.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4520854672backward&#45;&gt;4520854624backward -->\n",
+       "<g id=\"edge10\" class=\"edge\"><title>4520854672backward&#45;&gt;4520854624backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M141.431,-420.242C136.104,-407.772 129.656,-392.679 123.893,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"126.991,-377.531 119.844,-369.71 120.554,-380.281 126.991,-377.531\"/>\n",
+       "<text text-anchor=\"middle\" x=\"147.647\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854480backward -->\n",
+       "<g id=\"node13\" class=\"node\"><title>4520854480backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M170,-315.5C170,-315.5 209.295,-315.5 209.295,-315.5 215.295,-315.5 221.295,-321.5 221.295,-327.5 221.295,-327.5 221.295,-357.5 221.295,-357.5 221.295,-363.5 215.295,-369.5 209.295,-369.5 209.295,-369.5 170,-369.5 170,-369.5 164,-369.5 158,-363.5 158,-357.5 158,-357.5 158,-327.5 158,-327.5 158,-321.5 164,-315.5 170,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.647\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 2.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"158,-351.5 221.295,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.647\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"158,-333.5 221.295,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.647\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4520854672backward&#45;&gt;4520854480backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4520854672backward&#45;&gt;4520854480backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M162.08,-420.242C166.559,-407.772 171.981,-392.679 176.827,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"180.145,-380.304 180.232,-369.71 173.558,-377.938 180.145,-380.304\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.647\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4520854240backward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4520854240backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"188.647\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.647\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y1= 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4520853856backward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4520853856backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"257.647\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"257.647\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x1= 1.50</text>\n",
+       "</g>\n",
+       "<!-- 4520854384backward -->\n",
+       "<g id=\"node11\" class=\"node\"><title>4520854384backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M248,-210.5C248,-210.5 287.295,-210.5 287.295,-210.5 293.295,-210.5 299.295,-216.5 299.295,-222.5 299.295,-222.5 299.295,-252.5 299.295,-252.5 299.295,-258.5 293.295,-264.5 287.295,-264.5 287.295,-264.5 248,-264.5 248,-264.5 242,-264.5 236,-258.5 236,-252.5 236,-252.5 236,-222.5 236,-222.5 236,-216.5 242,-210.5 248,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"267.647\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;2.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"236,-246.5 299.295,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"267.647\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"236,-228.5 299.295,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"267.647\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4520854384backward&#45;&gt;4520854144backward -->\n",
+       "<g id=\"edge14\" class=\"edge\"><title>4520854384backward&#45;&gt;4520854144backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M241.39,-210.242C228.193,-197.046 212.059,-180.912 197.996,-166.849\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"200.403,-164.306 190.857,-159.71 195.453,-169.256 200.403,-164.306\"/>\n",
+       "<text text-anchor=\"middle\" x=\"236.229\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4520853952backward -->\n",
+       "<g id=\"node12\" class=\"node\"><title>4520853952backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M238,-105.5C238,-105.5 277.295,-105.5 277.295,-105.5 283.295,-105.5 289.295,-111.5 289.295,-117.5 289.295,-117.5 289.295,-147.5 289.295,-147.5 289.295,-153.5 283.295,-159.5 277.295,-159.5 277.295,-159.5 238,-159.5 238,-159.5 232,-159.5 226,-153.5 226,-147.5 226,-147.5 226,-117.5 226,-117.5 226,-111.5 232,-105.5 238,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"257.647\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;2.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"226,-141.5 289.295,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"257.647\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"226,-123.5 289.295,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"257.647\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4520854384backward&#45;&gt;4520853952backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4520854384backward&#45;&gt;4520853952backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M265.098,-210.242C263.911,-198.014 262.479,-183.264 261.189,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"264.642,-169.325 260.192,-159.71 257.675,-170.001 264.642,-169.325\"/>\n",
+       "<text text-anchor=\"middle\" x=\"277.229\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4520853952backward&#45;&gt;4520854096backward -->\n",
+       "<g id=\"edge13\" class=\"edge\"><title>4520853952backward&#45;&gt;4520854096backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M231.9,-105.242C219.078,-92.167 203.429,-76.2078 189.729,-62.2363\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"191.849,-59.3994 182.348,-54.7098 186.851,-64.3004 191.849,-59.3994\"/>\n",
+       "<text text-anchor=\"middle\" x=\"227.229\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.50</text>\n",
+       "</g>\n",
+       "<!-- 4520853952backward&#45;&gt;4520853856backward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4520853952backward&#45;&gt;4520853856backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M257.647,-105.242C257.647,-86.5272 257.647,-61.9035 257.647,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4520854480backward&#45;&gt;4520854240backward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4520854480backward&#45;&gt;4520854240backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M189.393,-315.242C189.211,-296.527 188.972,-271.903 188.814,-255.657\"/>\n",
+       "</g>\n",
+       "<!-- 4520854480backward&#45;&gt;4520854384backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4520854480backward&#45;&gt;4520854384backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M209.532,-315.242C219.25,-302.409 231.072,-286.798 241.51,-273.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"244.552,-274.795 247.799,-264.71 238.972,-270.569 244.552,-274.795\"/>\n",
+       "<text text-anchor=\"middle\" x=\"247.229\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x10d5bb9d0>"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 第二次触发方向传播\n",
+    "loss.backward()\n",
+    "draw_graph(loss, 'backward')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "\n",
+    "# 固定随机种子，使得运行结果可以稳定复现\n",
+    "torch.manual_seed(1024)\n",
+    "# 产生训练用的数据\n",
+    "x_origin = torch.linspace(100, 300, 200)\n",
+    "# 将变量X归一化，否则梯度下降法很容易不稳定\n",
+    "x = (x_origin - torch.mean(x_origin)) / torch.std(x_origin)\n",
+    "epsilon = torch.randn(x.shape)\n",
+    "y = 10 * x + 5 + epsilon"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 1, Result: y = 3.12 * x + -1.99\n",
+      "Step 2, Result: y = 3.48 * x + -2.28\n",
+      "Step 3, Result: y = 3.22 * x + -1.97\n",
+      "Step 4, Result: y = 2.85 * x + -1.22\n",
+      "Step 5, Result: y = 2.68 * x + -0.23\n",
+      "Step 6, Result: y = 2.92 * x + 1.08\n",
+      "Step 7, Result: y = 3.74 * x + 2.61\n",
+      "Step 8, Result: y = 5.07 * x + 4.15\n",
+      "Step 9, Result: y = 6.73 * x + 5.52\n",
+      "Step 10, Result: y = 8.22 * x + 6.48\n",
+      "Step 11, Result: y = 9.36 * x + 5.75\n",
+      "Step 12, Result: y = 9.75 * x + 5.42\n",
+      "Step 13, Result: y = 9.88 * x + 5.28\n",
+      "Step 14, Result: y = 9.89 * x + 5.26\n",
+      "Step 15, Result: y = 9.89 * x + 5.20\n",
+      "Step 16, Result: y = 9.88 * x + 5.18\n",
+      "Step 17, Result: y = 9.88 * x + 5.17\n",
+      "Step 18, Result: y = 9.84 * x + 5.14\n",
+      "Step 19, Result: y = 9.86 * x + 5.15\n",
+      "Step 20, Result: y = 9.94 * x + 5.21\n"
+     ]
+    }
+   ],
+   "source": [
+    "# 生成模型\n",
+    "model = Linear()\n",
+    "# 定义每批次用到的数据量\n",
+    "batch_size = 20\n",
+    "learning_rate = 0.1\n",
+    "\n",
+    "for t in range(20):\n",
+    "    # 选取当前批次的数据，用于训练模型\n",
+    "    ix = (t * batch_size) % len(x)\n",
+    "    xx = x[ix: ix + batch_size]\n",
+    "    yy = y[ix: ix + batch_size]\n",
+    "    # 计算当前批次数据的损失\n",
+    "    loss = mse([model.error(_x, _y) for _x, _y in zip(xx, yy)])\n",
+    "    # 计算损失函数的梯度\n",
+    "    loss.backward()\n",
+    "    # 迭代更新模型参数的估计值\n",
+    "    model.a -= learning_rate * model.a.grad\n",
+    "    model.b -= learning_rate * model.b.grad\n",
+    "    # 将使用完的梯度清零\n",
+    "    model.a.grad = 0.0\n",
+    "    model.b.grad = 0.0\n",
+    "    print(f'Step {t + 1}, Result: {model.string()}')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

ch07-autograd/parameter_freezing.ipynb

@@ -0,0 +1,652 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from utils import Scalar, draw_graph\n",
+    "from linear_model import Linear, mse"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"308pt\" height=\"483pt\"\n",
+       " viewBox=\"0.00 0.00 308.46 483.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 479)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-479 304.462,-479 304.462,4 -4,4\"/>\n",
+       "<!-- 4541826048backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4541826048backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"106.814\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x2= 2.00</text>\n",
+       "</g>\n",
+       "<!-- 4541826576backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4541826576backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M88.1667,-315.5C88.1667,-315.5 127.462,-315.5 127.462,-315.5 133.462,-315.5 139.462,-321.5 139.462,-327.5 139.462,-327.5 139.462,-357.5 139.462,-357.5 139.462,-363.5 133.462,-369.5 127.462,-369.5 127.462,-369.5 88.1667,-369.5 88.1667,-369.5 82.1667,-369.5 76.1667,-363.5 76.1667,-357.5 76.1667,-357.5 76.1667,-327.5 76.1667,-327.5 76.1667,-321.5 82.1667,-315.5 88.1667,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"107.814\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"76.1667,-351.5 139.462,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"107.814\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"76.1667,-333.5 139.462,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"107.814\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4541825664backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4541825664backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"29.8142\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"29.8142\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y2= 4.50</text>\n",
+       "</g>\n",
+       "<!-- 4541826576backward&#45;&gt;4541825664backward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4541826576backward&#45;&gt;4541825664backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M87.9297,-315.242C73.3057,-295.931 53.9177,-270.329 41.6514,-254.131\"/>\n",
+       "</g>\n",
+       "<!-- 4541826528backward -->\n",
+       "<g id=\"node13\" class=\"node\"><title>4541826528backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M89.1667,-210.5C89.1667,-210.5 128.462,-210.5 128.462,-210.5 134.462,-210.5 140.462,-216.5 140.462,-222.5 140.462,-222.5 140.462,-252.5 140.462,-252.5 140.462,-258.5 134.462,-264.5 128.462,-264.5 128.462,-264.5 89.1667,-264.5 89.1667,-264.5 83.1667,-264.5 77.1667,-258.5 77.1667,-252.5 77.1667,-252.5 77.1667,-222.5 77.1667,-222.5 77.1667,-216.5 83.1667,-210.5 89.1667,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.1667,-246.5 140.462,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77.1667,-228.5 140.462,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4541826576backward&#45;&gt;4541826528backward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4541826576backward&#45;&gt;4541826528backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M108.069,-315.242C108.188,-303.014 108.331,-288.264 108.46,-274.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"111.962,-274.743 108.56,-264.71 104.963,-274.675 111.962,-274.743\"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.395\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.50</text>\n",
+       "</g>\n",
+       "<!-- 4541826624backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4541826624backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M129.667,-420.5C129.667,-420.5 173.962,-420.5 173.962,-420.5 179.962,-420.5 185.962,-426.5 185.962,-432.5 185.962,-432.5 185.962,-462.5 185.962,-462.5 185.962,-468.5 179.962,-474.5 173.962,-474.5 173.962,-474.5 129.667,-474.5 129.667,-474.5 123.667,-474.5 117.667,-468.5 117.667,-462.5 117.667,-462.5 117.667,-432.5 117.667,-432.5 117.667,-426.5 123.667,-420.5 129.667,-420.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"151.814\" y=\"-462.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"117.667,-456.5 185.962,-456.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"151.814\" y=\"-444.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 10.62</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"117.667,-438.5 185.962,-438.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"151.814\" y=\"-426.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4541826624backward&#45;&gt;4541826576backward -->\n",
+       "<g id=\"edge14\" class=\"edge\"><title>4541826624backward&#45;&gt;4541826576backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M140.597,-420.242C135.27,-407.772 128.823,-392.679 123.06,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"126.158,-377.531 119.011,-369.71 119.72,-380.281 126.158,-377.531\"/>\n",
+       "<text text-anchor=\"middle\" x=\"146.814\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.50</text>\n",
+       "</g>\n",
+       "<!-- 4541826192backward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4541826192backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M169.167,-315.5C169.167,-315.5 208.462,-315.5 208.462,-315.5 214.462,-315.5 220.462,-321.5 220.462,-327.5 220.462,-327.5 220.462,-357.5 220.462,-357.5 220.462,-363.5 214.462,-369.5 208.462,-369.5 208.462,-369.5 169.167,-369.5 169.167,-369.5 163.167,-369.5 157.167,-363.5 157.167,-357.5 157.167,-357.5 157.167,-327.5 157.167,-327.5 157.167,-321.5 163.167,-315.5 169.167,-315.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.814\" y=\"-357.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"157.167,-351.5 220.462,-351.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.814\" y=\"-339.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"157.167,-333.5 220.462,-333.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.814\" y=\"-321.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4541826624backward&#45;&gt;4541826192backward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4541826624backward&#45;&gt;4541826192backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M161.247,-420.242C165.726,-407.772 171.148,-392.679 175.994,-379.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"179.312,-380.304 179.399,-369.71 172.724,-377.938 179.312,-380.304\"/>\n",
+       "<text text-anchor=\"middle\" x=\"185.814\" y=\"-390.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4541825616backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4541825616backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"187.814\" cy=\"-237.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">y1= 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4541826144backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4541826144backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"155.195,-0.5 155.195,-54.5 220.433,-54.5 220.433,-0.5 155.195,-0.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-42.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;10.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"155.195,-36.5 220.433,-36.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"155.195,-18.5 220.433,-18.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-6.5\" font-family=\"Times,serif\" font-size=\"10.00\">a</text>\n",
+       "</g>\n",
+       "<!-- 4541826192backward&#45;&gt;4541825616backward -->\n",
+       "<g id=\"edge10\" class=\"edge\"><title>4541826192backward&#45;&gt;4541825616backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M188.559,-315.242C188.378,-296.527 188.138,-271.903 187.981,-255.657\"/>\n",
+       "</g>\n",
+       "<!-- 4541826336backward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4541826336backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M247.167,-210.5C247.167,-210.5 286.462,-210.5 286.462,-210.5 292.462,-210.5 298.462,-216.5 298.462,-222.5 298.462,-222.5 298.462,-252.5 298.462,-252.5 298.462,-258.5 292.462,-264.5 286.462,-264.5 286.462,-264.5 247.167,-264.5 247.167,-264.5 241.167,-264.5 235.167,-258.5 235.167,-252.5 235.167,-252.5 235.167,-222.5 235.167,-222.5 235.167,-216.5 241.167,-210.5 247.167,-210.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-252.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"235.167,-246.5 298.462,-246.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-234.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"235.167,-228.5 298.462,-228.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-216.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4541826192backward&#45;&gt;4541826336backward -->\n",
+       "<g id=\"edge12\" class=\"edge\"><title>4541826192backward&#45;&gt;4541826336backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M208.699,-315.242C218.417,-302.409 230.239,-286.798 240.677,-273.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"243.719,-274.795 246.966,-264.71 238.139,-270.569 243.719,-274.795\"/>\n",
+       "<text text-anchor=\"middle\" x=\"246.395\" y=\"-285.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4541825712backward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4541825712backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"268.814\" cy=\"-27.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-24.5\" font-family=\"Times,serif\" font-size=\"10.00\">x1= 1.50</text>\n",
+       "</g>\n",
+       "<!-- 4541826288backward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4541826288backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-105.5 156.167,-159.5 219.462,-159.5 219.462,-105.5 156.167,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;5.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-141.5 219.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"156.167,-123.5 219.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4541826336backward&#45;&gt;4541826288backward -->\n",
+       "<g id=\"edge11\" class=\"edge\"><title>4541826336backward&#45;&gt;4541826288backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M246.675,-210.242C236.832,-197.409 224.858,-181.798 214.286,-168.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"216.78,-165.515 207.917,-159.71 211.226,-169.775 216.78,-165.515\"/>\n",
+       "<text text-anchor=\"middle\" x=\"246.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4541825856backward -->\n",
+       "<g id=\"node11\" class=\"node\"><title>4541825856backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M249.167,-105.5C249.167,-105.5 288.462,-105.5 288.462,-105.5 294.462,-105.5 300.462,-111.5 300.462,-117.5 300.462,-117.5 300.462,-147.5 300.462,-147.5 300.462,-153.5 294.462,-159.5 288.462,-159.5 288.462,-159.5 249.167,-159.5 249.167,-159.5 243.167,-159.5 237.167,-153.5 237.167,-147.5 237.167,-147.5 237.167,-117.5 237.167,-117.5 237.167,-111.5 243.167,-105.5 249.167,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"237.167,-141.5 300.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"237.167,-123.5 300.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"268.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4541826336backward&#45;&gt;4541825856backward -->\n",
+       "<g id=\"edge13\" class=\"edge\"><title>4541826336backward&#45;&gt;4541825856backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M267.324,-210.242C267.561,-198.014 267.848,-183.264 268.106,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"271.61,-169.776 268.305,-159.71 264.612,-169.64 271.61,-169.776\"/>\n",
+       "<text text-anchor=\"middle\" x=\"282.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4541825856backward&#45;&gt;4541826144backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4541825856backward&#45;&gt;4541826144backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M248.165,-105.242C237.978,-92.2881 225.564,-76.503 214.65,-62.6249\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"217.359,-60.4068 208.426,-54.7098 211.856,-64.7339 217.359,-60.4068\"/>\n",
+       "<text text-anchor=\"middle\" x=\"247.395\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.50</text>\n",
+       "</g>\n",
+       "<!-- 4541825856backward&#45;&gt;4541825712backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4541825856backward&#45;&gt;4541825712backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M268.814,-105.242C268.814,-86.5272 268.814,-61.9035 268.814,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4541826480backward -->\n",
+       "<g id=\"node12\" class=\"node\"><title>4541826480backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M87.1667,-105.5C87.1667,-105.5 126.462,-105.5 126.462,-105.5 132.462,-105.5 138.462,-111.5 138.462,-117.5 138.462,-117.5 138.462,-147.5 138.462,-147.5 138.462,-153.5 132.462,-159.5 126.462,-159.5 126.462,-159.5 87.1667,-159.5 87.1667,-159.5 81.1667,-159.5 75.1667,-153.5 75.1667,-147.5 75.1667,-147.5 75.1667,-117.5 75.1667,-117.5 75.1667,-111.5 81.1667,-105.5 87.1667,-105.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-147.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"75.1667,-141.5 138.462,-141.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-129.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"75.1667,-123.5 138.462,-123.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"106.814\" y=\"-111.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4541826480backward&#45;&gt;4541826048backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4541826480backward&#45;&gt;4541826048backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M106.814,-105.242C106.814,-86.5272 106.814,-61.9035 106.814,-45.6575\"/>\n",
+       "</g>\n",
+       "<!-- 4541826480backward&#45;&gt;4541826144backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4541826480backward&#45;&gt;4541826144backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M127.463,-105.242C137.651,-92.2881 150.064,-76.503 160.978,-62.6249\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"163.772,-64.7339 167.203,-54.7098 158.27,-60.4068 163.772,-64.7339\"/>\n",
+       "<text text-anchor=\"middle\" x=\"167.395\" y=\"-75.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;9.00</text>\n",
+       "</g>\n",
+       "<!-- 4541826528backward&#45;&gt;4541826288backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4541826528backward&#45;&gt;4541826288backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M128.954,-210.242C138.796,-197.409 150.77,-181.798 161.342,-168.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"164.403,-169.775 167.711,-159.71 158.848,-165.515 164.403,-169.775\"/>\n",
+       "<text text-anchor=\"middle\" x=\"168.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.50</text>\n",
+       "</g>\n",
+       "<!-- 4541826528backward&#45;&gt;4541826480backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4541826528backward&#45;&gt;4541826480backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M108.304,-210.242C108.067,-198.014 107.78,-183.264 107.522,-169.977\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"111.017,-169.64 107.323,-159.71 104.018,-169.776 111.017,-169.64\"/>\n",
+       "<text text-anchor=\"middle\" x=\"123.395\" y=\"-180.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.50</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x10eb6c790>"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "x1 = Scalar(1.5, label='x1', requires_grad=False)\n",
+    "y1 = Scalar(1.0, label='y1', requires_grad=False)\n",
+    "x2 = Scalar(2.0, label='x2', requires_grad=False)\n",
+    "y2 = Scalar(4.5, label='y2', requires_grad=False)\n",
+    "# 反向传播\n",
+    "model = Linear()\n",
+    "loss = mse([model.error(x1, y1), model.error(x2, y2)])\n",
+    "loss.backward()\n",
+    "draw_graph(loss, 'backward')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"307pt\" height=\"451pt\"\n",
+       " viewBox=\"0.00 0.00 307.29 451.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 447)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-447 303.295,-447 303.295,4 -4,4\"/>\n",
+       "<!-- 4541826048backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4541826048backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"77.6475\" cy=\"-18\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"77.6475\" y=\"-15\" font-family=\"Times,serif\" font-size=\"10.00\">x2= 2.00</text>\n",
+       "</g>\n",
+       "<!-- 4541826096backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4541826096backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M130.5,-388.5C130.5,-388.5 174.795,-388.5 174.795,-388.5 180.795,-388.5 186.795,-394.5 186.795,-400.5 186.795,-400.5 186.795,-430.5 186.795,-430.5 186.795,-436.5 180.795,-442.5 174.795,-442.5 174.795,-442.5 130.5,-442.5 130.5,-442.5 124.5,-442.5 118.5,-436.5 118.5,-430.5 118.5,-430.5 118.5,-400.5 118.5,-400.5 118.5,-394.5 124.5,-388.5 130.5,-388.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.647\" y=\"-430.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"118.5,-424.5 186.795,-424.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.647\" y=\"-412.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 10.62</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"118.5,-406.5 186.795,-406.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"152.647\" y=\"-394.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4541773376backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4541773376backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M89,-283.5C89,-283.5 128.295,-283.5 128.295,-283.5 134.295,-283.5 140.295,-289.5 140.295,-295.5 140.295,-295.5 140.295,-325.5 140.295,-325.5 140.295,-331.5 134.295,-337.5 128.295,-337.5 128.295,-337.5 89,-337.5 89,-337.5 83,-337.5 77,-331.5 77,-325.5 77,-325.5 77,-295.5 77,-295.5 77,-289.5 83,-283.5 89,-283.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.647\" y=\"-325.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77,-319.5 140.295,-319.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.647\" y=\"-307.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"77,-301.5 140.295,-301.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.647\" y=\"-289.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4541826096backward&#45;&gt;4541773376backward -->\n",
+       "<g id=\"edge14\" class=\"edge\"><title>4541826096backward&#45;&gt;4541773376backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M141.431,-388.242C136.104,-375.772 129.656,-360.679 123.893,-347.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"126.991,-345.531 119.844,-337.71 120.554,-348.281 126.991,-345.531\"/>\n",
+       "<text text-anchor=\"middle\" x=\"147.647\" y=\"-358.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4.50</text>\n",
+       "</g>\n",
+       "<!-- 4541775776backward -->\n",
+       "<g id=\"node13\" class=\"node\"><title>4541775776backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M170,-283.5C170,-283.5 209.295,-283.5 209.295,-283.5 215.295,-283.5 221.295,-289.5 221.295,-295.5 221.295,-295.5 221.295,-325.5 221.295,-325.5 221.295,-331.5 215.295,-337.5 209.295,-337.5 209.295,-337.5 170,-337.5 170,-337.5 164,-337.5 158,-331.5 158,-325.5 158,-325.5 158,-295.5 158,-295.5 158,-289.5 164,-283.5 170,-283.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.647\" y=\"-325.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"158,-319.5 221.295,-319.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.647\" y=\"-307.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"158,-301.5 221.295,-301.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"189.647\" y=\"-289.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4541826096backward&#45;&gt;4541775776backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4541826096backward&#45;&gt;4541775776backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M162.08,-388.242C166.559,-375.772 171.981,-360.679 176.827,-347.189\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"180.145,-348.304 180.232,-337.71 173.558,-345.938 180.145,-348.304\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.647\" y=\"-358.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4541773424backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4541773424backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M12,-178.5C12,-178.5 51.2949,-178.5 51.2949,-178.5 57.2949,-178.5 63.2949,-184.5 63.2949,-190.5 63.2949,-190.5 63.2949,-220.5 63.2949,-220.5 63.2949,-226.5 57.2949,-232.5 51.2949,-232.5 51.2949,-232.5 12,-232.5 12,-232.5 6,-232.5 0,-226.5 0,-220.5 0,-220.5 0,-190.5 0,-190.5 0,-184.5 6,-178.5 12,-178.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-220.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"0,-214.5 63.2949,-214.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-202.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"0,-196.5 63.2949,-196.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"31.6475\" y=\"-184.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4541773376backward&#45;&gt;4541773424backward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4541773376backward&#45;&gt;4541773424backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M89.0179,-283.242C79.4242,-270.409 67.7536,-254.798 57.4496,-241.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"60.0321,-238.623 51.2412,-232.71 54.4255,-242.815 60.0321,-238.623\"/>\n",
+       "<text text-anchor=\"middle\" x=\"90.2285\" y=\"-253.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.50</text>\n",
+       "</g>\n",
+       "<!-- 4541825664backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4541825664backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"110.647\" cy=\"-205.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"110.647\" y=\"-202.5\" font-family=\"Times,serif\" font-size=\"10.00\">y2= 4.50</text>\n",
+       "</g>\n",
+       "<!-- 4541773376backward&#45;&gt;4541825664backward -->\n",
+       "<g id=\"edge13\" class=\"edge\"><title>4541773376backward&#45;&gt;4541825664backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M109.157,-283.242C109.521,-264.527 109.999,-239.903 110.314,-223.657\"/>\n",
+       "</g>\n",
+       "<!-- 4541825616backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4541825616backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"188.647\" cy=\"-205.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.647\" y=\"-202.5\" font-family=\"Times,serif\" font-size=\"10.00\">y1= 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4541774528backward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4541774528backward</title>\n",
+       "<polygon fill=\"springgreen\" stroke=\"black\" stroke-width=\"2\" points=\"132,-73.5 132,-127.5 195.295,-127.5 195.295,-73.5 132,-73.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"163.647\" y=\"-115.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;5.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"132,-109.5 195.295,-109.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"163.647\" y=\"-97.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"132,-91.5 195.295,-91.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"163.647\" y=\"-79.5\" font-family=\"Times,serif\" font-size=\"10.00\">b</text>\n",
+       "</g>\n",
+       "<!-- 4541773424backward&#45;&gt;4541774528backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4541773424backward&#45;&gt;4541774528backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M63.601,-179.567C81.7605,-165.397 104.655,-147.532 123.959,-132.469\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"126.257,-135.115 131.988,-126.204 121.951,-129.596 126.257,-135.115\"/>\n",
+       "<text text-anchor=\"middle\" x=\"120.229\" y=\"-148.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;4.50</text>\n",
+       "</g>\n",
+       "<!-- 4541773616backward -->\n",
+       "<g id=\"node11\" class=\"node\"><title>4541773616backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"46,-73.5 46,-127.5 109.295,-127.5 109.295,-73.5 46,-73.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"77.6475\" y=\"-115.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"46,-109.5 109.295,-109.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"77.6475\" y=\"-97.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"46,-91.5 109.295,-91.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"77.6475\" y=\"-79.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4541773424backward&#45;&gt;4541773616backward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4541773424backward&#45;&gt;4541773616backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M43.3742,-178.242C50.295,-162.746 59.025,-143.198 65.9421,-127.71\"/>\n",
+       "</g>\n",
+       "<!-- 4541773472backward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4541773472backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"215,-73.5 215,-127.5 278.295,-127.5 278.295,-73.5 215,-73.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"246.647\" y=\"-115.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"215,-109.5 278.295,-109.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"246.647\" y=\"-97.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"215,-91.5 278.295,-91.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"246.647\" y=\"-79.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4541825712backward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4541825712backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"246.647\" cy=\"-18\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"246.647\" y=\"-15\" font-family=\"Times,serif\" font-size=\"10.00\">x1= 1.50</text>\n",
+       "</g>\n",
+       "<!-- 4541773472backward&#45;&gt;4541825712backward -->\n",
+       "<g id=\"edge10\" class=\"edge\"><title>4541773472backward&#45;&gt;4541825712backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M246.647,-73.1413C246.647,-61.0359 246.647,-47.0197 246.647,-36.2816\"/>\n",
+       "</g>\n",
+       "<!-- 4541775728backward -->\n",
+       "<g id=\"node12\" class=\"node\"><title>4541775728backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"162.647\" cy=\"-18\" rx=\"27\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"162.647\" y=\"-15\" font-family=\"Times,serif\" font-size=\"10.00\">a= 0.00</text>\n",
+       "</g>\n",
+       "<!-- 4541773472backward&#45;&gt;4541775728backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4541773472backward&#45;&gt;4541775728backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M219.143,-73.1413C205.363,-59.9354 189.21,-44.4555 177.713,-33.4382\"/>\n",
+       "</g>\n",
+       "<!-- 4541774576backward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4541774576backward</title>\n",
+       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M248,-178.5C248,-178.5 287.295,-178.5 287.295,-178.5 293.295,-178.5 299.295,-184.5 299.295,-190.5 299.295,-190.5 299.295,-220.5 299.295,-220.5 299.295,-226.5 293.295,-232.5 287.295,-232.5 287.295,-232.5 248,-232.5 248,-232.5 242,-232.5 236,-226.5 236,-220.5 236,-220.5 236,-190.5 236,-190.5 236,-184.5 242,-178.5 248,-178.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"267.647\" y=\"-220.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"236,-214.5 299.295,-214.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"267.647\" y=\"-202.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" points=\"236,-196.5 299.295,-196.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"267.647\" y=\"-184.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4541774576backward&#45;&gt;4541773472backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4541774576backward&#45;&gt;4541773472backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M262.294,-178.242C259.134,-162.746 255.149,-143.198 251.991,-127.71\"/>\n",
+       "</g>\n",
+       "<!-- 4541774576backward&#45;&gt;4541774528backward -->\n",
+       "<g id=\"edge12\" class=\"edge\"><title>4541774576backward&#45;&gt;4541774528backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M241.135,-178.242C227.81,-165.046 211.52,-148.912 197.32,-134.849\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"199.68,-132.26 190.112,-127.71 194.754,-137.233 199.68,-132.26\"/>\n",
+       "<text text-anchor=\"middle\" x=\"237.229\" y=\"-148.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "<!-- 4541773616backward&#45;&gt;4541826048backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4541773616backward&#45;&gt;4541826048backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M77.6475,-73.1413C77.6475,-61.0359 77.6475,-47.0197 77.6475,-36.2816\"/>\n",
+       "</g>\n",
+       "<!-- 4541773616backward&#45;&gt;4541775728backward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4541773616backward&#45;&gt;4541775728backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M105.48,-73.1413C119.424,-59.9354 135.769,-44.4555 147.402,-33.4382\"/>\n",
+       "</g>\n",
+       "<!-- 4541775776backward&#45;&gt;4541825616backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4541775776backward&#45;&gt;4541825616backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M189.393,-283.242C189.211,-264.527 188.972,-239.903 188.814,-223.657\"/>\n",
+       "</g>\n",
+       "<!-- 4541775776backward&#45;&gt;4541774576backward -->\n",
+       "<g id=\"edge11\" class=\"edge\"><title>4541775776backward&#45;&gt;4541774576backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M209.532,-283.242C219.25,-270.409 231.072,-254.798 241.51,-241.014\"/>\n",
+       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"244.552,-242.795 247.799,-232.71 238.972,-238.569 244.552,-242.795\"/>\n",
+       "<text text-anchor=\"middle\" x=\"248.229\" y=\"-253.8\" font-family=\"Times,serif\" font-size=\"14.00\">&#45;1.00</text>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x10eb6c040>"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 反向传播\n",
+    "model = Linear()\n",
+    "# 冻结参数a\n",
+    "model.a.requires_grad = False\n",
+    "loss = mse([model.error(x1, y1), model.error(x2, y2)])\n",
+    "loss.backward()\n",
+    "draw_graph(loss, 'backward')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/svg+xml": [
+       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
+       " -->\n",
+       "<!-- Title: %3 Pages: 1 -->\n",
+       "<svg width=\"306pt\" height=\"408pt\"\n",
+       " viewBox=\"0.00 0.00 306.46 408.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 404)\">\n",
+       "<title>%3</title>\n",
+       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-404 302.462,-404 302.462,4 -4,4\"/>\n",
+       "<!-- 4541826048backward -->\n",
+       "<g id=\"node1\" class=\"node\"><title>4541826048backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"110.814\" cy=\"-18\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"110.814\" y=\"-15\" font-family=\"Times,serif\" font-size=\"10.00\">x2= 2.00</text>\n",
+       "</g>\n",
+       "<!-- 4541825568backward -->\n",
+       "<g id=\"node2\" class=\"node\"><title>4541825568backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"76.1667,-254.5 76.1667,-308.5 139.462,-308.5 139.462,-254.5 76.1667,-254.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"107.814\" y=\"-296.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"76.1667,-290.5 139.462,-290.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"107.814\" y=\"-278.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"76.1667,-272.5 139.462,-272.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"107.814\" y=\"-260.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4541825664backward -->\n",
+       "<g id=\"node6\" class=\"node\"><title>4541825664backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"29.8142\" cy=\"-190.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"29.8142\" y=\"-187.5\" font-family=\"Times,serif\" font-size=\"10.00\">y2= 4.50</text>\n",
+       "</g>\n",
+       "<!-- 4541825568backward&#45;&gt;4541825664backward -->\n",
+       "<g id=\"edge4\" class=\"edge\"><title>4541825568backward&#45;&gt;4541825664backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M84.8335,-254.278C71.2623,-238.793 54.5588,-219.734 43.1397,-206.705\"/>\n",
+       "</g>\n",
+       "<!-- 4541825376backward -->\n",
+       "<g id=\"node10\" class=\"node\"><title>4541825376backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"77.1667,-163.5 77.1667,-217.5 140.462,-217.5 140.462,-163.5 77.1667,-163.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-205.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"77.1667,-199.5 140.462,-199.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-187.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"77.1667,-181.5 140.462,-181.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"108.814\" y=\"-169.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4541825568backward&#45;&gt;4541825376backward -->\n",
+       "<g id=\"edge9\" class=\"edge\"><title>4541825568backward&#45;&gt;4541825376backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M108.109,-254.278C108.239,-242.69 108.392,-229.101 108.522,-217.534\"/>\n",
+       "</g>\n",
+       "<!-- 4541825616backward -->\n",
+       "<g id=\"node3\" class=\"node\"><title>4541825616backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"187.814\" cy=\"-190.5\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"187.814\" y=\"-187.5\" font-family=\"Times,serif\" font-size=\"10.00\">y1= 1.00</text>\n",
+       "</g>\n",
+       "<!-- 4541827680backward -->\n",
+       "<g id=\"node4\" class=\"node\"><title>4541827680backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"157.167,-254.5 157.167,-308.5 220.462,-308.5 220.462,-254.5 157.167,-254.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.814\" y=\"-296.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"157.167,-290.5 220.462,-290.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.814\" y=\"-278.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"157.167,-272.5 220.462,-272.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.814\" y=\"-260.5\" font-family=\"Times,serif\" font-size=\"10.00\">&#45;</text>\n",
+       "</g>\n",
+       "<!-- 4541827680backward&#45;&gt;4541825616backward -->\n",
+       "<g id=\"edge14\" class=\"edge\"><title>4541827680backward&#45;&gt;4541825616backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M188.52,-254.278C188.354,-239.511 188.151,-221.494 188.006,-208.555\"/>\n",
+       "</g>\n",
+       "<!-- 4541827008backward -->\n",
+       "<g id=\"node9\" class=\"node\"><title>4541827008backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"235.167,-163.5 235.167,-217.5 298.462,-217.5 298.462,-163.5 235.167,-163.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-205.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"235.167,-199.5 298.462,-199.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-187.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"235.167,-181.5 298.462,-181.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"266.814\" y=\"-169.5\" font-family=\"Times,serif\" font-size=\"10.00\">+</text>\n",
+       "</g>\n",
+       "<!-- 4541827680backward&#45;&gt;4541827008backward -->\n",
+       "<g id=\"edge12\" class=\"edge\"><title>4541827680backward&#45;&gt;4541827008backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M211.795,-254.278C221.951,-242.69 233.861,-229.101 243.998,-217.534\"/>\n",
+       "</g>\n",
+       "<!-- 4541825136backward -->\n",
+       "<g id=\"node5\" class=\"node\"><title>4541825136backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"188.814\" cy=\"-18\" rx=\"27\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.814\" y=\"-15\" font-family=\"Times,serif\" font-size=\"10.00\">a= 0.00</text>\n",
+       "</g>\n",
+       "<!-- 4541825184backward -->\n",
+       "<g id=\"node7\" class=\"node\"><title>4541825184backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"79.1667,-72.5 79.1667,-126.5 142.462,-126.5 142.462,-72.5 79.1667,-72.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"110.814\" y=\"-114.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;4.50</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"79.1667,-108.5 142.462,-108.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"110.814\" y=\"-96.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"79.1667,-90.5 142.462,-90.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"110.814\" y=\"-78.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4541825184backward&#45;&gt;4541826048backward -->\n",
+       "<g id=\"edge6\" class=\"edge\"><title>4541825184backward&#45;&gt;4541826048backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M110.814,-72.4687C110.814,-60.5137 110.814,-46.6716 110.814,-36.0669\"/>\n",
+       "</g>\n",
+       "<!-- 4541825184backward&#45;&gt;4541825136backward -->\n",
+       "<g id=\"edge3\" class=\"edge\"><title>4541825184backward&#45;&gt;4541825136backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M136.354,-72.4687C148.972,-59.608 163.733,-44.5635 174.377,-33.7148\"/>\n",
+       "</g>\n",
+       "<!-- 4541825712backward -->\n",
+       "<g id=\"node8\" class=\"node\"><title>4541825712backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"265.814\" cy=\"-18\" rx=\"29.6295\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"265.814\" y=\"-15\" font-family=\"Times,serif\" font-size=\"10.00\">x1= 1.50</text>\n",
+       "</g>\n",
+       "<!-- 4541826432backward -->\n",
+       "<g id=\"node12\" class=\"node\"><title>4541826432backward</title>\n",
+       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"188.814\" cy=\"-99.5\" rx=\"27\" ry=\"18\"/>\n",
+       "<text text-anchor=\"middle\" x=\"188.814\" y=\"-96.5\" font-family=\"Times,serif\" font-size=\"10.00\">b= 0.00</text>\n",
+       "</g>\n",
+       "<!-- 4541827008backward&#45;&gt;4541826432backward -->\n",
+       "<g id=\"edge11\" class=\"edge\"><title>4541827008backward&#45;&gt;4541826432backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M243.833,-163.278C230.172,-147.691 213.337,-128.482 201.914,-115.447\"/>\n",
+       "</g>\n",
+       "<!-- 4541825472backward -->\n",
+       "<g id=\"node13\" class=\"node\"><title>4541825472backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"234.167,-72.5 234.167,-126.5 297.462,-126.5 297.462,-72.5 234.167,-72.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"265.814\" y=\"-114.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad=&#45;1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"234.167,-108.5 297.462,-108.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"265.814\" y=\"-96.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 0.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"234.167,-90.5 297.462,-90.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"265.814\" y=\"-78.5\" font-family=\"Times,serif\" font-size=\"10.00\">*</text>\n",
+       "</g>\n",
+       "<!-- 4541827008backward&#45;&gt;4541825472backward -->\n",
+       "<g id=\"edge1\" class=\"edge\"><title>4541827008backward&#45;&gt;4541825472backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M266.52,-163.278C266.389,-151.69 266.237,-138.101 266.107,-126.534\"/>\n",
+       "</g>\n",
+       "<!-- 4541825376backward&#45;&gt;4541825184backward -->\n",
+       "<g id=\"edge13\" class=\"edge\"><title>4541825376backward&#45;&gt;4541825184backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M109.403,-163.278C109.664,-151.69 109.969,-138.101 110.229,-126.534\"/>\n",
+       "</g>\n",
+       "<!-- 4541825376backward&#45;&gt;4541826432backward -->\n",
+       "<g id=\"edge7\" class=\"edge\"><title>4541825376backward&#45;&gt;4541826432backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M132.384,-163.278C146.395,-147.691 163.662,-128.482 175.379,-115.447\"/>\n",
+       "</g>\n",
+       "<!-- 4541825904backward -->\n",
+       "<g id=\"node11\" class=\"node\"><title>4541825904backward</title>\n",
+       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"113.667,-345.5 113.667,-399.5 181.962,-399.5 181.962,-345.5 113.667,-345.5\"/>\n",
+       "<text text-anchor=\"middle\" x=\"147.814\" y=\"-387.5\" font-family=\"Times,serif\" font-size=\"10.00\">grad= 1.00</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"113.667,-381.5 181.962,-381.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"147.814\" y=\"-369.5\" font-family=\"Times,serif\" font-size=\"10.00\">value= 10.62</text>\n",
+       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"113.667,-363.5 181.962,-363.5 \"/>\n",
+       "<text text-anchor=\"middle\" x=\"147.814\" y=\"-351.5\" font-family=\"Times,serif\" font-size=\"10.00\">mse</text>\n",
+       "</g>\n",
+       "<!-- 4541825904backward&#45;&gt;4541825568backward -->\n",
+       "<g id=\"edge2\" class=\"edge\"><title>4541825904backward&#45;&gt;4541825568backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M136.029,-345.278C130.821,-333.69 124.713,-320.101 119.515,-308.534\"/>\n",
+       "</g>\n",
+       "<!-- 4541825904backward&#45;&gt;4541827680backward -->\n",
+       "<g id=\"edge5\" class=\"edge\"><title>4541825904backward&#45;&gt;4541827680backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M159.894,-345.278C165.232,-333.69 171.493,-320.101 176.821,-308.534\"/>\n",
+       "</g>\n",
+       "<!-- 4541825472backward&#45;&gt;4541825136backward -->\n",
+       "<g id=\"edge10\" class=\"edge\"><title>4541825472backward&#45;&gt;4541825136backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M240.601,-72.4687C228.145,-59.608 213.574,-44.5635 203.066,-33.7148\"/>\n",
+       "</g>\n",
+       "<!-- 4541825472backward&#45;&gt;4541825712backward -->\n",
+       "<g id=\"edge8\" class=\"edge\"><title>4541825472backward&#45;&gt;4541825712backward</title>\n",
+       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M265.814,-72.4687C265.814,-60.5137 265.814,-46.6716 265.814,-36.0669\"/>\n",
+       "</g>\n",
+       "</g>\n",
+       "</svg>\n"
+      ],
+      "text/plain": [
+       "<graphviz.graphs.Digraph at 0x10eb6c310>"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 反向传播\n",
+    "model = Linear()\n",
+    "# 冻结参数a和参数b\n",
+    "model.a.requires_grad = False\n",
+    "model.b.requires_grad = False\n",
+    "loss = mse([model.error(x1, y1), model.error(x2, y2)])\n",
+    "loss.backward()\n",
+    "draw_graph(loss, 'backward')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

ch07-autograd/utils.py

@@ -0,0 +1,294 @@
+# -*- coding: UTF-8 -*-
+"""
+此脚本用于定义Scalar类，以及相应的可视化工具
+"""
+
+
+from graphviz import Digraph
+
+
+class Scalar:
+    def __init__(self, value, prevs=[], op=None, label='', requires_grad=True):
+        # 节点的值
+        self.value = value
+        # 节点的标识（label）和对应的运算（op），用于作图
+        self.label = label
+        self.op = op
+        # 节点的前节点
+        self.prevs = prevs
+        # 是否需要计算该节点偏导数，即∂loss/∂self
+        self.requires_grad = requires_grad
+        # 存储该节点偏导数，即∂loss/∂self
+        self.grad = 0.0
+        # 如果该节点的prevs非空，存储所有的∂self/∂prev
+        self.grad_wrt = dict()
+        # 作图需要，实际上对计算没有作用
+        self.back_prop = dict()
+        
+    def __repr__(self):
+        return f'Scalar(value={self.value:.2f}, grad={self.grad:.2f})'
+    
+    def __add__(self, other):
+        """
+        定义加法，self + other将触发该函数
+        """
+        if not isinstance(other, Scalar):
+            other = Scalar(other, requires_grad=False)
+        # output = self + other
+        output = Scalar(self.value + other.value, [self, other], '+')
+        output.requires_grad = self.requires_grad or other.requires_grad
+        # 计算偏导数 ∂output/∂self = 1
+        output.grad_wrt[self] = 1
+        # 计算偏导数 ∂output/∂other = 1
+        output.grad_wrt[other] = 1
+        return output
+    
+    def __sub__(self, other):
+        """
+        定义减法，self - other将触发该函数
+        """
+        if not isinstance(other, Scalar):
+            other = Scalar(other, requires_grad=False)
+        # output = self - other
+        output = Scalar(self.value - other.value, [self, other], '-')
+        output.requires_grad = self.requires_grad or other.requires_grad
+        # 计算偏导数 ∂output/∂self = 1
+        output.grad_wrt[self] = 1
+        # 计算偏导数 ∂output/∂other = -1
+        output.grad_wrt[other] = -1
+        return output
+    
+    def __mul__(self, other):
+        """
+        定义乘法，self * other将触发该函数
+        """
+        if not isinstance(other, Scalar):
+            other = Scalar(other, requires_grad=False)
+        # output = self * other
+        output = Scalar(self.value * other.value, [self, other], '*')
+        output.requires_grad = self.requires_grad or other.requires_grad
+        # 计算偏导数 ∂output/∂self = other
+        output.grad_wrt[self] = other.value
+        # 计算偏导数 ∂output/∂other = self
+        output.grad_wrt[other] = self.value
+        return output
+    
+    def __pow__(self, other):
+        """
+        定义乘方，self**other将触发该函数
+        """
+        assert isinstance(other, (int, float)), 'support only int or float in the exponent'
+        # output = self ** other
+        output = Scalar(self.value ** other, [self], f'^{other}')
+        output.requires_grad = self.requires_grad
+        # 计算偏导数 ∂output/∂self = other * self**(other-1)
+        output.grad_wrt[self] = other * self.value**(other - 1)
+        return output
+    
+    def sigmoid(self):
+        """
+        定义sigmoid
+        """
+        s = 1 / (1 + math.exp(-1 * self.value))
+        output = Scalar(s, [self], 'sigmoid')
+        output.requires_grad = self.requires_grad
+        # 计算偏导数 ∂output/∂self = output * (1 - output)
+        output.grad_wrt[self] = s * (1 - s)
+        return output
+    
+    def __rsub__(self, other):
+        """
+        定义右减法，other - self将触发该函数
+        """
+        if not isinstance(other, Scalar):
+            other = Scalar(other, requires_grad=False)
+        output = Scalar(other.value - self.value, [self, other], '-')
+        output.requires_grad = self.requires_grad or other.requires_grad
+        output.grad_wrt[self] = -1
+        output.grad_wrt[other] = 1
+        return output
+    
+    def __radd__(self, other):
+        """
+        定义右加法，other + self将触发该函数
+        """
+        return self.__add__(other)
+    
+    def __rmul__(self, other):
+        """
+        定义右乘法，other * self将触发该函数
+        """
+        return self * other
+    
+    def backward(self, fn=None):
+        """
+        由当前节点出发，求解以当前节点为顶点的计算图中每个节点的梯度，i.e. ∂self/∂node
+        参数
+        ----
+        fn ：画图函数，如果该变量不等于None，则会返回向后传播每一步的计算的记录
+        返回
+        ----
+        re ：向后传播每一步的计算的记录
+        """
+        def _topological_order():
+            """
+            利用深度优先算法，返回计算图的拓扑排序（topological sorting）
+            """
+            def _add_prevs(node):
+                if node not in visited:
+                    visited.add(node)
+                    for prev in node.prevs:
+                        _add_prevs(prev)
+                    ordered.append(node)
+            ordered, visited = [], set()
+            _add_prevs(self)
+            return ordered
+
+        def _compute_grad_of_prevs(node):
+            """
+            由node节点出发，向后传播
+            """
+            # 作图需要，实际上对计算没有作用
+            node.back_prop = dict()
+            # 得到当前节点在计算图中的梯度。由于一个节点可以在多个计算图中出现，
+            # 使用cg_grad记录当前计算图的梯度
+            dnode = cg_grad[node]
+            # 使用node.grad记录节点的累积梯度
+            node.grad += dnode
+            for prev in node.prevs:
+                # 由于node节点的偏导数已经计算完成，可以向后传播
+                # 需要注意的是，向后传播到上游节点是累加关系
+                grad_spread = dnode * node.grad_wrt[prev]
+                cg_grad[prev] = cg_grad.get(prev, 0.0) + grad_spread
+                node.back_prop[prev] = node.back_prop.get(prev, 0.0) + grad_spread
+        
+        # 当前节点的偏导数等于1，因为∂self/∂self = 1。这是反向传播算法的起点
+        cg_grad = {self: 1}
+        # 为了计算每个节点的偏导数，需要使用拓扑排序的倒序来遍历计算图
+        ordered = reversed(_topological_order())
+        re = []
+        for node in ordered:
+            _compute_grad_of_prevs(node)
+            # 作图需要，实际上对计算没有作用
+            if fn is not None:
+                re.append(fn(self, 'backward'))
+        return re
+
+
+def _get_node_attr(node, direction='forward'):
+    """
+    节点的属性
+    """
+    node_type = _get_node_type(node)
+    def _forward_attr():
+        if node_type == 'param':
+            node_text = f'{{ grad=None | value={node.value: .2f} | {node.label}}}'
+            return dict(label=node_text, shape='record', fontsize='10', fillcolor='springgreen', style='filled, bold')
+        elif node_type == 'computation':
+            node_text = f'{{ grad=None | value={node.value: .2f} | {node.op}}}'
+            return dict(label=node_text, shape='record', fontsize='10', fillcolor='gray94', style='filled, rounded')
+        elif node_type == 'input':
+            if node.label == '':
+                node_text = f'input={node.value: .2f}'
+            else:
+                node_text = f'{node.label}={node.value: .2f}'
+            return dict(label=node_text, shape='oval', fontsize='10')
+    
+    def _backward_attr():
+        attr = _forward_attr()
+        attr['label'] = attr['label'].replace('grad=None', f'grad={node.grad: .2f}')
+        if not node.requires_grad:
+            attr['style'] = 'dashed'
+        # 如果向后传播的梯度要么等于0，要么传给不需要梯度的节点，那么该节点用虚线表示
+        grad_back = [v if k.requires_grad else 0 for (k, v) in node.back_prop.items()]
+        if len(grad_back) > 0 and sum(grad_back) == 0:
+            attr['style'] = 'dashed'
+        return attr 
+    
+    if direction == 'forward':
+        return _forward_attr()
+    else:
+        return _backward_attr()
+    
+    
+def _get_node_type(node):
+    """
+    决定节点的类型，计算节点、参数以及输入数据
+    """
+    if node.op is not None:
+        return 'computation'
+    if node.requires_grad:
+        return 'param'
+    return 'input'
+
+
+def _trace(root):
+    """
+    遍历图中的所有点和边
+    """
+    nodes, edges = set(), set()
+    def _build(v):
+        if v not in nodes:
+            nodes.add(v)
+            for prev in v.prevs:
+                edges.add((prev, v))
+                _build(prev)
+    _build(root)
+    return nodes, edges
+
+
+def _draw_node(graph, node, direction='forward'):
+    """
+    画节点
+    """
+    node_attr = _get_node_attr(node, direction)
+    uid = str(id(node)) + direction
+    graph.node(name=uid, **node_attr)
+
+
+def _draw_edge(graph, n1, n2, direction='forward'):
+    """
+    画边
+    """
+    uid1 = str(id(n1)) + direction
+    uid2 = str(id(n2)) + direction
+    def _draw_back_edge():
+        if n1.requires_grad and n2.requires_grad:
+            grad = n2.back_prop.get(n1, None)
+            if grad is None:
+                graph.edge(uid2, uid1, arrowhead='none', color='deepskyblue')   
+            elif grad == 0:
+                graph.edge(uid2, uid1, style='dashed', label=f'{grad: .2f}', color='deepskyblue')
+            else: 
+                graph.edge(uid2, uid1, label=f'{grad: .2f}', color='deepskyblue')
+        else:
+            graph.edge(uid2, uid1, style='dashed', arrowhead='none', color='deepskyblue')
+
+    if direction == 'forward':
+        graph.edge(uid1, uid2)
+    elif direction == 'backward':
+        _draw_back_edge()
+    else:
+        _draw_back_edge()
+        graph.edge(uid1, uid2)
+
+
+def draw_graph(root, direction='forward'):
+    """
+    图形化展示由root为顶点的计算图
+    参数
+    ----
+    root ：Scalar，计算图的顶点
+    direction ：str，向前传播（forward）或者反向传播（backward）
+    返回
+    ----
+    re ：Digraph，计算图
+    """
+    nodes, edges = _trace(root)
+    rankdir = 'BT' if direction == 'forward' else 'TB'
+    graph = Digraph(format='svg', graph_attr={'rankdir': rankdir})
+    for item in nodes:
+        _draw_node(graph, item, direction)
+    for n1, n2 in edges:
+        _draw_edge(graph, n1, n2, direction)
+    return graph