230_多GPU训练实现.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. 多GPU训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "from d2l import torch as d2l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.0241, -0.0077,  0.0035,  0.0003],\n",
      "        [-0.0109,  0.0084,  0.0114,  0.0123],\n",
      "        [ 0.0036, -0.0087,  0.0080,  0.0018]])\n"
     ]
    }
   ],
   "source": [
    "W = torch.randn(size=(3,4)) # 从标准正态分布（均值为0，方差为1）中抽取的一组随机数。\n",
    "scale = 0.01\n",
    "print(W * scale)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 简单网络\n",
    "scale = 0.01\n",
    "W1 = torch.randn(size=(20,1,3,3)) * scale\n",
    "b1 = torch.zeros(20)\n",
    "W2 = torch.randn(size=(50,20,5,5)) * scale\n",
    "b2 = torch.zeros(50)\n",
    "W3 = torch.randn(size=(800,128)) * scale\n",
    "b3 = torch.zeros(128)\n",
    "W4 = torch.randn(size=(128,10)) * scale\n",
    "b4 = torch.zeros(10)\n",
    "params = [W1,b1,W2,b2,W3,b3,W4,b4]\n",
    "\n",
    "def lenet(X, params):\n",
    "    h1_conv = F.conv2d(input=X, weight=params[0],bias=params[1])\n",
    "    h1_activation = F.relu(h1_conv)\n",
    "    h1 = F.avg_pool2d(input=h1_activation, kernel_size=(2,2), stride=(2,2))\n",
    "    h2_conv = F.conv2d(input=h1, weight=params[2], bias=params[3])\n",
    "    h2_activation = F.relu(h2_conv)\n",
    "    h2 = F.avg_pool2d(input=h2_activation,kernel_size=(2,2),stride=(2,2))\n",
    "    h2 = h2.reshape(h2.shape[0],-1)\n",
    "    h3_linear = torch.mm(h2, params[4]) + params[5]\n",
    "    h3 = F.relu(h3_linear)\n",
    "    y_hat = torch.mm(h3, params[6]) + params[7]\n",
    "    return y_hat\n",
    "\n",
    "loss = nn.CrossEntropyLoss(reduction='none')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b1 weight： tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "       device='cuda:0', requires_grad=True)\n",
      "b1 grad： None\n"
     ]
    }
   ],
   "source": [
    "# 向多个设备分发参数\n",
    "def get_params(params, device):\n",
    "    new_params = [p.clone().to(device) for p in params] # 把params中所有参数挪到GPU上  \n",
    "    for p in new_params:\n",
    "        p.requires_grad_() # 每一个参数都要计算梯度\n",
    "    return new_params\n",
    "\n",
    "new_params = get_params(params, d2l.try_gpu(0))\n",
    "print('b1 weight：', new_params[1])\n",
    "print('b1 grad：', new_params[1].grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "before allreduce:\n",
      " tensor([[1., 1.]], device='cuda:0') \n",
      " tensor([[2., 2.]])\n",
      "before allreduce:\n",
      " tensor([[3., 3.]], device='cuda:0') \n",
      " tensor([[3., 3.]])\n"
     ]
    }
   ],
   "source": [
    "# allreduce函数将所有向量相加，并将结果广播给所有GPU\n",
    "def allreduce(data): # 如果有四个GPU的话，data这个list里面就有四个元素\n",
    "    for i in range(1, len(data)):\n",
    "        data[0][:] += data[i].to(data[0].device) # 把其余三个GPU上元素，拷贝到0号GPU上\n",
    "    for i in range(1, len(data)):\n",
    "        data[i] = data[0].to(data[i].device) # 把相加后的结果复制回所有GPU上\n",
    "        \n",
    "data = [torch.ones((1,2),device=d2l.try_gpu(i)) * (i + 1) for i in range(2)]     \n",
    "print('before allreduce:\\n', data[0], '\\n', data[1])\n",
    "allreduce(data) # allreduce函数可以用于各个GPU的梯度加起来，然后各个GPU拿到合梯度\n",
    "print('before allreduce:\\n', data[0], '\\n', data[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input: tensor([[ 0,  1,  2,  3,  4],\n",
      "        [ 5,  6,  7,  8,  9],\n",
      "        [10, 11, 12, 13, 14],\n",
      "        [15, 16, 17, 18, 19]])\n",
      "load into [device(type='cuda', index=0), device(type='cuda', index=1)]\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-67c72ea6039f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'input:'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'load into'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdevices\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m \u001b[0msplit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparallel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscatter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdevices\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# 将data均匀的切开，放到不同的GPU上\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'output:'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\nn\\parallel\\scatter_gather.py\u001b[0m in \u001b[0;36mscatter\u001b[1;34m(inputs, target_gpus, dim)\u001b[0m\n\u001b[0;32m     34\u001b[0m     \u001b[1;31m# None, clearing the cell\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m         \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mscatter_map\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m     \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m         \u001b[0mscatter_map\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\nn\\parallel\\scatter_gather.py\u001b[0m in \u001b[0;36mscatter_map\u001b[1;34m(obj)\u001b[0m\n\u001b[0;32m     17\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mscatter_map\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 19\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mScatter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget_gpus\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     20\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mis_namedtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     21\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0margs\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mmap\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscatter_map\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\nn\\parallel\\_functions.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(ctx, target_gpus, chunk_sizes, dim, input)\u001b[0m\n\u001b[0;32m     93\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_available\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minput_device\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m             \u001b[1;31m# Perform CPU to GPU copies in a background stream\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 95\u001b[1;33m             \u001b[0mstreams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0m_get_stream\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mdevice\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtarget_gpus\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     96\u001b[0m         \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcomm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscatter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_gpus\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchunk_sizes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstreams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     97\u001b[0m         \u001b[1;31m# Synchronize with the copy stream\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\nn\\parallel\\_functions.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m     93\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_available\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minput_device\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m             \u001b[1;31m# Perform CPU to GPU copies in a background stream\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 95\u001b[1;33m             \u001b[0mstreams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0m_get_stream\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mdevice\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtarget_gpus\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     96\u001b[0m         \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcomm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscatter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_gpus\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchunk_sizes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstreams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     97\u001b[0m         \u001b[1;31m# Synchronize with the copy stream\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\nn\\parallel\\_functions.py\u001b[0m in \u001b[0;36m_get_stream\u001b[1;34m(device)\u001b[0m\n\u001b[0;32m    120\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0m_streams\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    121\u001b[0m         \u001b[0m_streams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 122\u001b[1;33m     \u001b[1;32mif\u001b[0m \u001b[0m_streams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    123\u001b[0m         \u001b[0m_streams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mStream\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    124\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0m_streams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "# 将一个小批量数据均匀地分布在多个GPU上\n",
    "data = torch.arange(20).reshape(4,5)\n",
    "devices = [torch.device('cuda:0'),torch.device('cuda:1')]\n",
    "print('input:',data)\n",
    "print('load into',devices)\n",
    "split = nn.parallel.scatter(data,devices) # 将data均匀的切开，放到不同的GPU上   \n",
    "print('output:',split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_batch(X, y, devices):\n",
    "    \"\"\"将X和y拆分到多个设备上\"\"\"\n",
    "    assert X.shape[0] == y.shape[0] # 确定样本数与标号数是一样的\n",
    "    return (nn.parallel.scatter(X, devices), nn.parallel.scatter(y, devices)) # 样本与标号都均匀分布到不同GPU上    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 在一个小批量上实现多GPU训练\n",
    "def train_batch(X, y, device_params, devices, lr):\n",
    "    X_shards, y_shards = split_batch(X, y, devices) # 把X，y分到不同GPU上\n",
    "    # 拿到每一个GPU上的X_shard、y_shard以及对应的device_W\n",
    "    # sum是对每一个GPU上所有样本的损失求和\n",
    "    # ls返回的是每一个GPU上对应的损失\n",
    "    ls = [loss(lenet(X_shard, device_W),y_shard).sum() \n",
    "          for X_shard, y_shard, device_W in zip(X_shards, y_shards, device_params)]                         \n",
    "    for l in ls:\n",
    "        l.backward() # 对每一个GPU的loss做负反馈，算梯度\n",
    "    with torch.no_grad():\n",
    "        for i in range(len(device_params[0])):\n",
    "            # i是层，c是GPU\n",
    "            allreduce([device_params[c][i].grad for c in range(len(devices))])\n",
    "        for param in device_params: # 拿到所有梯度后，对每一个GPU做SGD\n",
    "            d2l.sgd(param, lr, X.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练\n",
    "def train(num_gpus, batch_size, lr):\n",
    "    train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)\n",
    "    devices = [d2l.try_gpu(i) for i in range(num_gpus)] # 创建多少个GPU\n",
    "    device_params = [get_params(params, d) for d in devices] # 初始化的params复制到每个GPU上\n",
    "    num_epochs = 10\n",
    "    animator = d2l.Animator('epoch','test acc',xlim=[1,num_epochs])\n",
    "    timer = d2l.Timer()\n",
    "    for epoch in range(num_epochs):\n",
    "        timer.start()\n",
    "        for X, y in train_iter:\n",
    "            train_batch(X, y, device_params, devices, lr)\n",
    "            torch.cuda.synchronize() # 等待所有的GPU算完\n",
    "        timer.stop()\n",
    "        animator.add(epoch+1,(d2l.evaluate_accuracy_gpu(\n",
    "            lambda x: lenet(x,device_params[0]),test_iter,devices[0]),))  \n",
    "    print(f'test acc: {animator.Y[0][-1]:.2f}, {timer.avg():.1f} sec/epoch'\n",
    "         f'on {str(devices)}')   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function synchronize in module torch.cuda:\n",
      "\n",
      "synchronize(device:Union[torch.device, str, int, NoneType]=None) -> None\n",
      "    Waits for all kernels in all streams on a CUDA device to complete.\n",
      "    \n",
      "    Args:\n",
      "        device (torch.device or int, optional): device for which to synchronize.\n",
      "            It uses the current device, given by :func:`~torch.cuda.current_device`,\n",
      "            if :attr:`device` is ``None`` (default).\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(torch.cuda.synchronize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test acc: 0.84, 3.7 sec/epochon [device(type='cuda', index=0)]\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Created with matplotlib (https://matplotlib.org/) -->\r\n",
       "<svg height=\"180.65625pt\" version=\"1.1\" viewBox=\"0 0 252.64375 180.65625\" width=\"252.64375pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       " <metadata>\r\n",
       "  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\r\n",
       "   <cc:Work>\r\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\r\n",
       "    <dc:date>2022-06-11T11:55:34.768472</dc:date>\r\n",
       "    <dc:format>image/svg+xml</dc:format>\r\n",
       "    <dc:creator>\r\n",
       "     <cc:Agent>\r\n",
       "      <dc:title>Matplotlib v3.3.4, https://matplotlib.org/</dc:title>\r\n",
       "     </cc:Agent>\r\n",
       "    </dc:creator>\r\n",
       "   </cc:Work>\r\n",
       "  </rdf:RDF>\r\n",
       " </metadata>\r\n",
       " <defs>\r\n",
       "  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\r\n",
       " </defs>\r\n",
       " <g id=\"figure_1\">\r\n",
       "  <g id=\"patch_1\">\r\n",
       "   <path d=\"M 0 180.65625 \r\n",
       "L 252.64375 180.65625 \r\n",
       "L 252.64375 0 \r\n",
       "L 0 0 \r\n",
       "z\r\n",
       "\" style=\"fill:none;\"/>\r\n",
       "  </g>\r\n",
       "  <g id=\"axes_1\">\r\n",
       "   <g id=\"patch_2\">\r\n",
       "    <path d=\"M 43.78125 143.1 \r\n",
       "L 239.08125 143.1 \r\n",
       "L 239.08125 7.2 \r\n",
       "L 43.78125 7.2 \r\n",
       "z\r\n",
       "\" style=\"fill:#ffffff;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"matplotlib.axis_1\">\r\n",
       "    <g id=\"xtick_1\">\r\n",
       "     <g id=\"line2d_1\">\r\n",
       "      <path clip-path=\"url(#p1ab6f81cfd)\" d=\"M 65.48125 143.1 \r\n",
       "L 65.48125 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_2\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L 0 3.5 \r\n",
       "\" id=\"m3caea1796b\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"65.48125\" xlink:href=\"#m3caea1796b\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_1\">\r\n",
       "      <!-- 2 -->\r\n",
       "      <g transform=\"translate(62.3 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 19.1875 8.296875 \r\n",
       "L 53.609375 8.296875 \r\n",
       "L 53.609375 0 \r\n",
       "L 7.328125 0 \r\n",
       "L 7.328125 8.296875 \r\n",
       "Q 12.9375 14.109375 22.625 23.890625 \r\n",
       "Q 32.328125 33.6875 34.8125 36.53125 \r\n",
       "Q 39.546875 41.84375 41.421875 45.53125 \r\n",
       "Q 43.3125 49.21875 43.3125 52.78125 \r\n",
       "Q 43.3125 58.59375 39.234375 62.25 \r\n",
       "Q 35.15625 65.921875 28.609375 65.921875 \r\n",
       "Q 23.96875 65.921875 18.8125 64.3125 \r\n",
       "Q 13.671875 62.703125 7.8125 59.421875 \r\n",
       "L 7.8125 69.390625 \r\n",
       "Q 13.765625 71.78125 18.9375 73 \r\n",
       "Q 24.125 74.21875 28.421875 74.21875 \r\n",
       "Q 39.75 74.21875 46.484375 68.546875 \r\n",
       "Q 53.21875 62.890625 53.21875 53.421875 \r\n",
       "Q 53.21875 48.921875 51.53125 44.890625 \r\n",
       "Q 49.859375 40.875 45.40625 35.40625 \r\n",
       "Q 44.1875 33.984375 37.640625 27.21875 \r\n",
       "Q 31.109375 20.453125 19.1875 8.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-50\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_2\">\r\n",
       "     <g id=\"line2d_3\">\r\n",
       "      <path clip-path=\"url(#p1ab6f81cfd)\" d=\"M 108.88125 143.1 \r\n",
       "L 108.88125 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_4\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"108.88125\" xlink:href=\"#m3caea1796b\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_2\">\r\n",
       "      <!-- 4 -->\r\n",
       "      <g transform=\"translate(105.7 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 37.796875 64.3125 \r\n",
       "L 12.890625 25.390625 \r\n",
       "L 37.796875 25.390625 \r\n",
       "z\r\n",
       "M 35.203125 72.90625 \r\n",
       "L 47.609375 72.90625 \r\n",
       "L 47.609375 25.390625 \r\n",
       "L 58.015625 25.390625 \r\n",
       "L 58.015625 17.1875 \r\n",
       "L 47.609375 17.1875 \r\n",
       "L 47.609375 0 \r\n",
       "L 37.796875 0 \r\n",
       "L 37.796875 17.1875 \r\n",
       "L 4.890625 17.1875 \r\n",
       "L 4.890625 26.703125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-52\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-52\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_3\">\r\n",
       "     <g id=\"line2d_5\">\r\n",
       "      <path clip-path=\"url(#p1ab6f81cfd)\" d=\"M 152.28125 143.1 \r\n",
       "L 152.28125 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_6\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"152.28125\" xlink:href=\"#m3caea1796b\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_3\">\r\n",
       "      <!-- 6 -->\r\n",
       "      <g transform=\"translate(149.1 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 33.015625 40.375 \r\n",
       "Q 26.375 40.375 22.484375 35.828125 \r\n",
       "Q 18.609375 31.296875 18.609375 23.390625 \r\n",
       "Q 18.609375 15.53125 22.484375 10.953125 \r\n",
       "Q 26.375 6.390625 33.015625 6.390625 \r\n",
       "Q 39.65625 6.390625 43.53125 10.953125 \r\n",
       "Q 47.40625 15.53125 47.40625 23.390625 \r\n",
       "Q 47.40625 31.296875 43.53125 35.828125 \r\n",
       "Q 39.65625 40.375 33.015625 40.375 \r\n",
       "z\r\n",
       "M 52.59375 71.296875 \r\n",
       "L 52.59375 62.3125 \r\n",
       "Q 48.875 64.0625 45.09375 64.984375 \r\n",
       "Q 41.3125 65.921875 37.59375 65.921875 \r\n",
       "Q 27.828125 65.921875 22.671875 59.328125 \r\n",
       "Q 17.53125 52.734375 16.796875 39.40625 \r\n",
       "Q 19.671875 43.65625 24.015625 45.921875 \r\n",
       "Q 28.375 48.1875 33.59375 48.1875 \r\n",
       "Q 44.578125 48.1875 50.953125 41.515625 \r\n",
       "Q 57.328125 34.859375 57.328125 23.390625 \r\n",
       "Q 57.328125 12.15625 50.6875 5.359375 \r\n",
       "Q 44.046875 -1.421875 33.015625 -1.421875 \r\n",
       "Q 20.359375 -1.421875 13.671875 8.265625 \r\n",
       "Q 6.984375 17.96875 6.984375 36.375 \r\n",
       "Q 6.984375 53.65625 15.1875 63.9375 \r\n",
       "Q 23.390625 74.21875 37.203125 74.21875 \r\n",
       "Q 40.921875 74.21875 44.703125 73.484375 \r\n",
       "Q 48.484375 72.75 52.59375 71.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-54\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-54\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_4\">\r\n",
       "     <g id=\"line2d_7\">\r\n",
       "      <path clip-path=\"url(#p1ab6f81cfd)\" d=\"M 195.68125 143.1 \r\n",
       "L 195.68125 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_8\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"195.68125\" xlink:href=\"#m3caea1796b\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_4\">\r\n",
       "      <!-- 8 -->\r\n",
       "      <g transform=\"translate(192.5 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 31.78125 34.625 \r\n",
       "Q 24.75 34.625 20.71875 30.859375 \r\n",
       "Q 16.703125 27.09375 16.703125 20.515625 \r\n",
       "Q 16.703125 13.921875 20.71875 10.15625 \r\n",
       "Q 24.75 6.390625 31.78125 6.390625 \r\n",
       "Q 38.8125 6.390625 42.859375 10.171875 \r\n",
       "Q 46.921875 13.96875 46.921875 20.515625 \r\n",
       "Q 46.921875 27.09375 42.890625 30.859375 \r\n",
       "Q 38.875 34.625 31.78125 34.625 \r\n",
       "z\r\n",
       "M 21.921875 38.8125 \r\n",
       "Q 15.578125 40.375 12.03125 44.71875 \r\n",
       "Q 8.5 49.078125 8.5 55.328125 \r\n",
       "Q 8.5 64.0625 14.71875 69.140625 \r\n",
       "Q 20.953125 74.21875 31.78125 74.21875 \r\n",
       "Q 42.671875 74.21875 48.875 69.140625 \r\n",
       "Q 55.078125 64.0625 55.078125 55.328125 \r\n",
       "Q 55.078125 49.078125 51.53125 44.71875 \r\n",
       "Q 48 40.375 41.703125 38.8125 \r\n",
       "Q 48.828125 37.15625 52.796875 32.3125 \r\n",
       "Q 56.78125 27.484375 56.78125 20.515625 \r\n",
       "Q 56.78125 9.90625 50.3125 4.234375 \r\n",
       "Q 43.84375 -1.421875 31.78125 -1.421875 \r\n",
       "Q 19.734375 -1.421875 13.25 4.234375 \r\n",
       "Q 6.78125 9.90625 6.78125 20.515625 \r\n",
       "Q 6.78125 27.484375 10.78125 32.3125 \r\n",
       "Q 14.796875 37.15625 21.921875 38.8125 \r\n",
       "z\r\n",
       "M 18.3125 54.390625 \r\n",
       "Q 18.3125 48.734375 21.84375 45.5625 \r\n",
       "Q 25.390625 42.390625 31.78125 42.390625 \r\n",
       "Q 38.140625 42.390625 41.71875 45.5625 \r\n",
       "Q 45.3125 48.734375 45.3125 54.390625 \r\n",
       "Q 45.3125 60.0625 41.71875 63.234375 \r\n",
       "Q 38.140625 66.40625 31.78125 66.40625 \r\n",
       "Q 25.390625 66.40625 21.84375 63.234375 \r\n",
       "Q 18.3125 60.0625 18.3125 54.390625 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-56\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-56\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_5\">\r\n",
       "     <g id=\"line2d_9\">\r\n",
       "      <path clip-path=\"url(#p1ab6f81cfd)\" d=\"M 239.08125 143.1 \r\n",
       "L 239.08125 7.2 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_10\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"239.08125\" xlink:href=\"#m3caea1796b\" y=\"143.1\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_5\">\r\n",
       "      <!-- 10 -->\r\n",
       "      <g transform=\"translate(232.71875 157.698438)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 12.40625 8.296875 \r\n",
       "L 28.515625 8.296875 \r\n",
       "L 28.515625 63.921875 \r\n",
       "L 10.984375 60.40625 \r\n",
       "L 10.984375 69.390625 \r\n",
       "L 28.421875 72.90625 \r\n",
       "L 38.28125 72.90625 \r\n",
       "L 38.28125 8.296875 \r\n",
       "L 54.390625 8.296875 \r\n",
       "L 54.390625 0 \r\n",
       "L 12.40625 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-49\"/>\r\n",
       "        <path d=\"M 31.78125 66.40625 \r\n",
       "Q 24.171875 66.40625 20.328125 58.90625 \r\n",
       "Q 16.5 51.421875 16.5 36.375 \r\n",
       "Q 16.5 21.390625 20.328125 13.890625 \r\n",
       "Q 24.171875 6.390625 31.78125 6.390625 \r\n",
       "Q 39.453125 6.390625 43.28125 13.890625 \r\n",
       "Q 47.125 21.390625 47.125 36.375 \r\n",
       "Q 47.125 51.421875 43.28125 58.90625 \r\n",
       "Q 39.453125 66.40625 31.78125 66.40625 \r\n",
       "z\r\n",
       "M 31.78125 74.21875 \r\n",
       "Q 44.046875 74.21875 50.515625 64.515625 \r\n",
       "Q 56.984375 54.828125 56.984375 36.375 \r\n",
       "Q 56.984375 17.96875 50.515625 8.265625 \r\n",
       "Q 44.046875 -1.421875 31.78125 -1.421875 \r\n",
       "Q 19.53125 -1.421875 13.0625 8.265625 \r\n",
       "Q 6.59375 17.96875 6.59375 36.375 \r\n",
       "Q 6.59375 54.828125 13.0625 64.515625 \r\n",
       "Q 19.53125 74.21875 31.78125 74.21875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-48\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"text_6\">\r\n",
       "     <!-- epoch -->\r\n",
       "     <g transform=\"translate(126.203125 171.376563)scale(0.1 -0.1)\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 56.203125 29.59375 \r\n",
       "L 56.203125 25.203125 \r\n",
       "L 14.890625 25.203125 \r\n",
       "Q 15.484375 15.921875 20.484375 11.0625 \r\n",
       "Q 25.484375 6.203125 34.421875 6.203125 \r\n",
       "Q 39.59375 6.203125 44.453125 7.46875 \r\n",
       "Q 49.3125 8.734375 54.109375 11.28125 \r\n",
       "L 54.109375 2.78125 \r\n",
       "Q 49.265625 0.734375 44.1875 -0.34375 \r\n",
       "Q 39.109375 -1.421875 33.890625 -1.421875 \r\n",
       "Q 20.796875 -1.421875 13.15625 6.1875 \r\n",
       "Q 5.515625 13.8125 5.515625 26.8125 \r\n",
       "Q 5.515625 40.234375 12.765625 48.109375 \r\n",
       "Q 20.015625 56 32.328125 56 \r\n",
       "Q 43.359375 56 49.78125 48.890625 \r\n",
       "Q 56.203125 41.796875 56.203125 29.59375 \r\n",
       "z\r\n",
       "M 47.21875 32.234375 \r\n",
       "Q 47.125 39.59375 43.09375 43.984375 \r\n",
       "Q 39.0625 48.390625 32.421875 48.390625 \r\n",
       "Q 24.90625 48.390625 20.390625 44.140625 \r\n",
       "Q 15.875 39.890625 15.1875 32.171875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-101\"/>\r\n",
       "       <path d=\"M 18.109375 8.203125 \r\n",
       "L 18.109375 -20.796875 \r\n",
       "L 9.078125 -20.796875 \r\n",
       "L 9.078125 54.6875 \r\n",
       "L 18.109375 54.6875 \r\n",
       "L 18.109375 46.390625 \r\n",
       "Q 20.953125 51.265625 25.265625 53.625 \r\n",
       "Q 29.59375 56 35.59375 56 \r\n",
       "Q 45.5625 56 51.78125 48.09375 \r\n",
       "Q 58.015625 40.1875 58.015625 27.296875 \r\n",
       "Q 58.015625 14.40625 51.78125 6.484375 \r\n",
       "Q 45.5625 -1.421875 35.59375 -1.421875 \r\n",
       "Q 29.59375 -1.421875 25.265625 0.953125 \r\n",
       "Q 20.953125 3.328125 18.109375 8.203125 \r\n",
       "z\r\n",
       "M 48.6875 27.296875 \r\n",
       "Q 48.6875 37.203125 44.609375 42.84375 \r\n",
       "Q 40.53125 48.484375 33.40625 48.484375 \r\n",
       "Q 26.265625 48.484375 22.1875 42.84375 \r\n",
       "Q 18.109375 37.203125 18.109375 27.296875 \r\n",
       "Q 18.109375 17.390625 22.1875 11.75 \r\n",
       "Q 26.265625 6.109375 33.40625 6.109375 \r\n",
       "Q 40.53125 6.109375 44.609375 11.75 \r\n",
       "Q 48.6875 17.390625 48.6875 27.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-112\"/>\r\n",
       "       <path d=\"M 30.609375 48.390625 \r\n",
       "Q 23.390625 48.390625 19.1875 42.75 \r\n",
       "Q 14.984375 37.109375 14.984375 27.296875 \r\n",
       "Q 14.984375 17.484375 19.15625 11.84375 \r\n",
       "Q 23.34375 6.203125 30.609375 6.203125 \r\n",
       "Q 37.796875 6.203125 41.984375 11.859375 \r\n",
       "Q 46.1875 17.53125 46.1875 27.296875 \r\n",
       "Q 46.1875 37.015625 41.984375 42.703125 \r\n",
       "Q 37.796875 48.390625 30.609375 48.390625 \r\n",
       "z\r\n",
       "M 30.609375 56 \r\n",
       "Q 42.328125 56 49.015625 48.375 \r\n",
       "Q 55.71875 40.765625 55.71875 27.296875 \r\n",
       "Q 55.71875 13.875 49.015625 6.21875 \r\n",
       "Q 42.328125 -1.421875 30.609375 -1.421875 \r\n",
       "Q 18.84375 -1.421875 12.171875 6.21875 \r\n",
       "Q 5.515625 13.875 5.515625 27.296875 \r\n",
       "Q 5.515625 40.765625 12.171875 48.375 \r\n",
       "Q 18.84375 56 30.609375 56 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-111\"/>\r\n",
       "       <path d=\"M 48.78125 52.59375 \r\n",
       "L 48.78125 44.1875 \r\n",
       "Q 44.96875 46.296875 41.140625 47.34375 \r\n",
       "Q 37.3125 48.390625 33.40625 48.390625 \r\n",
       "Q 24.65625 48.390625 19.8125 42.84375 \r\n",
       "Q 14.984375 37.3125 14.984375 27.296875 \r\n",
       "Q 14.984375 17.28125 19.8125 11.734375 \r\n",
       "Q 24.65625 6.203125 33.40625 6.203125 \r\n",
       "Q 37.3125 6.203125 41.140625 7.25 \r\n",
       "Q 44.96875 8.296875 48.78125 10.40625 \r\n",
       "L 48.78125 2.09375 \r\n",
       "Q 45.015625 0.34375 40.984375 -0.53125 \r\n",
       "Q 36.96875 -1.421875 32.421875 -1.421875 \r\n",
       "Q 20.0625 -1.421875 12.78125 6.34375 \r\n",
       "Q 5.515625 14.109375 5.515625 27.296875 \r\n",
       "Q 5.515625 40.671875 12.859375 48.328125 \r\n",
       "Q 20.21875 56 33.015625 56 \r\n",
       "Q 37.15625 56 41.109375 55.140625 \r\n",
       "Q 45.0625 54.296875 48.78125 52.59375 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-99\"/>\r\n",
       "       <path d=\"M 54.890625 33.015625 \r\n",
       "L 54.890625 0 \r\n",
       "L 45.90625 0 \r\n",
       "L 45.90625 32.71875 \r\n",
       "Q 45.90625 40.484375 42.875 44.328125 \r\n",
       "Q 39.84375 48.1875 33.796875 48.1875 \r\n",
       "Q 26.515625 48.1875 22.3125 43.546875 \r\n",
       "Q 18.109375 38.921875 18.109375 30.90625 \r\n",
       "L 18.109375 0 \r\n",
       "L 9.078125 0 \r\n",
       "L 9.078125 75.984375 \r\n",
       "L 18.109375 75.984375 \r\n",
       "L 18.109375 46.1875 \r\n",
       "Q 21.34375 51.125 25.703125 53.5625 \r\n",
       "Q 30.078125 56 35.796875 56 \r\n",
       "Q 45.21875 56 50.046875 50.171875 \r\n",
       "Q 54.890625 44.34375 54.890625 33.015625 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-104\"/>\r\n",
       "      </defs>\r\n",
       "      <use xlink:href=\"#DejaVuSans-101\"/>\r\n",
       "      <use x=\"61.523438\" xlink:href=\"#DejaVuSans-112\"/>\r\n",
       "      <use x=\"125\" xlink:href=\"#DejaVuSans-111\"/>\r\n",
       "      <use x=\"186.181641\" xlink:href=\"#DejaVuSans-99\"/>\r\n",
       "      <use x=\"241.162109\" xlink:href=\"#DejaVuSans-104\"/>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"matplotlib.axis_2\">\r\n",
       "    <g id=\"ytick_1\">\r\n",
       "     <g id=\"line2d_11\">\r\n",
       "      <path clip-path=\"url(#p1ab6f81cfd)\" d=\"M 43.78125 120.154886 \r\n",
       "L 239.08125 120.154886 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_12\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L -3.5 0 \r\n",
       "\" id=\"m204ffd59a7\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m204ffd59a7\" y=\"120.154886\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_7\">\r\n",
       "      <!-- 0.2 -->\r\n",
       "      <g transform=\"translate(20.878125 123.954105)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 10.6875 12.40625 \r\n",
       "L 21 12.40625 \r\n",
       "L 21 0 \r\n",
       "L 10.6875 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-46\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_2\">\r\n",
       "     <g id=\"line2d_13\">\r\n",
       "      <path clip-path=\"url(#p1ab6f81cfd)\" d=\"M 43.78125 86.619203 \r\n",
       "L 239.08125 86.619203 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_14\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m204ffd59a7\" y=\"86.619203\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_8\">\r\n",
       "      <!-- 0.4 -->\r\n",
       "      <g transform=\"translate(20.878125 90.418422)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-52\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_3\">\r\n",
       "     <g id=\"line2d_15\">\r\n",
       "      <path clip-path=\"url(#p1ab6f81cfd)\" d=\"M 43.78125 53.083521 \r\n",
       "L 239.08125 53.083521 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_16\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m204ffd59a7\" y=\"53.083521\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_9\">\r\n",
       "      <!-- 0.6 -->\r\n",
       "      <g transform=\"translate(20.878125 56.88274)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-54\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_4\">\r\n",
       "     <g id=\"line2d_17\">\r\n",
       "      <path clip-path=\"url(#p1ab6f81cfd)\" d=\"M 43.78125 19.547838 \r\n",
       "L 239.08125 19.547838 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_18\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m204ffd59a7\" y=\"19.547838\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_10\">\r\n",
       "      <!-- 0.8 -->\r\n",
       "      <g transform=\"translate(20.878125 23.347057)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"text_11\">\r\n",
       "     <!-- test acc -->\r\n",
       "     <g transform=\"translate(14.798438 94.903125)rotate(-90)scale(0.1 -0.1)\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 18.3125 70.21875 \r\n",
       "L 18.3125 54.6875 \r\n",
       "L 36.8125 54.6875 \r\n",
       "L 36.8125 47.703125 \r\n",
       "L 18.3125 47.703125 \r\n",
       "L 18.3125 18.015625 \r\n",
       "Q 18.3125 11.328125 20.140625 9.421875 \r\n",
       "Q 21.96875 7.515625 27.59375 7.515625 \r\n",
       "L 36.8125 7.515625 \r\n",
       "L 36.8125 0 \r\n",
       "L 27.59375 0 \r\n",
       "Q 17.1875 0 13.234375 3.875 \r\n",
       "Q 9.28125 7.765625 9.28125 18.015625 \r\n",
       "L 9.28125 47.703125 \r\n",
       "L 2.6875 47.703125 \r\n",
       "L 2.6875 54.6875 \r\n",
       "L 9.28125 54.6875 \r\n",
       "L 9.28125 70.21875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-116\"/>\r\n",
       "       <path d=\"M 44.28125 53.078125 \r\n",
       "L 44.28125 44.578125 \r\n",
       "Q 40.484375 46.53125 36.375 47.5 \r\n",
       "Q 32.28125 48.484375 27.875 48.484375 \r\n",
       "Q 21.1875 48.484375 17.84375 46.4375 \r\n",
       "Q 14.5 44.390625 14.5 40.28125 \r\n",
       "Q 14.5 37.15625 16.890625 35.375 \r\n",
       "Q 19.28125 33.59375 26.515625 31.984375 \r\n",
       "L 29.59375 31.296875 \r\n",
       "Q 39.15625 29.25 43.1875 25.515625 \r\n",
       "Q 47.21875 21.78125 47.21875 15.09375 \r\n",
       "Q 47.21875 7.46875 41.1875 3.015625 \r\n",
       "Q 35.15625 -1.421875 24.609375 -1.421875 \r\n",
       "Q 20.21875 -1.421875 15.453125 -0.5625 \r\n",
       "Q 10.6875 0.296875 5.421875 2 \r\n",
       "L 5.421875 11.28125 \r\n",
       "Q 10.40625 8.6875 15.234375 7.390625 \r\n",
       "Q 20.0625 6.109375 24.8125 6.109375 \r\n",
       "Q 31.15625 6.109375 34.5625 8.28125 \r\n",
       "Q 37.984375 10.453125 37.984375 14.40625 \r\n",
       "Q 37.984375 18.0625 35.515625 20.015625 \r\n",
       "Q 33.0625 21.96875 24.703125 23.78125 \r\n",
       "L 21.578125 24.515625 \r\n",
       "Q 13.234375 26.265625 9.515625 29.90625 \r\n",
       "Q 5.8125 33.546875 5.8125 39.890625 \r\n",
       "Q 5.8125 47.609375 11.28125 51.796875 \r\n",
       "Q 16.75 56 26.8125 56 \r\n",
       "Q 31.78125 56 36.171875 55.265625 \r\n",
       "Q 40.578125 54.546875 44.28125 53.078125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-115\"/>\r\n",
       "       <path id=\"DejaVuSans-32\"/>\r\n",
       "       <path d=\"M 34.28125 27.484375 \r\n",
       "Q 23.390625 27.484375 19.1875 25 \r\n",
       "Q 14.984375 22.515625 14.984375 16.5 \r\n",
       "Q 14.984375 11.71875 18.140625 8.90625 \r\n",
       "Q 21.296875 6.109375 26.703125 6.109375 \r\n",
       "Q 34.1875 6.109375 38.703125 11.40625 \r\n",
       "Q 43.21875 16.703125 43.21875 25.484375 \r\n",
       "L 43.21875 27.484375 \r\n",
       "z\r\n",
       "M 52.203125 31.203125 \r\n",
       "L 52.203125 0 \r\n",
       "L 43.21875 0 \r\n",
       "L 43.21875 8.296875 \r\n",
       "Q 40.140625 3.328125 35.546875 0.953125 \r\n",
       "Q 30.953125 -1.421875 24.3125 -1.421875 \r\n",
       "Q 15.921875 -1.421875 10.953125 3.296875 \r\n",
       "Q 6 8.015625 6 15.921875 \r\n",
       "Q 6 25.140625 12.171875 29.828125 \r\n",
       "Q 18.359375 34.515625 30.609375 34.515625 \r\n",
       "L 43.21875 34.515625 \r\n",
       "L 43.21875 35.40625 \r\n",
       "Q 43.21875 41.609375 39.140625 45 \r\n",
       "Q 35.0625 48.390625 27.6875 48.390625 \r\n",
       "Q 23 48.390625 18.546875 47.265625 \r\n",
       "Q 14.109375 46.140625 10.015625 43.890625 \r\n",
       "L 10.015625 52.203125 \r\n",
       "Q 14.9375 54.109375 19.578125 55.046875 \r\n",
       "Q 24.21875 56 28.609375 56 \r\n",
       "Q 40.484375 56 46.34375 49.84375 \r\n",
       "Q 52.203125 43.703125 52.203125 31.203125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-97\"/>\r\n",
       "      </defs>\r\n",
       "      <use xlink:href=\"#DejaVuSans-116\"/>\r\n",
       "      <use x=\"39.208984\" xlink:href=\"#DejaVuSans-101\"/>\r\n",
       "      <use x=\"100.732422\" xlink:href=\"#DejaVuSans-115\"/>\r\n",
       "      <use x=\"152.832031\" xlink:href=\"#DejaVuSans-116\"/>\r\n",
       "      <use x=\"192.041016\" xlink:href=\"#DejaVuSans-32\"/>\r\n",
       "      <use x=\"223.828125\" xlink:href=\"#DejaVuSans-97\"/>\r\n",
       "      <use x=\"285.107422\" xlink:href=\"#DejaVuSans-99\"/>\r\n",
       "      <use x=\"340.087891\" xlink:href=\"#DejaVuSans-99\"/>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"line2d_19\">\r\n",
       "    <path clip-path=\"url(#p1ab6f81cfd)\" d=\"M 43.78125 136.922727 \r\n",
       "L 65.48125 56.487393 \r\n",
       "L 87.18125 35.309609 \r\n",
       "L 108.88125 28.300651 \r\n",
       "L 130.58125 19.178946 \r\n",
       "L 152.28125 16.311645 \r\n",
       "L 173.98125 23.823638 \r\n",
       "L 195.68125 19.715517 \r\n",
       "L 217.38125 17.116501 \r\n",
       "L 239.08125 13.377273 \r\n",
       "\" style=\"fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_3\">\r\n",
       "    <path d=\"M 43.78125 143.1 \r\n",
       "L 43.78125 7.2 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_4\">\r\n",
       "    <path d=\"M 239.08125 143.1 \r\n",
       "L 239.08125 7.2 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_5\">\r\n",
       "    <path d=\"M 43.78125 143.1 \r\n",
       "L 239.08125 143.1 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_6\">\r\n",
       "    <path d=\"M 43.78125 7.2 \r\n",
       "L 239.08125 7.2 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "  </g>\r\n",
       " </g>\r\n",
       " <defs>\r\n",
       "  <clipPath id=\"p1ab6f81cfd\">\r\n",
       "   <rect height=\"135.9\" width=\"195.3\" x=\"43.78125\" y=\"7.2\"/>\r\n",
       "  </clipPath>\r\n",
       " </defs>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 在单个GPU上运行\n",
    "train(num_gpus=1, batch_size=256, lr=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Expected a non cpu device, but got: cpu",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-ef7689592cb4>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# 增加为2个GPU\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnum_gpus\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m256\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.2\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# 本电脑只有一个GPU，呜呜呜呜\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-9-ddae8883987e>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(num_gpus, batch_size, lr)\u001b[0m\n\u001b[0;32m     10\u001b[0m         \u001b[0mtimer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtrain_iter\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m             \u001b[0mtrain_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdevice_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdevices\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msynchronize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# 等待所有的GPU算完\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m         \u001b[0mtimer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-8-055fd69e9627>\u001b[0m in \u001b[0;36mtrain_batch\u001b[1;34m(X, y, device_params, devices, lr)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# 在一个小批量上实现多GPU训练\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mtrain_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdevice_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdevices\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m     \u001b[0mX_shards\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_shards\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msplit_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdevices\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# 把X，y分到不同GPU上\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m     \u001b[1;31m# 拿到每一个GPU上的X_shard、y_shard以及对应的device_W\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[1;31m# sum是对每一个GPU上所有样本的损失求和\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-7-82122d5c5691>\u001b[0m in \u001b[0;36msplit_batch\u001b[1;34m(X, y, devices)\u001b[0m\n\u001b[0;32m      2\u001b[0m     \u001b[1;34m\"\"\"将X和y拆分到多个设备上\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m     \u001b[1;32massert\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;31m# 确定样本数与标号数是一样的\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparallel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscatter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdevices\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparallel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscatter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdevices\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# 样本与标号都均匀分布到不同GPU上\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\nn\\parallel\\scatter_gather.py\u001b[0m in \u001b[0;36mscatter\u001b[1;34m(inputs, target_gpus, dim)\u001b[0m\n\u001b[0;32m     34\u001b[0m     \u001b[1;31m# None, clearing the cell\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m         \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mscatter_map\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m     \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m         \u001b[0mscatter_map\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\nn\\parallel\\scatter_gather.py\u001b[0m in \u001b[0;36mscatter_map\u001b[1;34m(obj)\u001b[0m\n\u001b[0;32m     17\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mscatter_map\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 19\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mScatter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget_gpus\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     20\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mis_namedtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     21\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0margs\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mmap\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscatter_map\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\nn\\parallel\\_functions.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(ctx, target_gpus, chunk_sizes, dim, input)\u001b[0m\n\u001b[0;32m     87\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     88\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_gpus\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchunk_sizes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 89\u001b[1;33m         \u001b[0mtarget_gpus\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0m_get_device_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtarget_gpus\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     90\u001b[0m         \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdim\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdim\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m         \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minput_device\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_device\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;34m\"cpu\"\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\nn\\parallel\\_functions.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m     87\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     88\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_gpus\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchunk_sizes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 89\u001b[1;33m         \u001b[0mtarget_gpus\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0m_get_device_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtarget_gpus\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     90\u001b[0m         \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdim\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdim\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m         \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minput_device\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_device\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;34m\"cpu\"\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\_utils.py\u001b[0m in \u001b[0;36m_get_device_index\u001b[1;34m(device, optional, allow_cpu)\u001b[0m\n\u001b[0;32m    495\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    496\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mallow_cpu\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mdevice\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'cpu'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 497\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Expected a non cpu device, but got: {}'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    498\u001b[0m         \u001b[0mdevice_idx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mdevice\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'cpu'\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mdevice\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    499\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mint\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Expected a non cpu device, but got: cpu"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Created with matplotlib (https://matplotlib.org/) -->\r\n",
       "<svg height=\"170.777344pt\" version=\"1.1\" viewBox=\"0 0 240.554688 170.777344\" width=\"240.554688pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       " <metadata>\r\n",
       "  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\r\n",
       "   <cc:Work>\r\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\r\n",
       "    <dc:date>2022-06-11T11:55:36.768740</dc:date>\r\n",
       "    <dc:format>image/svg+xml</dc:format>\r\n",
       "    <dc:creator>\r\n",
       "     <cc:Agent>\r\n",
       "      <dc:title>Matplotlib v3.3.4, https://matplotlib.org/</dc:title>\r\n",
       "     </cc:Agent>\r\n",
       "    </dc:creator>\r\n",
       "   </cc:Work>\r\n",
       "  </rdf:RDF>\r\n",
       " </metadata>\r\n",
       " <defs>\r\n",
       "  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\r\n",
       " </defs>\r\n",
       " <g id=\"figure_1\">\r\n",
       "  <g id=\"patch_1\">\r\n",
       "   <path d=\"M 0 170.777344 \r\n",
       "L 240.554688 170.777344 \r\n",
       "L 240.554688 0 \r\n",
       "L 0 0 \r\n",
       "z\r\n",
       "\" style=\"fill:none;\"/>\r\n",
       "  </g>\r\n",
       "  <g id=\"axes_1\">\r\n",
       "   <g id=\"patch_2\">\r\n",
       "    <path d=\"M 30.103125 146.899219 \r\n",
       "L 225.403125 146.899219 \r\n",
       "L 225.403125 10.999219 \r\n",
       "L 30.103125 10.999219 \r\n",
       "z\r\n",
       "\" style=\"fill:#ffffff;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"matplotlib.axis_1\">\r\n",
       "    <g id=\"xtick_1\">\r\n",
       "     <g id=\"line2d_1\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L 0 3.5 \r\n",
       "\" id=\"mad6cee5e8b\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"30.103125\" xlink:href=\"#mad6cee5e8b\" y=\"146.899219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_1\">\r\n",
       "      <!-- 0.0 -->\r\n",
       "      <g transform=\"translate(22.151563 161.497656)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 31.78125 66.40625 \r\n",
       "Q 24.171875 66.40625 20.328125 58.90625 \r\n",
       "Q 16.5 51.421875 16.5 36.375 \r\n",
       "Q 16.5 21.390625 20.328125 13.890625 \r\n",
       "Q 24.171875 6.390625 31.78125 6.390625 \r\n",
       "Q 39.453125 6.390625 43.28125 13.890625 \r\n",
       "Q 47.125 21.390625 47.125 36.375 \r\n",
       "Q 47.125 51.421875 43.28125 58.90625 \r\n",
       "Q 39.453125 66.40625 31.78125 66.40625 \r\n",
       "z\r\n",
       "M 31.78125 74.21875 \r\n",
       "Q 44.046875 74.21875 50.515625 64.515625 \r\n",
       "Q 56.984375 54.828125 56.984375 36.375 \r\n",
       "Q 56.984375 17.96875 50.515625 8.265625 \r\n",
       "Q 44.046875 -1.421875 31.78125 -1.421875 \r\n",
       "Q 19.53125 -1.421875 13.0625 8.265625 \r\n",
       "Q 6.59375 17.96875 6.59375 36.375 \r\n",
       "Q 6.59375 54.828125 13.0625 64.515625 \r\n",
       "Q 19.53125 74.21875 31.78125 74.21875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-48\"/>\r\n",
       "        <path d=\"M 10.6875 12.40625 \r\n",
       "L 21 12.40625 \r\n",
       "L 21 0 \r\n",
       "L 10.6875 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-46\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_2\">\r\n",
       "     <g id=\"line2d_2\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"69.163125\" xlink:href=\"#mad6cee5e8b\" y=\"146.899219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_2\">\r\n",
       "      <!-- 0.2 -->\r\n",
       "      <g transform=\"translate(61.211563 161.497656)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 19.1875 8.296875 \r\n",
       "L 53.609375 8.296875 \r\n",
       "L 53.609375 0 \r\n",
       "L 7.328125 0 \r\n",
       "L 7.328125 8.296875 \r\n",
       "Q 12.9375 14.109375 22.625 23.890625 \r\n",
       "Q 32.328125 33.6875 34.8125 36.53125 \r\n",
       "Q 39.546875 41.84375 41.421875 45.53125 \r\n",
       "Q 43.3125 49.21875 43.3125 52.78125 \r\n",
       "Q 43.3125 58.59375 39.234375 62.25 \r\n",
       "Q 35.15625 65.921875 28.609375 65.921875 \r\n",
       "Q 23.96875 65.921875 18.8125 64.3125 \r\n",
       "Q 13.671875 62.703125 7.8125 59.421875 \r\n",
       "L 7.8125 69.390625 \r\n",
       "Q 13.765625 71.78125 18.9375 73 \r\n",
       "Q 24.125 74.21875 28.421875 74.21875 \r\n",
       "Q 39.75 74.21875 46.484375 68.546875 \r\n",
       "Q 53.21875 62.890625 53.21875 53.421875 \r\n",
       "Q 53.21875 48.921875 51.53125 44.890625 \r\n",
       "Q 49.859375 40.875 45.40625 35.40625 \r\n",
       "Q 44.1875 33.984375 37.640625 27.21875 \r\n",
       "Q 31.109375 20.453125 19.1875 8.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-50\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_3\">\r\n",
       "     <g id=\"line2d_3\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"108.223125\" xlink:href=\"#mad6cee5e8b\" y=\"146.899219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_3\">\r\n",
       "      <!-- 0.4 -->\r\n",
       "      <g transform=\"translate(100.271563 161.497656)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 37.796875 64.3125 \r\n",
       "L 12.890625 25.390625 \r\n",
       "L 37.796875 25.390625 \r\n",
       "z\r\n",
       "M 35.203125 72.90625 \r\n",
       "L 47.609375 72.90625 \r\n",
       "L 47.609375 25.390625 \r\n",
       "L 58.015625 25.390625 \r\n",
       "L 58.015625 17.1875 \r\n",
       "L 47.609375 17.1875 \r\n",
       "L 47.609375 0 \r\n",
       "L 37.796875 0 \r\n",
       "L 37.796875 17.1875 \r\n",
       "L 4.890625 17.1875 \r\n",
       "L 4.890625 26.703125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-52\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-52\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_4\">\r\n",
       "     <g id=\"line2d_4\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"147.283125\" xlink:href=\"#mad6cee5e8b\" y=\"146.899219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_4\">\r\n",
       "      <!-- 0.6 -->\r\n",
       "      <g transform=\"translate(139.331563 161.497656)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 33.015625 40.375 \r\n",
       "Q 26.375 40.375 22.484375 35.828125 \r\n",
       "Q 18.609375 31.296875 18.609375 23.390625 \r\n",
       "Q 18.609375 15.53125 22.484375 10.953125 \r\n",
       "Q 26.375 6.390625 33.015625 6.390625 \r\n",
       "Q 39.65625 6.390625 43.53125 10.953125 \r\n",
       "Q 47.40625 15.53125 47.40625 23.390625 \r\n",
       "Q 47.40625 31.296875 43.53125 35.828125 \r\n",
       "Q 39.65625 40.375 33.015625 40.375 \r\n",
       "z\r\n",
       "M 52.59375 71.296875 \r\n",
       "L 52.59375 62.3125 \r\n",
       "Q 48.875 64.0625 45.09375 64.984375 \r\n",
       "Q 41.3125 65.921875 37.59375 65.921875 \r\n",
       "Q 27.828125 65.921875 22.671875 59.328125 \r\n",
       "Q 17.53125 52.734375 16.796875 39.40625 \r\n",
       "Q 19.671875 43.65625 24.015625 45.921875 \r\n",
       "Q 28.375 48.1875 33.59375 48.1875 \r\n",
       "Q 44.578125 48.1875 50.953125 41.515625 \r\n",
       "Q 57.328125 34.859375 57.328125 23.390625 \r\n",
       "Q 57.328125 12.15625 50.6875 5.359375 \r\n",
       "Q 44.046875 -1.421875 33.015625 -1.421875 \r\n",
       "Q 20.359375 -1.421875 13.671875 8.265625 \r\n",
       "Q 6.984375 17.96875 6.984375 36.375 \r\n",
       "Q 6.984375 53.65625 15.1875 63.9375 \r\n",
       "Q 23.390625 74.21875 37.203125 74.21875 \r\n",
       "Q 40.921875 74.21875 44.703125 73.484375 \r\n",
       "Q 48.484375 72.75 52.59375 71.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-54\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-54\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_5\">\r\n",
       "     <g id=\"line2d_5\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"186.343125\" xlink:href=\"#mad6cee5e8b\" y=\"146.899219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_5\">\r\n",
       "      <!-- 0.8 -->\r\n",
       "      <g transform=\"translate(178.391563 161.497656)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 31.78125 34.625 \r\n",
       "Q 24.75 34.625 20.71875 30.859375 \r\n",
       "Q 16.703125 27.09375 16.703125 20.515625 \r\n",
       "Q 16.703125 13.921875 20.71875 10.15625 \r\n",
       "Q 24.75 6.390625 31.78125 6.390625 \r\n",
       "Q 38.8125 6.390625 42.859375 10.171875 \r\n",
       "Q 46.921875 13.96875 46.921875 20.515625 \r\n",
       "Q 46.921875 27.09375 42.890625 30.859375 \r\n",
       "Q 38.875 34.625 31.78125 34.625 \r\n",
       "z\r\n",
       "M 21.921875 38.8125 \r\n",
       "Q 15.578125 40.375 12.03125 44.71875 \r\n",
       "Q 8.5 49.078125 8.5 55.328125 \r\n",
       "Q 8.5 64.0625 14.71875 69.140625 \r\n",
       "Q 20.953125 74.21875 31.78125 74.21875 \r\n",
       "Q 42.671875 74.21875 48.875 69.140625 \r\n",
       "Q 55.078125 64.0625 55.078125 55.328125 \r\n",
       "Q 55.078125 49.078125 51.53125 44.71875 \r\n",
       "Q 48 40.375 41.703125 38.8125 \r\n",
       "Q 48.828125 37.15625 52.796875 32.3125 \r\n",
       "Q 56.78125 27.484375 56.78125 20.515625 \r\n",
       "Q 56.78125 9.90625 50.3125 4.234375 \r\n",
       "Q 43.84375 -1.421875 31.78125 -1.421875 \r\n",
       "Q 19.734375 -1.421875 13.25 4.234375 \r\n",
       "Q 6.78125 9.90625 6.78125 20.515625 \r\n",
       "Q 6.78125 27.484375 10.78125 32.3125 \r\n",
       "Q 14.796875 37.15625 21.921875 38.8125 \r\n",
       "z\r\n",
       "M 18.3125 54.390625 \r\n",
       "Q 18.3125 48.734375 21.84375 45.5625 \r\n",
       "Q 25.390625 42.390625 31.78125 42.390625 \r\n",
       "Q 38.140625 42.390625 41.71875 45.5625 \r\n",
       "Q 45.3125 48.734375 45.3125 54.390625 \r\n",
       "Q 45.3125 60.0625 41.71875 63.234375 \r\n",
       "Q 38.140625 66.40625 31.78125 66.40625 \r\n",
       "Q 25.390625 66.40625 21.84375 63.234375 \r\n",
       "Q 18.3125 60.0625 18.3125 54.390625 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-56\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_6\">\r\n",
       "     <g id=\"line2d_6\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"225.403125\" xlink:href=\"#mad6cee5e8b\" y=\"146.899219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_6\">\r\n",
       "      <!-- 1.0 -->\r\n",
       "      <g transform=\"translate(217.451563 161.497656)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 12.40625 8.296875 \r\n",
       "L 28.515625 8.296875 \r\n",
       "L 28.515625 63.921875 \r\n",
       "L 10.984375 60.40625 \r\n",
       "L 10.984375 69.390625 \r\n",
       "L 28.421875 72.90625 \r\n",
       "L 38.28125 72.90625 \r\n",
       "L 38.28125 8.296875 \r\n",
       "L 54.390625 8.296875 \r\n",
       "L 54.390625 0 \r\n",
       "L 12.40625 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-49\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"matplotlib.axis_2\">\r\n",
       "    <g id=\"ytick_1\">\r\n",
       "     <g id=\"line2d_7\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L -3.5 0 \r\n",
       "\" id=\"m2a4f1780f0\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"30.103125\" xlink:href=\"#m2a4f1780f0\" y=\"146.899219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_7\">\r\n",
       "      <!-- 0.0 -->\r\n",
       "      <g transform=\"translate(7.2 150.698437)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_2\">\r\n",
       "     <g id=\"line2d_8\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"30.103125\" xlink:href=\"#m2a4f1780f0\" y=\"119.719219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_8\">\r\n",
       "      <!-- 0.2 -->\r\n",
       "      <g transform=\"translate(7.2 123.518437)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_3\">\r\n",
       "     <g id=\"line2d_9\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"30.103125\" xlink:href=\"#m2a4f1780f0\" y=\"92.539219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_9\">\r\n",
       "      <!-- 0.4 -->\r\n",
       "      <g transform=\"translate(7.2 96.338437)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-52\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_4\">\r\n",
       "     <g id=\"line2d_10\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"30.103125\" xlink:href=\"#m2a4f1780f0\" y=\"65.359219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_10\">\r\n",
       "      <!-- 0.6 -->\r\n",
       "      <g transform=\"translate(7.2 69.158437)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-54\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_5\">\r\n",
       "     <g id=\"line2d_11\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"30.103125\" xlink:href=\"#m2a4f1780f0\" y=\"38.179219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_11\">\r\n",
       "      <!-- 0.8 -->\r\n",
       "      <g transform=\"translate(7.2 41.978437)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_6\">\r\n",
       "     <g id=\"line2d_12\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"30.103125\" xlink:href=\"#m2a4f1780f0\" y=\"10.999219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_12\">\r\n",
       "      <!-- 1.0 -->\r\n",
       "      <g transform=\"translate(7.2 14.798437)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_3\">\r\n",
       "    <path d=\"M 30.103125 146.899219 \r\n",
       "L 30.103125 10.999219 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_4\">\r\n",
       "    <path d=\"M 225.403125 146.899219 \r\n",
       "L 225.403125 10.999219 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_5\">\r\n",
       "    <path d=\"M 30.103125 146.899219 \r\n",
       "L 225.403125 146.899219 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_6\">\r\n",
       "    <path d=\"M 30.103125 10.999219 \r\n",
       "L 225.403125 10.999219 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "  </g>\r\n",
       " </g>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 增加为2个GPU\n",
    "train(num_gpus=2, batch_size=256, lr=0.2) # 本电脑只有一个GPU，呜呜呜呜"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. 多GPU的简洁实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from d2l import torch as d2l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet18(num_classes, in_channels=1):\n",
    "    \"\"\"稍加修改的ResNet-18模型\"\"\"\n",
    "    def resnet_block(in_channels, out_channels, num_residuals, first_block=False):   \n",
    "        blk = []\n",
    "        for i in range(num_residuals):\n",
    "            if i == 0 and not first_block:\n",
    "                blk.append(d2l.Residual(in_channels, out_channels, use_1x1conv=True,strides=2))    \n",
    "            else:\n",
    "                blk.append(d2l.Residual(out_channels,out_channels))\n",
    "        return nn.Sequential(*blk)\n",
    "        \n",
    "    net = nn.Sequential(\n",
    "        nn.Conv2d(in_channels,64,kernel_size=3,stride=1,padding=1),\n",
    "        nn.BatchNorm2d(64),\n",
    "        nn.ReLU())  \n",
    "        \n",
    "    net.add_module(\"resnet_block1\", resnet_block(64,64,2,first_block=True))\n",
    "    net.add_module(\"resnet_block2\", resnet_block(64,128,2))\n",
    "    net.add_module(\"resnet_block3\", resnet_block(128,256,2))\n",
    "    net.add_module(\"resnet_block4\", resnet_block(256,512,2))\n",
    "    net.add_module(\"resnet_avg_pool\", nn.AdaptiveAvgPool2d((1,1)))\n",
    "    net.add_module(\"fc\", nn.Sequential(nn.Flatten(), nn.Linear(512, num_classes)))  \n",
    "\n",
    "    return net\n",
    "    \n",
    "net = resnet18(10)\n",
    "devices = d2l.try_all_gpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练\n",
    "def train(net, num_gpus, batch_size, lr):\n",
    "    train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)\n",
    "    devices = [d2l.try_gpu(i) for i in range(num_gpus)]\n",
    "    \n",
    "    def init_weights(m):\n",
    "        if type(m) in [nn.Linear, nn.Conv2d]:\n",
    "            nn.init.normal_(m.weight, std=0.01)\n",
    "            \n",
    "    net.apply(init_weights)\n",
    "    # nn.DataParallel会的是X切开并行到各个GPU上，并行算梯度，然后loss加起来，它重新定义了net的forward函数\n",
    "    net = nn.DataParallel(net, device_ids=devices) # net会复制到每一个GPU上\n",
    "    trainer = torch.optim.SGD(net.parameters(),lr)\n",
    "    loss = nn.CrossEntropyLoss()\n",
    "    timer, num_epochs = d2l.Timer(), 10\n",
    "    animator = d2l.Animator('epoch','test acc', xlim=[1, num_epochs])  \n",
    "    for epoch in range(num_epochs):\n",
    "        net.train()\n",
    "        timer.start()\n",
    "        for X, y in train_iter:\n",
    "            trainer.zero_grad()\n",
    "            X, y = X.to(devices[0]), y.to(devices[0])\n",
    "            l = loss(net(X), y)\n",
    "            l.backward()\n",
    "            trainer.step()\n",
    "        timer.stop()\n",
    "        animator.add(epoch+1, (d2l.evaluate_accuracy_gpu(net, test_iter),))    \n",
    "    print(f'test acc: {animator.Y[0][-1]:.2f}, {timer.avg():.1f} sec/epoch'\n",
    "         f'on {str(devices)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on class DataParallel in module torch.nn.parallel.data_parallel:\n",
      "\n",
      "class DataParallel(torch.nn.modules.module.Module)\n",
      " |  Implements data parallelism at the module level.\n",
      " |  \n",
      " |  This container parallelizes the application of the given :attr:`module` by\n",
      " |  splitting the input across the specified devices by chunking in the batch\n",
      " |  dimension (other objects will be copied once per device). In the forward\n",
      " |  pass, the module is replicated on each device, and each replica handles a\n",
      " |  portion of the input. During the backwards pass, gradients from each replica\n",
      " |  are summed into the original module.\n",
      " |  \n",
      " |  The batch size should be larger than the number of GPUs used.\n",
      " |  \n",
      " |  .. warning::\n",
      " |      It is recommended to use :class:`~torch.nn.parallel.DistributedDataParallel`,\n",
      " |      instead of this class, to do multi-GPU training, even if there is only a single\n",
      " |      node. See: :ref:`cuda-nn-ddp-instead` and :ref:`ddp`.\n",
      " |  \n",
      " |  Arbitrary positional and keyword inputs are allowed to be passed into\n",
      " |  DataParallel but some types are specially handled. tensors will be\n",
      " |  **scattered** on dim specified (default 0). tuple, list and dict types will\n",
      " |  be shallow copied. The other types will be shared among different threads\n",
      " |  and can be corrupted if written to in the model's forward pass.\n",
      " |  \n",
      " |  The parallelized :attr:`module` must have its parameters and buffers on\n",
      " |  ``device_ids[0]`` before running this :class:`~torch.nn.DataParallel`\n",
      " |  module.\n",
      " |  \n",
      " |  .. warning::\n",
      " |      In each forward, :attr:`module` is **replicated** on each device, so any\n",
      " |      updates to the running module in ``forward`` will be lost. For example,\n",
      " |      if :attr:`module` has a counter attribute that is incremented in each\n",
      " |      ``forward``, it will always stay at the initial value because the update\n",
      " |      is done on the replicas which are destroyed after ``forward``. However,\n",
      " |      :class:`~torch.nn.DataParallel` guarantees that the replica on\n",
      " |      ``device[0]`` will have its parameters and buffers sharing storage with\n",
      " |      the base parallelized :attr:`module`. So **in-place** updates to the\n",
      " |      parameters or buffers on ``device[0]`` will be recorded. E.g.,\n",
      " |      :class:`~torch.nn.BatchNorm2d` and :func:`~torch.nn.utils.spectral_norm`\n",
      " |      rely on this behavior to update the buffers.\n",
      " |  \n",
      " |  .. warning::\n",
      " |      Forward and backward hooks defined on :attr:`module` and its submodules\n",
      " |      will be invoked ``len(device_ids)`` times, each with inputs located on\n",
      " |      a particular device. Particularly, the hooks are only guaranteed to be\n",
      " |      executed in correct order with respect to operations on corresponding\n",
      " |      devices. For example, it is not guaranteed that hooks set via\n",
      " |      :meth:`~torch.nn.Module.register_forward_pre_hook` be executed before\n",
      " |      `all` ``len(device_ids)`` :meth:`~torch.nn.Module.forward` calls, but\n",
      " |      that each such hook be executed before the corresponding\n",
      " |      :meth:`~torch.nn.Module.forward` call of that device.\n",
      " |  \n",
      " |  .. warning::\n",
      " |      When :attr:`module` returns a scalar (i.e., 0-dimensional tensor) in\n",
      " |      :func:`forward`, this wrapper will return a vector of length equal to\n",
      " |      number of devices used in data parallelism, containing the result from\n",
      " |      each device.\n",
      " |  \n",
      " |  .. note::\n",
      " |      There is a subtlety in using the\n",
      " |      ``pack sequence -> recurrent network -> unpack sequence`` pattern in a\n",
      " |      :class:`~torch.nn.Module` wrapped in :class:`~torch.nn.DataParallel`.\n",
      " |      See :ref:`pack-rnn-unpack-with-data-parallelism` section in FAQ for\n",
      " |      details.\n",
      " |  \n",
      " |  \n",
      " |  Args:\n",
      " |      module (Module): module to be parallelized\n",
      " |      device_ids (list of int or torch.device): CUDA devices (default: all devices)\n",
      " |      output_device (int or torch.device): device location of output (default: device_ids[0])\n",
      " |  \n",
      " |  Attributes:\n",
      " |      module (Module): the module to be parallelized\n",
      " |  \n",
      " |  Example::\n",
      " |  \n",
      " |      >>> net = torch.nn.DataParallel(model, device_ids=[0, 1, 2])\n",
      " |      >>> output = net(input_var)  # input_var can be on any device, including CPU\n",
      " |  \n",
      " |  Method resolution order:\n",
      " |      DataParallel\n",
      " |      torch.nn.modules.module.Module\n",
      " |      builtins.object\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __init__(self, module, device_ids=None, output_device=None, dim=0)\n",
      " |      Initializes internal Module state, shared by both nn.Module and ScriptModule.\n",
      " |  \n",
      " |  forward(self, *inputs, **kwargs)\n",
      " |      Defines the computation performed at every call.\n",
      " |      \n",
      " |      Should be overridden by all subclasses.\n",
      " |      \n",
      " |      .. note::\n",
      " |          Although the recipe for forward pass needs to be defined within\n",
      " |          this function, one should call the :class:`Module` instance afterwards\n",
      " |          instead of this since the former takes care of running the\n",
      " |          registered hooks while the latter silently ignores them.\n",
      " |  \n",
      " |  gather(self, outputs, output_device)\n",
      " |  \n",
      " |  parallel_apply(self, replicas, inputs, kwargs)\n",
      " |  \n",
      " |  replicate(self, module, device_ids)\n",
      " |  \n",
      " |  scatter(self, inputs, kwargs, device_ids)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from torch.nn.modules.module.Module:\n",
      " |  \n",
      " |  __call__ = _call_impl(self, *input, **kwargs)\n",
      " |  \n",
      " |  __delattr__(self, name)\n",
      " |      Implement delattr(self, name).\n",
      " |  \n",
      " |  __dir__(self)\n",
      " |      __dir__() -> list\n",
      " |      default dir() implementation\n",
      " |  \n",
      " |  __getattr__(self, name:str) -> Union[torch.Tensor, _ForwardRef('Module')]\n",
      " |  \n",
      " |  __repr__(self)\n",
      " |      Return repr(self).\n",
      " |  \n",
      " |  __setattr__(self, name:str, value:Union[torch.Tensor, _ForwardRef('Module')]) -> None\n",
      " |      Implement setattr(self, name, value).\n",
      " |  \n",
      " |  __setstate__(self, state)\n",
      " |  \n",
      " |  add_module(self, name:str, module:Union[_ForwardRef('Module'), NoneType]) -> None\n",
      " |      Adds a child module to the current module.\n",
      " |      \n",
      " |      The module can be accessed as an attribute using the given name.\n",
      " |      \n",
      " |      Args:\n",
      " |          name (string): name of the child module. The child module can be\n",
      " |              accessed from this module using the given name\n",
      " |          module (Module): child module to be added to the module.\n",
      " |  \n",
      " |  apply(self:~T, fn:Callable[[_ForwardRef('Module')], NoneType]) -> ~T\n",
      " |      Applies ``fn`` recursively to every submodule (as returned by ``.children()``)\n",
      " |      as well as self. Typical use includes initializing the parameters of a model\n",
      " |      (see also :ref:`nn-init-doc`).\n",
      " |      \n",
      " |      Args:\n",
      " |          fn (:class:`Module` -> None): function to be applied to each submodule\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> @torch.no_grad()\n",
      " |          >>> def init_weights(m):\n",
      " |          >>>     print(m)\n",
      " |          >>>     if type(m) == nn.Linear:\n",
      " |          >>>         m.weight.fill_(1.0)\n",
      " |          >>>         print(m.weight)\n",
      " |          >>> net = nn.Sequential(nn.Linear(2, 2), nn.Linear(2, 2))\n",
      " |          >>> net.apply(init_weights)\n",
      " |          Linear(in_features=2, out_features=2, bias=True)\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 1.,  1.],\n",
      " |                  [ 1.,  1.]])\n",
      " |          Linear(in_features=2, out_features=2, bias=True)\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 1.,  1.],\n",
      " |                  [ 1.,  1.]])\n",
      " |          Sequential(\n",
      " |            (0): Linear(in_features=2, out_features=2, bias=True)\n",
      " |            (1): Linear(in_features=2, out_features=2, bias=True)\n",
      " |          )\n",
      " |          Sequential(\n",
      " |            (0): Linear(in_features=2, out_features=2, bias=True)\n",
      " |            (1): Linear(in_features=2, out_features=2, bias=True)\n",
      " |          )\n",
      " |  \n",
      " |  bfloat16(self:~T) -> ~T\n",
      " |      Casts all floating point parameters and buffers to ``bfloat16`` datatype.\n",
      " |      \n",
      " |      .. note::\n",
      " |          This method modifies the module in-place.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  buffers(self, recurse:bool=True) -> Iterator[torch.Tensor]\n",
      " |      Returns an iterator over module buffers.\n",
      " |      \n",
      " |      Args:\n",
      " |          recurse (bool): if True, then yields buffers of this module\n",
      " |              and all submodules. Otherwise, yields only buffers that\n",
      " |              are direct members of this module.\n",
      " |      \n",
      " |      Yields:\n",
      " |          torch.Tensor: module buffer\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> for buf in model.buffers():\n",
      " |          >>>     print(type(buf), buf.size())\n",
      " |          <class 'torch.Tensor'> (20L,)\n",
      " |          <class 'torch.Tensor'> (20L, 1L, 5L, 5L)\n",
      " |  \n",
      " |  children(self) -> Iterator[_ForwardRef('Module')]\n",
      " |      Returns an iterator over immediate children modules.\n",
      " |      \n",
      " |      Yields:\n",
      " |          Module: a child module\n",
      " |  \n",
      " |  cpu(self:~T) -> ~T\n",
      " |      Moves all model parameters and buffers to the CPU.\n",
      " |      \n",
      " |      .. note::\n",
      " |          This method modifies the module in-place.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  cuda(self:~T, device:Union[int, torch.device, NoneType]=None) -> ~T\n",
      " |      Moves all model parameters and buffers to the GPU.\n",
      " |      \n",
      " |      This also makes associated parameters and buffers different objects. So\n",
      " |      it should be called before constructing optimizer if the module will\n",
      " |      live on GPU while being optimized.\n",
      " |      \n",
      " |      .. note::\n",
      " |          This method modifies the module in-place.\n",
      " |      \n",
      " |      Args:\n",
      " |          device (int, optional): if specified, all parameters will be\n",
      " |              copied to that device\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  double(self:~T) -> ~T\n",
      " |      Casts all floating point parameters and buffers to ``double`` datatype.\n",
      " |      \n",
      " |      .. note::\n",
      " |          This method modifies the module in-place.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  eval(self:~T) -> ~T\n",
      " |      Sets the module in evaluation mode.\n",
      " |      \n",
      " |      This has any effect only on certain modules. See documentations of\n",
      " |      particular modules for details of their behaviors in training/evaluation\n",
      " |      mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`,\n",
      " |      etc.\n",
      " |      \n",
      " |      This is equivalent with :meth:`self.train(False) <torch.nn.Module.train>`.\n",
      " |      \n",
      " |      See :ref:`locally-disable-grad-doc` for a comparison between\n",
      " |      `.eval()` and several similar mechanisms that may be confused with it.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  extra_repr(self) -> str\n",
      " |      Set the extra representation of the module\n",
      " |      \n",
      " |      To print customized extra information, you should re-implement\n",
      " |      this method in your own modules. Both single-line and multi-line\n",
      " |      strings are acceptable.\n",
      " |  \n",
      " |  float(self:~T) -> ~T\n",
      " |      Casts all floating point parameters and buffers to ``float`` datatype.\n",
      " |      \n",
      " |      .. note::\n",
      " |          This method modifies the module in-place.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  get_buffer(self, target:str) -> 'Tensor'\n",
      " |      Returns the buffer given by ``target`` if it exists,\n",
      " |      otherwise throws an error.\n",
      " |      \n",
      " |      See the docstring for ``get_submodule`` for a more detailed\n",
      " |      explanation of this method's functionality as well as how to\n",
      " |      correctly specify ``target``.\n",
      " |      \n",
      " |      Args:\n",
      " |          target: The fully-qualified string name of the buffer\n",
      " |              to look for. (See ``get_submodule`` for how to specify a\n",
      " |              fully-qualified string.)\n",
      " |      \n",
      " |      Returns:\n",
      " |          torch.Tensor: The buffer referenced by ``target``\n",
      " |      \n",
      " |      Raises:\n",
      " |          AttributeError: If the target string references an invalid\n",
      " |              path or resolves to something that is not a\n",
      " |              buffer\n",
      " |  \n",
      " |  get_extra_state(self) -> Any\n",
      " |      Returns any extra state to include in the module's state_dict.\n",
      " |      Implement this and a corresponding :func:`set_extra_state` for your module\n",
      " |      if you need to store extra state. This function is called when building the\n",
      " |      module's `state_dict()`.\n",
      " |      \n",
      " |      Note that extra state should be pickleable to ensure working serialization\n",
      " |      of the state_dict. We only provide provide backwards compatibility guarantees\n",
      " |      for serializing Tensors; other objects may break backwards compatibility if\n",
      " |      their serialized pickled form changes.\n",
      " |      \n",
      " |      Returns:\n",
      " |          object: Any extra state to store in the module's state_dict\n",
      " |  \n",
      " |  get_parameter(self, target:str) -> 'Parameter'\n",
      " |      Returns the parameter given by ``target`` if it exists,\n",
      " |      otherwise throws an error.\n",
      " |      \n",
      " |      See the docstring for ``get_submodule`` for a more detailed\n",
      " |      explanation of this method's functionality as well as how to\n",
      " |      correctly specify ``target``.\n",
      " |      \n",
      " |      Args:\n",
      " |          target: The fully-qualified string name of the Parameter\n",
      " |              to look for. (See ``get_submodule`` for how to specify a\n",
      " |              fully-qualified string.)\n",
      " |      \n",
      " |      Returns:\n",
      " |          torch.nn.Parameter: The Parameter referenced by ``target``\n",
      " |      \n",
      " |      Raises:\n",
      " |          AttributeError: If the target string references an invalid\n",
      " |              path or resolves to something that is not an\n",
      " |              ``nn.Parameter``\n",
      " |  \n",
      " |  get_submodule(self, target:str) -> 'Module'\n",
      " |      Returns the submodule given by ``target`` if it exists,\n",
      " |      otherwise throws an error.\n",
      " |      \n",
      " |      For example, let's say you have an ``nn.Module`` ``A`` that\n",
      " |      looks like this:\n",
      " |      \n",
      " |      .. code-block::text\n",
      " |      \n",
      " |          A(\n",
      " |              (net_b): Module(\n",
      " |                  (net_c): Module(\n",
      " |                      (conv): Conv2d(16, 33, kernel_size=(3, 3), stride=(2, 2))\n",
      " |                  )\n",
      " |                  (linear): Linear(in_features=100, out_features=200, bias=True)\n",
      " |              )\n",
      " |          )\n",
      " |      \n",
      " |      (The diagram shows an ``nn.Module`` ``A``. ``A`` has a nested\n",
      " |      submodule ``net_b``, which itself has two submodules ``net_c``\n",
      " |      and ``linear``. ``net_c`` then has a submodule ``conv``.)\n",
      " |      \n",
      " |      To check whether or not we have the ``linear`` submodule, we\n",
      " |      would call ``get_submodule(\"net_b.linear\")``. To check whether\n",
      " |      we have the ``conv`` submodule, we would call\n",
      " |      ``get_submodule(\"net_b.net_c.conv\")``.\n",
      " |      \n",
      " |      The runtime of ``get_submodule`` is bounded by the degree\n",
      " |      of module nesting in ``target``. A query against\n",
      " |      ``named_modules`` achieves the same result, but it is O(N) in\n",
      " |      the number of transitive modules. So, for a simple check to see\n",
      " |      if some submodule exists, ``get_submodule`` should always be\n",
      " |      used.\n",
      " |      \n",
      " |      Args:\n",
      " |          target: The fully-qualified string name of the submodule\n",
      " |              to look for. (See above example for how to specify a\n",
      " |              fully-qualified string.)\n",
      " |      \n",
      " |      Returns:\n",
      " |          torch.nn.Module: The submodule referenced by ``target``\n",
      " |      \n",
      " |      Raises:\n",
      " |          AttributeError: If the target string references an invalid\n",
      " |              path or resolves to something that is not an\n",
      " |              ``nn.Module``\n",
      " |  \n",
      " |  half(self:~T) -> ~T\n",
      " |      Casts all floating point parameters and buffers to ``half`` datatype.\n",
      " |      \n",
      " |      .. note::\n",
      " |          This method modifies the module in-place.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  load_state_dict(self, state_dict:'OrderedDict[str, Tensor]', strict:bool=True)\n",
      " |      Copies parameters and buffers from :attr:`state_dict` into\n",
      " |      this module and its descendants. If :attr:`strict` is ``True``, then\n",
      " |      the keys of :attr:`state_dict` must exactly match the keys returned\n",
      " |      by this module's :meth:`~torch.nn.Module.state_dict` function.\n",
      " |      \n",
      " |      Args:\n",
      " |          state_dict (dict): a dict containing parameters and\n",
      " |              persistent buffers.\n",
      " |          strict (bool, optional): whether to strictly enforce that the keys\n",
      " |              in :attr:`state_dict` match the keys returned by this module's\n",
      " |              :meth:`~torch.nn.Module.state_dict` function. Default: ``True``\n",
      " |      \n",
      " |      Returns:\n",
      " |          ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:\n",
      " |              * **missing_keys** is a list of str containing the missing keys\n",
      " |              * **unexpected_keys** is a list of str containing the unexpected keys\n",
      " |      \n",
      " |      Note:\n",
      " |          If a parameter or buffer is registered as ``None`` and its corresponding key\n",
      " |          exists in :attr:`state_dict`, :meth:`load_state_dict` will raise a\n",
      " |          ``RuntimeError``.\n",
      " |  \n",
      " |  modules(self) -> Iterator[_ForwardRef('Module')]\n",
      " |      Returns an iterator over all modules in the network.\n",
      " |      \n",
      " |      Yields:\n",
      " |          Module: a module in the network\n",
      " |      \n",
      " |      Note:\n",
      " |          Duplicate modules are returned only once. In the following\n",
      " |          example, ``l`` will be returned only once.\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> l = nn.Linear(2, 2)\n",
      " |          >>> net = nn.Sequential(l, l)\n",
      " |          >>> for idx, m in enumerate(net.modules()):\n",
      " |                  print(idx, '->', m)\n",
      " |      \n",
      " |          0 -> Sequential(\n",
      " |            (0): Linear(in_features=2, out_features=2, bias=True)\n",
      " |            (1): Linear(in_features=2, out_features=2, bias=True)\n",
      " |          )\n",
      " |          1 -> Linear(in_features=2, out_features=2, bias=True)\n",
      " |  \n",
      " |  named_buffers(self, prefix:str='', recurse:bool=True) -> Iterator[Tuple[str, torch.Tensor]]\n",
      " |      Returns an iterator over module buffers, yielding both the\n",
      " |      name of the buffer as well as the buffer itself.\n",
      " |      \n",
      " |      Args:\n",
      " |          prefix (str): prefix to prepend to all buffer names.\n",
      " |          recurse (bool): if True, then yields buffers of this module\n",
      " |              and all submodules. Otherwise, yields only buffers that\n",
      " |              are direct members of this module.\n",
      " |      \n",
      " |      Yields:\n",
      " |          (string, torch.Tensor): Tuple containing the name and buffer\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> for name, buf in self.named_buffers():\n",
      " |          >>>    if name in ['running_var']:\n",
      " |          >>>        print(buf.size())\n",
      " |  \n",
      " |  named_children(self) -> Iterator[Tuple[str, _ForwardRef('Module')]]\n",
      " |      Returns an iterator over immediate children modules, yielding both\n",
      " |      the name of the module as well as the module itself.\n",
      " |      \n",
      " |      Yields:\n",
      " |          (string, Module): Tuple containing a name and child module\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> for name, module in model.named_children():\n",
      " |          >>>     if name in ['conv4', 'conv5']:\n",
      " |          >>>         print(module)\n",
      " |  \n",
      " |  named_modules(self, memo:Union[Set[_ForwardRef('Module')], NoneType]=None, prefix:str='', remove_duplicate:bool=True)\n",
      " |      Returns an iterator over all modules in the network, yielding\n",
      " |      both the name of the module as well as the module itself.\n",
      " |      \n",
      " |      Args:\n",
      " |          memo: a memo to store the set of modules already added to the result\n",
      " |          prefix: a prefix that will be added to the name of the module\n",
      " |          remove_duplicate: whether to remove the duplicated module instances in the result\n",
      " |          or not\n",
      " |      \n",
      " |      Yields:\n",
      " |          (string, Module): Tuple of name and module\n",
      " |      \n",
      " |      Note:\n",
      " |          Duplicate modules are returned only once. In the following\n",
      " |          example, ``l`` will be returned only once.\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> l = nn.Linear(2, 2)\n",
      " |          >>> net = nn.Sequential(l, l)\n",
      " |          >>> for idx, m in enumerate(net.named_modules()):\n",
      " |                  print(idx, '->', m)\n",
      " |      \n",
      " |          0 -> ('', Sequential(\n",
      " |            (0): Linear(in_features=2, out_features=2, bias=True)\n",
      " |            (1): Linear(in_features=2, out_features=2, bias=True)\n",
      " |          ))\n",
      " |          1 -> ('0', Linear(in_features=2, out_features=2, bias=True))\n",
      " |  \n",
      " |  named_parameters(self, prefix:str='', recurse:bool=True) -> Iterator[Tuple[str, torch.nn.parameter.Parameter]]\n",
      " |      Returns an iterator over module parameters, yielding both the\n",
      " |      name of the parameter as well as the parameter itself.\n",
      " |      \n",
      " |      Args:\n",
      " |          prefix (str): prefix to prepend to all parameter names.\n",
      " |          recurse (bool): if True, then yields parameters of this module\n",
      " |              and all submodules. Otherwise, yields only parameters that\n",
      " |              are direct members of this module.\n",
      " |      \n",
      " |      Yields:\n",
      " |          (string, Parameter): Tuple containing the name and parameter\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> for name, param in self.named_parameters():\n",
      " |          >>>    if name in ['bias']:\n",
      " |          >>>        print(param.size())\n",
      " |  \n",
      " |  parameters(self, recurse:bool=True) -> Iterator[torch.nn.parameter.Parameter]\n",
      " |      Returns an iterator over module parameters.\n",
      " |      \n",
      " |      This is typically passed to an optimizer.\n",
      " |      \n",
      " |      Args:\n",
      " |          recurse (bool): if True, then yields parameters of this module\n",
      " |              and all submodules. Otherwise, yields only parameters that\n",
      " |              are direct members of this module.\n",
      " |      \n",
      " |      Yields:\n",
      " |          Parameter: module parameter\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> for param in model.parameters():\n",
      " |          >>>     print(type(param), param.size())\n",
      " |          <class 'torch.Tensor'> (20L,)\n",
      " |          <class 'torch.Tensor'> (20L, 1L, 5L, 5L)\n",
      " |  \n",
      " |  register_backward_hook(self, hook:Callable[[_ForwardRef('Module'), Union[Tuple[torch.Tensor, ...], torch.Tensor], Union[Tuple[torch.Tensor, ...], torch.Tensor]], Union[NoneType, torch.Tensor]]) -> torch.utils.hooks.RemovableHandle\n",
      " |      Registers a backward hook on the module.\n",
      " |      \n",
      " |      This function is deprecated in favor of :meth:`~torch.nn.Module.register_full_backward_hook` and\n",
      " |      the behavior of this function will change in future versions.\n",
      " |      \n",
      " |      Returns:\n",
      " |          :class:`torch.utils.hooks.RemovableHandle`:\n",
      " |              a handle that can be used to remove the added hook by calling\n",
      " |              ``handle.remove()``\n",
      " |  \n",
      " |  register_buffer(self, name:str, tensor:Union[torch.Tensor, NoneType], persistent:bool=True) -> None\n",
      " |      Adds a buffer to the module.\n",
      " |      \n",
      " |      This is typically used to register a buffer that should not to be\n",
      " |      considered a model parameter. For example, BatchNorm's ``running_mean``\n",
      " |      is not a parameter, but is part of the module's state. Buffers, by\n",
      " |      default, are persistent and will be saved alongside parameters. This\n",
      " |      behavior can be changed by setting :attr:`persistent` to ``False``. The\n",
      " |      only difference between a persistent buffer and a non-persistent buffer\n",
      " |      is that the latter will not be a part of this module's\n",
      " |      :attr:`state_dict`.\n",
      " |      \n",
      " |      Buffers can be accessed as attributes using given names.\n",
      " |      \n",
      " |      Args:\n",
      " |          name (string): name of the buffer. The buffer can be accessed\n",
      " |              from this module using the given name\n",
      " |          tensor (Tensor or None): buffer to be registered. If ``None``, then operations\n",
      " |              that run on buffers, such as :attr:`cuda`, are ignored. If ``None``,\n",
      " |              the buffer is **not** included in the module's :attr:`state_dict`.\n",
      " |          persistent (bool): whether the buffer is part of this module's\n",
      " |              :attr:`state_dict`.\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> self.register_buffer('running_mean', torch.zeros(num_features))\n",
      " |  \n",
      " |  register_forward_hook(self, hook:Callable[..., NoneType]) -> torch.utils.hooks.RemovableHandle\n",
      " |      Registers a forward hook on the module.\n",
      " |      \n",
      " |      The hook will be called every time after :func:`forward` has computed an output.\n",
      " |      It should have the following signature::\n",
      " |      \n",
      " |          hook(module, input, output) -> None or modified output\n",
      " |      \n",
      " |      The input contains only the positional arguments given to the module.\n",
      " |      Keyword arguments won't be passed to the hooks and only to the ``forward``.\n",
      " |      The hook can modify the output. It can modify the input inplace but\n",
      " |      it will not have effect on forward since this is called after\n",
      " |      :func:`forward` is called.\n",
      " |      \n",
      " |      Returns:\n",
      " |          :class:`torch.utils.hooks.RemovableHandle`:\n",
      " |              a handle that can be used to remove the added hook by calling\n",
      " |              ``handle.remove()``\n",
      " |  \n",
      " |  register_forward_pre_hook(self, hook:Callable[..., NoneType]) -> torch.utils.hooks.RemovableHandle\n",
      " |      Registers a forward pre-hook on the module.\n",
      " |      \n",
      " |      The hook will be called every time before :func:`forward` is invoked.\n",
      " |      It should have the following signature::\n",
      " |      \n",
      " |          hook(module, input) -> None or modified input\n",
      " |      \n",
      " |      The input contains only the positional arguments given to the module.\n",
      " |      Keyword arguments won't be passed to the hooks and only to the ``forward``.\n",
      " |      The hook can modify the input. User can either return a tuple or a\n",
      " |      single modified value in the hook. We will wrap the value into a tuple\n",
      " |      if a single value is returned(unless that value is already a tuple).\n",
      " |      \n",
      " |      Returns:\n",
      " |          :class:`torch.utils.hooks.RemovableHandle`:\n",
      " |              a handle that can be used to remove the added hook by calling\n",
      " |              ``handle.remove()``\n",
      " |  \n",
      " |  register_full_backward_hook(self, hook:Callable[[_ForwardRef('Module'), Union[Tuple[torch.Tensor, ...], torch.Tensor], Union[Tuple[torch.Tensor, ...], torch.Tensor]], Union[NoneType, torch.Tensor]]) -> torch.utils.hooks.RemovableHandle\n",
      " |      Registers a backward hook on the module.\n",
      " |      \n",
      " |      The hook will be called every time the gradients with respect to module\n",
      " |      inputs are computed. The hook should have the following signature::\n",
      " |      \n",
      " |          hook(module, grad_input, grad_output) -> tuple(Tensor) or None\n",
      " |      \n",
      " |      The :attr:`grad_input` and :attr:`grad_output` are tuples that contain the gradients\n",
      " |      with respect to the inputs and outputs respectively. The hook should\n",
      " |      not modify its arguments, but it can optionally return a new gradient with\n",
      " |      respect to the input that will be used in place of :attr:`grad_input` in\n",
      " |      subsequent computations. :attr:`grad_input` will only correspond to the inputs given\n",
      " |      as positional arguments and all kwarg arguments are ignored. Entries\n",
      " |      in :attr:`grad_input` and :attr:`grad_output` will be ``None`` for all non-Tensor\n",
      " |      arguments.\n",
      " |      \n",
      " |      For technical reasons, when this hook is applied to a Module, its forward function will\n",
      " |      receive a view of each Tensor passed to the Module. Similarly the caller will receive a view\n",
      " |      of each Tensor returned by the Module's forward function.\n",
      " |      \n",
      " |      .. warning ::\n",
      " |          Modifying inputs or outputs inplace is not allowed when using backward hooks and\n",
      " |          will raise an error.\n",
      " |      \n",
      " |      Returns:\n",
      " |          :class:`torch.utils.hooks.RemovableHandle`:\n",
      " |              a handle that can be used to remove the added hook by calling\n",
      " |              ``handle.remove()``\n",
      " |  \n",
      " |  register_parameter(self, name:str, param:Union[torch.nn.parameter.Parameter, NoneType]) -> None\n",
      " |      Adds a parameter to the module.\n",
      " |      \n",
      " |      The parameter can be accessed as an attribute using given name.\n",
      " |      \n",
      " |      Args:\n",
      " |          name (string): name of the parameter. The parameter can be accessed\n",
      " |              from this module using the given name\n",
      " |          param (Parameter or None): parameter to be added to the module. If\n",
      " |              ``None``, then operations that run on parameters, such as :attr:`cuda`,\n",
      " |              are ignored. If ``None``, the parameter is **not** included in the\n",
      " |              module's :attr:`state_dict`.\n",
      " |  \n",
      " |  requires_grad_(self:~T, requires_grad:bool=True) -> ~T\n",
      " |      Change if autograd should record operations on parameters in this\n",
      " |      module.\n",
      " |      \n",
      " |      This method sets the parameters' :attr:`requires_grad` attributes\n",
      " |      in-place.\n",
      " |      \n",
      " |      This method is helpful for freezing part of the module for finetuning\n",
      " |      or training parts of a model individually (e.g., GAN training).\n",
      " |      \n",
      " |      See :ref:`locally-disable-grad-doc` for a comparison between\n",
      " |      `.requires_grad_()` and several similar mechanisms that may be confused with it.\n",
      " |      \n",
      " |      Args:\n",
      " |          requires_grad (bool): whether autograd should record operations on\n",
      " |                                parameters in this module. Default: ``True``.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  set_extra_state(self, state:Any)\n",
      " |      This function is called from :func:`load_state_dict` to handle any extra state\n",
      " |      found within the `state_dict`. Implement this function and a corresponding\n",
      " |      :func:`get_extra_state` for your module if you need to store extra state within its\n",
      " |      `state_dict`.\n",
      " |      \n",
      " |      Args:\n",
      " |          state (dict): Extra state from the `state_dict`\n",
      " |  \n",
      " |  share_memory(self:~T) -> ~T\n",
      " |      See :meth:`torch.Tensor.share_memory_`\n",
      " |  \n",
      " |  state_dict(self, destination=None, prefix='', keep_vars=False)\n",
      " |      Returns a dictionary containing a whole state of the module.\n",
      " |      \n",
      " |      Both parameters and persistent buffers (e.g. running averages) are\n",
      " |      included. Keys are corresponding parameter and buffer names.\n",
      " |      Parameters and buffers set to ``None`` are not included.\n",
      " |      \n",
      " |      Returns:\n",
      " |          dict:\n",
      " |              a dictionary containing a whole state of the module\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> module.state_dict().keys()\n",
      " |          ['bias', 'weight']\n",
      " |  \n",
      " |  to(self, *args, **kwargs)\n",
      " |      Moves and/or casts the parameters and buffers.\n",
      " |      \n",
      " |      This can be called as\n",
      " |      \n",
      " |      .. function:: to(device=None, dtype=None, non_blocking=False)\n",
      " |         :noindex:\n",
      " |      \n",
      " |      .. function:: to(dtype, non_blocking=False)\n",
      " |         :noindex:\n",
      " |      \n",
      " |      .. function:: to(tensor, non_blocking=False)\n",
      " |         :noindex:\n",
      " |      \n",
      " |      .. function:: to(memory_format=torch.channels_last)\n",
      " |         :noindex:\n",
      " |      \n",
      " |      Its signature is similar to :meth:`torch.Tensor.to`, but only accepts\n",
      " |      floating point or complex :attr:`dtype`\\ s. In addition, this method will\n",
      " |      only cast the floating point or complex parameters and buffers to :attr:`dtype`\n",
      " |      (if given). The integral parameters and buffers will be moved\n",
      " |      :attr:`device`, if that is given, but with dtypes unchanged. When\n",
      " |      :attr:`non_blocking` is set, it tries to convert/move asynchronously\n",
      " |      with respect to the host if possible, e.g., moving CPU Tensors with\n",
      " |      pinned memory to CUDA devices.\n",
      " |      \n",
      " |      See below for examples.\n",
      " |      \n",
      " |      .. note::\n",
      " |          This method modifies the module in-place.\n",
      " |      \n",
      " |      Args:\n",
      " |          device (:class:`torch.device`): the desired device of the parameters\n",
      " |              and buffers in this module\n",
      " |          dtype (:class:`torch.dtype`): the desired floating point or complex dtype of\n",
      " |              the parameters and buffers in this module\n",
      " |          tensor (torch.Tensor): Tensor whose dtype and device are the desired\n",
      " |              dtype and device for all parameters and buffers in this module\n",
      " |          memory_format (:class:`torch.memory_format`): the desired memory\n",
      " |              format for 4D parameters and buffers in this module (keyword\n",
      " |              only argument)\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |      \n",
      " |      Examples::\n",
      " |      \n",
      " |          >>> linear = nn.Linear(2, 2)\n",
      " |          >>> linear.weight\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 0.1913, -0.3420],\n",
      " |                  [-0.5113, -0.2325]])\n",
      " |          >>> linear.to(torch.double)\n",
      " |          Linear(in_features=2, out_features=2, bias=True)\n",
      " |          >>> linear.weight\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 0.1913, -0.3420],\n",
      " |                  [-0.5113, -0.2325]], dtype=torch.float64)\n",
      " |          >>> gpu1 = torch.device(\"cuda:1\")\n",
      " |          >>> linear.to(gpu1, dtype=torch.half, non_blocking=True)\n",
      " |          Linear(in_features=2, out_features=2, bias=True)\n",
      " |          >>> linear.weight\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 0.1914, -0.3420],\n",
      " |                  [-0.5112, -0.2324]], dtype=torch.float16, device='cuda:1')\n",
      " |          >>> cpu = torch.device(\"cpu\")\n",
      " |          >>> linear.to(cpu)\n",
      " |          Linear(in_features=2, out_features=2, bias=True)\n",
      " |          >>> linear.weight\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 0.1914, -0.3420],\n",
      " |                  [-0.5112, -0.2324]], dtype=torch.float16)\n",
      " |      \n",
      " |          >>> linear = nn.Linear(2, 2, bias=None).to(torch.cdouble)\n",
      " |          >>> linear.weight\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 0.3741+0.j,  0.2382+0.j],\n",
      " |                  [ 0.5593+0.j, -0.4443+0.j]], dtype=torch.complex128)\n",
      " |          >>> linear(torch.ones(3, 2, dtype=torch.cdouble))\n",
      " |          tensor([[0.6122+0.j, 0.1150+0.j],\n",
      " |                  [0.6122+0.j, 0.1150+0.j],\n",
      " |                  [0.6122+0.j, 0.1150+0.j]], dtype=torch.complex128)\n",
      " |  \n",
      " |  to_empty(self:~T, *, device:Union[str, torch.device]) -> ~T\n",
      " |      Moves the parameters and buffers to the specified device without copying storage.\n",
      " |      \n",
      " |      Args:\n",
      " |          device (:class:`torch.device`): The desired device of the parameters\n",
      " |              and buffers in this module.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  train(self:~T, mode:bool=True) -> ~T\n",
      " |      Sets the module in training mode.\n",
      " |      \n",
      " |      This has any effect only on certain modules. See documentations of\n",
      " |      particular modules for details of their behaviors in training/evaluation\n",
      " |      mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`,\n",
      " |      etc.\n",
      " |      \n",
      " |      Args:\n",
      " |          mode (bool): whether to set training mode (``True``) or evaluation\n",
      " |                       mode (``False``). Default: ``True``.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  type(self:~T, dst_type:Union[torch.dtype, str]) -> ~T\n",
      " |      Casts all parameters and buffers to :attr:`dst_type`.\n",
      " |      \n",
      " |      .. note::\n",
      " |          This method modifies the module in-place.\n",
      " |      \n",
      " |      Args:\n",
      " |          dst_type (type or string): the desired type\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  xpu(self:~T, device:Union[int, torch.device, NoneType]=None) -> ~T\n",
      " |      Moves all model parameters and buffers to the XPU.\n",
      " |      \n",
      " |      This also makes associated parameters and buffers different objects. So\n",
      " |      it should be called before constructing optimizer if the module will\n",
      " |      live on XPU while being optimized.\n",
      " |      \n",
      " |      .. note::\n",
      " |          This method modifies the module in-place.\n",
      " |      \n",
      " |      Arguments:\n",
      " |          device (int, optional): if specified, all parameters will be\n",
      " |              copied to that device\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  zero_grad(self, set_to_none:bool=False) -> None\n",
      " |      Sets gradients of all model parameters to zero. See similar function\n",
      " |      under :class:`torch.optim.Optimizer` for more context.\n",
      " |      \n",
      " |      Args:\n",
      " |          set_to_none (bool): instead of setting to zero, set the grads to None.\n",
      " |              See :meth:`torch.optim.Optimizer.zero_grad` for details.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors inherited from torch.nn.modules.module.Module:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data and other attributes inherited from torch.nn.modules.module.Module:\n",
      " |  \n",
      " |  T_destination = ~T_destination\n",
      " |      Type variable.\n",
      " |      \n",
      " |      Usage::\n",
      " |      \n",
      " |        T = TypeVar('T')  # Can be anything\n",
      " |        A = TypeVar('A', str, bytes)  # Must be str or bytes\n",
      " |      \n",
      " |      Type variables exist primarily for the benefit of static type\n",
      " |      checkers.  They serve as the parameters for generic types as well\n",
      " |      as for generic function definitions.  See class Generic for more\n",
      " |      information on generic types.  Generic functions work as follows:\n",
      " |      \n",
      " |        def repeat(x: T, n: int) -> List[T]:\n",
      " |            '''Return a list containing n references to x.'''\n",
      " |            return [x]*n\n",
      " |      \n",
      " |        def longest(x: A, y: A) -> A:\n",
      " |            '''Return the longest of two strings.'''\n",
      " |            return x if len(x) >= len(y) else y\n",
      " |      \n",
      " |      The latter example's signature is essentially the overloading\n",
      " |      of (str, str) -> str and (bytes, bytes) -> bytes.  Also note\n",
      " |      that if the arguments are instances of some subclass of str,\n",
      " |      the return type is still plain str.\n",
      " |      \n",
      " |      At runtime, isinstance(x, T) and issubclass(C, T) will raise TypeError.\n",
      " |      \n",
      " |      Type variables defined with covariant=True or contravariant=True\n",
      " |      can be used do declare covariant or contravariant generic types.\n",
      " |      See PEP 484 for more details. By default generic types are invariant\n",
      " |      in all type variables.\n",
      " |      \n",
      " |      Type variables can be introspected. e.g.:\n",
      " |      \n",
      " |        T.__name__ == 'T'\n",
      " |        T.__constraints__ == ()\n",
      " |        T.__covariant__ == False\n",
      " |        T.__contravariant__ = False\n",
      " |        A.__constraints__ == (str, bytes)\n",
      " |  \n",
      " |  __annotations__ = {'__call__': typing.Callable[..., typing.Any], '_is_...\n",
      " |  \n",
      " |  dump_patches = False\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(nn.DataParallel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test acc: 0.92, 27.5 sec/epochon [device(type='cuda', index=0)]\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Created with matplotlib (https://matplotlib.org/) -->\r\n",
       "<svg height=\"181.284833pt\" version=\"1.1\" viewBox=\"0 0 259.00625 181.284833\" width=\"259.00625pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       " <metadata>\r\n",
       "  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\r\n",
       "   <cc:Work>\r\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\r\n",
       "    <dc:date>2022-06-11T12:02:06.012855</dc:date>\r\n",
       "    <dc:format>image/svg+xml</dc:format>\r\n",
       "    <dc:creator>\r\n",
       "     <cc:Agent>\r\n",
       "      <dc:title>Matplotlib v3.3.4, https://matplotlib.org/</dc:title>\r\n",
       "     </cc:Agent>\r\n",
       "    </dc:creator>\r\n",
       "   </cc:Work>\r\n",
       "  </rdf:RDF>\r\n",
       " </metadata>\r\n",
       " <defs>\r\n",
       "  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\r\n",
       " </defs>\r\n",
       " <g id=\"figure_1\">\r\n",
       "  <g id=\"patch_1\">\r\n",
       "   <path d=\"M 0 181.284833 \r\n",
       "L 259.00625 181.284833 \r\n",
       "L 259.00625 -0 \r\n",
       "L 0 -0 \r\n",
       "z\r\n",
       "\" style=\"fill:none;\"/>\r\n",
       "  </g>\r\n",
       "  <g id=\"axes_1\">\r\n",
       "   <g id=\"patch_2\">\r\n",
       "    <path d=\"M 50.14375 143.728583 \r\n",
       "L 245.44375 143.728583 \r\n",
       "L 245.44375 7.828583 \r\n",
       "L 50.14375 7.828583 \r\n",
       "z\r\n",
       "\" style=\"fill:#ffffff;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"matplotlib.axis_1\">\r\n",
       "    <g id=\"xtick_1\">\r\n",
       "     <g id=\"line2d_1\">\r\n",
       "      <path clip-path=\"url(#pf831abab33)\" d=\"M 71.84375 143.728583 \r\n",
       "L 71.84375 7.828583 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_2\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L 0 3.5 \r\n",
       "\" id=\"mea90d692ed\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"71.84375\" xlink:href=\"#mea90d692ed\" y=\"143.728583\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_1\">\r\n",
       "      <!-- 2 -->\r\n",
       "      <g transform=\"translate(68.6625 158.327021)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 19.1875 8.296875 \r\n",
       "L 53.609375 8.296875 \r\n",
       "L 53.609375 0 \r\n",
       "L 7.328125 0 \r\n",
       "L 7.328125 8.296875 \r\n",
       "Q 12.9375 14.109375 22.625 23.890625 \r\n",
       "Q 32.328125 33.6875 34.8125 36.53125 \r\n",
       "Q 39.546875 41.84375 41.421875 45.53125 \r\n",
       "Q 43.3125 49.21875 43.3125 52.78125 \r\n",
       "Q 43.3125 58.59375 39.234375 62.25 \r\n",
       "Q 35.15625 65.921875 28.609375 65.921875 \r\n",
       "Q 23.96875 65.921875 18.8125 64.3125 \r\n",
       "Q 13.671875 62.703125 7.8125 59.421875 \r\n",
       "L 7.8125 69.390625 \r\n",
       "Q 13.765625 71.78125 18.9375 73 \r\n",
       "Q 24.125 74.21875 28.421875 74.21875 \r\n",
       "Q 39.75 74.21875 46.484375 68.546875 \r\n",
       "Q 53.21875 62.890625 53.21875 53.421875 \r\n",
       "Q 53.21875 48.921875 51.53125 44.890625 \r\n",
       "Q 49.859375 40.875 45.40625 35.40625 \r\n",
       "Q 44.1875 33.984375 37.640625 27.21875 \r\n",
       "Q 31.109375 20.453125 19.1875 8.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-50\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_2\">\r\n",
       "     <g id=\"line2d_3\">\r\n",
       "      <path clip-path=\"url(#pf831abab33)\" d=\"M 115.24375 143.728583 \r\n",
       "L 115.24375 7.828583 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_4\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"115.24375\" xlink:href=\"#mea90d692ed\" y=\"143.728583\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_2\">\r\n",
       "      <!-- 4 -->\r\n",
       "      <g transform=\"translate(112.0625 158.327021)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 37.796875 64.3125 \r\n",
       "L 12.890625 25.390625 \r\n",
       "L 37.796875 25.390625 \r\n",
       "z\r\n",
       "M 35.203125 72.90625 \r\n",
       "L 47.609375 72.90625 \r\n",
       "L 47.609375 25.390625 \r\n",
       "L 58.015625 25.390625 \r\n",
       "L 58.015625 17.1875 \r\n",
       "L 47.609375 17.1875 \r\n",
       "L 47.609375 0 \r\n",
       "L 37.796875 0 \r\n",
       "L 37.796875 17.1875 \r\n",
       "L 4.890625 17.1875 \r\n",
       "L 4.890625 26.703125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-52\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-52\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_3\">\r\n",
       "     <g id=\"line2d_5\">\r\n",
       "      <path clip-path=\"url(#pf831abab33)\" d=\"M 158.64375 143.728583 \r\n",
       "L 158.64375 7.828583 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_6\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"158.64375\" xlink:href=\"#mea90d692ed\" y=\"143.728583\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_3\">\r\n",
       "      <!-- 6 -->\r\n",
       "      <g transform=\"translate(155.4625 158.327021)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 33.015625 40.375 \r\n",
       "Q 26.375 40.375 22.484375 35.828125 \r\n",
       "Q 18.609375 31.296875 18.609375 23.390625 \r\n",
       "Q 18.609375 15.53125 22.484375 10.953125 \r\n",
       "Q 26.375 6.390625 33.015625 6.390625 \r\n",
       "Q 39.65625 6.390625 43.53125 10.953125 \r\n",
       "Q 47.40625 15.53125 47.40625 23.390625 \r\n",
       "Q 47.40625 31.296875 43.53125 35.828125 \r\n",
       "Q 39.65625 40.375 33.015625 40.375 \r\n",
       "z\r\n",
       "M 52.59375 71.296875 \r\n",
       "L 52.59375 62.3125 \r\n",
       "Q 48.875 64.0625 45.09375 64.984375 \r\n",
       "Q 41.3125 65.921875 37.59375 65.921875 \r\n",
       "Q 27.828125 65.921875 22.671875 59.328125 \r\n",
       "Q 17.53125 52.734375 16.796875 39.40625 \r\n",
       "Q 19.671875 43.65625 24.015625 45.921875 \r\n",
       "Q 28.375 48.1875 33.59375 48.1875 \r\n",
       "Q 44.578125 48.1875 50.953125 41.515625 \r\n",
       "Q 57.328125 34.859375 57.328125 23.390625 \r\n",
       "Q 57.328125 12.15625 50.6875 5.359375 \r\n",
       "Q 44.046875 -1.421875 33.015625 -1.421875 \r\n",
       "Q 20.359375 -1.421875 13.671875 8.265625 \r\n",
       "Q 6.984375 17.96875 6.984375 36.375 \r\n",
       "Q 6.984375 53.65625 15.1875 63.9375 \r\n",
       "Q 23.390625 74.21875 37.203125 74.21875 \r\n",
       "Q 40.921875 74.21875 44.703125 73.484375 \r\n",
       "Q 48.484375 72.75 52.59375 71.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-54\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-54\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_4\">\r\n",
       "     <g id=\"line2d_7\">\r\n",
       "      <path clip-path=\"url(#pf831abab33)\" d=\"M 202.04375 143.728583 \r\n",
       "L 202.04375 7.828583 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_8\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"202.04375\" xlink:href=\"#mea90d692ed\" y=\"143.728583\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_4\">\r\n",
       "      <!-- 8 -->\r\n",
       "      <g transform=\"translate(198.8625 158.327021)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 31.78125 34.625 \r\n",
       "Q 24.75 34.625 20.71875 30.859375 \r\n",
       "Q 16.703125 27.09375 16.703125 20.515625 \r\n",
       "Q 16.703125 13.921875 20.71875 10.15625 \r\n",
       "Q 24.75 6.390625 31.78125 6.390625 \r\n",
       "Q 38.8125 6.390625 42.859375 10.171875 \r\n",
       "Q 46.921875 13.96875 46.921875 20.515625 \r\n",
       "Q 46.921875 27.09375 42.890625 30.859375 \r\n",
       "Q 38.875 34.625 31.78125 34.625 \r\n",
       "z\r\n",
       "M 21.921875 38.8125 \r\n",
       "Q 15.578125 40.375 12.03125 44.71875 \r\n",
       "Q 8.5 49.078125 8.5 55.328125 \r\n",
       "Q 8.5 64.0625 14.71875 69.140625 \r\n",
       "Q 20.953125 74.21875 31.78125 74.21875 \r\n",
       "Q 42.671875 74.21875 48.875 69.140625 \r\n",
       "Q 55.078125 64.0625 55.078125 55.328125 \r\n",
       "Q 55.078125 49.078125 51.53125 44.71875 \r\n",
       "Q 48 40.375 41.703125 38.8125 \r\n",
       "Q 48.828125 37.15625 52.796875 32.3125 \r\n",
       "Q 56.78125 27.484375 56.78125 20.515625 \r\n",
       "Q 56.78125 9.90625 50.3125 4.234375 \r\n",
       "Q 43.84375 -1.421875 31.78125 -1.421875 \r\n",
       "Q 19.734375 -1.421875 13.25 4.234375 \r\n",
       "Q 6.78125 9.90625 6.78125 20.515625 \r\n",
       "Q 6.78125 27.484375 10.78125 32.3125 \r\n",
       "Q 14.796875 37.15625 21.921875 38.8125 \r\n",
       "z\r\n",
       "M 18.3125 54.390625 \r\n",
       "Q 18.3125 48.734375 21.84375 45.5625 \r\n",
       "Q 25.390625 42.390625 31.78125 42.390625 \r\n",
       "Q 38.140625 42.390625 41.71875 45.5625 \r\n",
       "Q 45.3125 48.734375 45.3125 54.390625 \r\n",
       "Q 45.3125 60.0625 41.71875 63.234375 \r\n",
       "Q 38.140625 66.40625 31.78125 66.40625 \r\n",
       "Q 25.390625 66.40625 21.84375 63.234375 \r\n",
       "Q 18.3125 60.0625 18.3125 54.390625 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-56\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-56\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"xtick_5\">\r\n",
       "     <g id=\"line2d_9\">\r\n",
       "      <path clip-path=\"url(#pf831abab33)\" d=\"M 245.44375 143.728583 \r\n",
       "L 245.44375 7.828583 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_10\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"245.44375\" xlink:href=\"#mea90d692ed\" y=\"143.728583\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_5\">\r\n",
       "      <!-- 10 -->\r\n",
       "      <g transform=\"translate(239.08125 158.327021)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 12.40625 8.296875 \r\n",
       "L 28.515625 8.296875 \r\n",
       "L 28.515625 63.921875 \r\n",
       "L 10.984375 60.40625 \r\n",
       "L 10.984375 69.390625 \r\n",
       "L 28.421875 72.90625 \r\n",
       "L 38.28125 72.90625 \r\n",
       "L 38.28125 8.296875 \r\n",
       "L 54.390625 8.296875 \r\n",
       "L 54.390625 0 \r\n",
       "L 12.40625 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-49\"/>\r\n",
       "        <path d=\"M 31.78125 66.40625 \r\n",
       "Q 24.171875 66.40625 20.328125 58.90625 \r\n",
       "Q 16.5 51.421875 16.5 36.375 \r\n",
       "Q 16.5 21.390625 20.328125 13.890625 \r\n",
       "Q 24.171875 6.390625 31.78125 6.390625 \r\n",
       "Q 39.453125 6.390625 43.28125 13.890625 \r\n",
       "Q 47.125 21.390625 47.125 36.375 \r\n",
       "Q 47.125 51.421875 43.28125 58.90625 \r\n",
       "Q 39.453125 66.40625 31.78125 66.40625 \r\n",
       "z\r\n",
       "M 31.78125 74.21875 \r\n",
       "Q 44.046875 74.21875 50.515625 64.515625 \r\n",
       "Q 56.984375 54.828125 56.984375 36.375 \r\n",
       "Q 56.984375 17.96875 50.515625 8.265625 \r\n",
       "Q 44.046875 -1.421875 31.78125 -1.421875 \r\n",
       "Q 19.53125 -1.421875 13.0625 8.265625 \r\n",
       "Q 6.59375 17.96875 6.59375 36.375 \r\n",
       "Q 6.59375 54.828125 13.0625 64.515625 \r\n",
       "Q 19.53125 74.21875 31.78125 74.21875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-48\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-49\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"text_6\">\r\n",
       "     <!-- epoch -->\r\n",
       "     <g transform=\"translate(132.565625 172.005146)scale(0.1 -0.1)\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 56.203125 29.59375 \r\n",
       "L 56.203125 25.203125 \r\n",
       "L 14.890625 25.203125 \r\n",
       "Q 15.484375 15.921875 20.484375 11.0625 \r\n",
       "Q 25.484375 6.203125 34.421875 6.203125 \r\n",
       "Q 39.59375 6.203125 44.453125 7.46875 \r\n",
       "Q 49.3125 8.734375 54.109375 11.28125 \r\n",
       "L 54.109375 2.78125 \r\n",
       "Q 49.265625 0.734375 44.1875 -0.34375 \r\n",
       "Q 39.109375 -1.421875 33.890625 -1.421875 \r\n",
       "Q 20.796875 -1.421875 13.15625 6.1875 \r\n",
       "Q 5.515625 13.8125 5.515625 26.8125 \r\n",
       "Q 5.515625 40.234375 12.765625 48.109375 \r\n",
       "Q 20.015625 56 32.328125 56 \r\n",
       "Q 43.359375 56 49.78125 48.890625 \r\n",
       "Q 56.203125 41.796875 56.203125 29.59375 \r\n",
       "z\r\n",
       "M 47.21875 32.234375 \r\n",
       "Q 47.125 39.59375 43.09375 43.984375 \r\n",
       "Q 39.0625 48.390625 32.421875 48.390625 \r\n",
       "Q 24.90625 48.390625 20.390625 44.140625 \r\n",
       "Q 15.875 39.890625 15.1875 32.171875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-101\"/>\r\n",
       "       <path d=\"M 18.109375 8.203125 \r\n",
       "L 18.109375 -20.796875 \r\n",
       "L 9.078125 -20.796875 \r\n",
       "L 9.078125 54.6875 \r\n",
       "L 18.109375 54.6875 \r\n",
       "L 18.109375 46.390625 \r\n",
       "Q 20.953125 51.265625 25.265625 53.625 \r\n",
       "Q 29.59375 56 35.59375 56 \r\n",
       "Q 45.5625 56 51.78125 48.09375 \r\n",
       "Q 58.015625 40.1875 58.015625 27.296875 \r\n",
       "Q 58.015625 14.40625 51.78125 6.484375 \r\n",
       "Q 45.5625 -1.421875 35.59375 -1.421875 \r\n",
       "Q 29.59375 -1.421875 25.265625 0.953125 \r\n",
       "Q 20.953125 3.328125 18.109375 8.203125 \r\n",
       "z\r\n",
       "M 48.6875 27.296875 \r\n",
       "Q 48.6875 37.203125 44.609375 42.84375 \r\n",
       "Q 40.53125 48.484375 33.40625 48.484375 \r\n",
       "Q 26.265625 48.484375 22.1875 42.84375 \r\n",
       "Q 18.109375 37.203125 18.109375 27.296875 \r\n",
       "Q 18.109375 17.390625 22.1875 11.75 \r\n",
       "Q 26.265625 6.109375 33.40625 6.109375 \r\n",
       "Q 40.53125 6.109375 44.609375 11.75 \r\n",
       "Q 48.6875 17.390625 48.6875 27.296875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-112\"/>\r\n",
       "       <path d=\"M 30.609375 48.390625 \r\n",
       "Q 23.390625 48.390625 19.1875 42.75 \r\n",
       "Q 14.984375 37.109375 14.984375 27.296875 \r\n",
       "Q 14.984375 17.484375 19.15625 11.84375 \r\n",
       "Q 23.34375 6.203125 30.609375 6.203125 \r\n",
       "Q 37.796875 6.203125 41.984375 11.859375 \r\n",
       "Q 46.1875 17.53125 46.1875 27.296875 \r\n",
       "Q 46.1875 37.015625 41.984375 42.703125 \r\n",
       "Q 37.796875 48.390625 30.609375 48.390625 \r\n",
       "z\r\n",
       "M 30.609375 56 \r\n",
       "Q 42.328125 56 49.015625 48.375 \r\n",
       "Q 55.71875 40.765625 55.71875 27.296875 \r\n",
       "Q 55.71875 13.875 49.015625 6.21875 \r\n",
       "Q 42.328125 -1.421875 30.609375 -1.421875 \r\n",
       "Q 18.84375 -1.421875 12.171875 6.21875 \r\n",
       "Q 5.515625 13.875 5.515625 27.296875 \r\n",
       "Q 5.515625 40.765625 12.171875 48.375 \r\n",
       "Q 18.84375 56 30.609375 56 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-111\"/>\r\n",
       "       <path d=\"M 48.78125 52.59375 \r\n",
       "L 48.78125 44.1875 \r\n",
       "Q 44.96875 46.296875 41.140625 47.34375 \r\n",
       "Q 37.3125 48.390625 33.40625 48.390625 \r\n",
       "Q 24.65625 48.390625 19.8125 42.84375 \r\n",
       "Q 14.984375 37.3125 14.984375 27.296875 \r\n",
       "Q 14.984375 17.28125 19.8125 11.734375 \r\n",
       "Q 24.65625 6.203125 33.40625 6.203125 \r\n",
       "Q 37.3125 6.203125 41.140625 7.25 \r\n",
       "Q 44.96875 8.296875 48.78125 10.40625 \r\n",
       "L 48.78125 2.09375 \r\n",
       "Q 45.015625 0.34375 40.984375 -0.53125 \r\n",
       "Q 36.96875 -1.421875 32.421875 -1.421875 \r\n",
       "Q 20.0625 -1.421875 12.78125 6.34375 \r\n",
       "Q 5.515625 14.109375 5.515625 27.296875 \r\n",
       "Q 5.515625 40.671875 12.859375 48.328125 \r\n",
       "Q 20.21875 56 33.015625 56 \r\n",
       "Q 37.15625 56 41.109375 55.140625 \r\n",
       "Q 45.0625 54.296875 48.78125 52.59375 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-99\"/>\r\n",
       "       <path d=\"M 54.890625 33.015625 \r\n",
       "L 54.890625 0 \r\n",
       "L 45.90625 0 \r\n",
       "L 45.90625 32.71875 \r\n",
       "Q 45.90625 40.484375 42.875 44.328125 \r\n",
       "Q 39.84375 48.1875 33.796875 48.1875 \r\n",
       "Q 26.515625 48.1875 22.3125 43.546875 \r\n",
       "Q 18.109375 38.921875 18.109375 30.90625 \r\n",
       "L 18.109375 0 \r\n",
       "L 9.078125 0 \r\n",
       "L 9.078125 75.984375 \r\n",
       "L 18.109375 75.984375 \r\n",
       "L 18.109375 46.1875 \r\n",
       "Q 21.34375 51.125 25.703125 53.5625 \r\n",
       "Q 30.078125 56 35.796875 56 \r\n",
       "Q 45.21875 56 50.046875 50.171875 \r\n",
       "Q 54.890625 44.34375 54.890625 33.015625 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-104\"/>\r\n",
       "      </defs>\r\n",
       "      <use xlink:href=\"#DejaVuSans-101\"/>\r\n",
       "      <use x=\"61.523438\" xlink:href=\"#DejaVuSans-112\"/>\r\n",
       "      <use x=\"125\" xlink:href=\"#DejaVuSans-111\"/>\r\n",
       "      <use x=\"186.181641\" xlink:href=\"#DejaVuSans-99\"/>\r\n",
       "      <use x=\"241.162109\" xlink:href=\"#DejaVuSans-104\"/>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"matplotlib.axis_2\">\r\n",
       "    <g id=\"ytick_1\">\r\n",
       "     <g id=\"line2d_11\">\r\n",
       "      <path clip-path=\"url(#pf831abab33)\" d=\"M 50.14375 120.331479 \r\n",
       "L 245.44375 120.331479 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_12\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 0 0 \r\n",
       "L -3.5 0 \r\n",
       "\" id=\"m01a5cd0860\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n",
       "      </defs>\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#m01a5cd0860\" y=\"120.331479\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_7\">\r\n",
       "      <!-- 0.84 -->\r\n",
       "      <g transform=\"translate(20.878125 124.130698)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 10.6875 12.40625 \r\n",
       "L 21 12.40625 \r\n",
       "L 21 0 \r\n",
       "L 10.6875 0 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-46\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-52\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_2\">\r\n",
       "     <g id=\"line2d_13\">\r\n",
       "      <path clip-path=\"url(#pf831abab33)\" d=\"M 50.14375 92.998414 \r\n",
       "L 245.44375 92.998414 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_14\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#m01a5cd0860\" y=\"92.998414\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_8\">\r\n",
       "      <!-- 0.86 -->\r\n",
       "      <g transform=\"translate(20.878125 96.797633)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-54\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_3\">\r\n",
       "     <g id=\"line2d_15\">\r\n",
       "      <path clip-path=\"url(#pf831abab33)\" d=\"M 50.14375 65.665349 \r\n",
       "L 245.44375 65.665349 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_16\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#m01a5cd0860\" y=\"65.665349\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_9\">\r\n",
       "      <!-- 0.88 -->\r\n",
       "      <g transform=\"translate(20.878125 69.464568)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-56\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_4\">\r\n",
       "     <g id=\"line2d_17\">\r\n",
       "      <path clip-path=\"url(#pf831abab33)\" d=\"M 50.14375 38.332284 \r\n",
       "L 245.44375 38.332284 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_18\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#m01a5cd0860\" y=\"38.332284\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_10\">\r\n",
       "      <!-- 0.90 -->\r\n",
       "      <g transform=\"translate(20.878125 42.131503)scale(0.1 -0.1)\">\r\n",
       "       <defs>\r\n",
       "        <path d=\"M 10.984375 1.515625 \r\n",
       "L 10.984375 10.5 \r\n",
       "Q 14.703125 8.734375 18.5 7.8125 \r\n",
       "Q 22.3125 6.890625 25.984375 6.890625 \r\n",
       "Q 35.75 6.890625 40.890625 13.453125 \r\n",
       "Q 46.046875 20.015625 46.78125 33.40625 \r\n",
       "Q 43.953125 29.203125 39.59375 26.953125 \r\n",
       "Q 35.25 24.703125 29.984375 24.703125 \r\n",
       "Q 19.046875 24.703125 12.671875 31.3125 \r\n",
       "Q 6.296875 37.9375 6.296875 49.421875 \r\n",
       "Q 6.296875 60.640625 12.9375 67.421875 \r\n",
       "Q 19.578125 74.21875 30.609375 74.21875 \r\n",
       "Q 43.265625 74.21875 49.921875 64.515625 \r\n",
       "Q 56.59375 54.828125 56.59375 36.375 \r\n",
       "Q 56.59375 19.140625 48.40625 8.859375 \r\n",
       "Q 40.234375 -1.421875 26.421875 -1.421875 \r\n",
       "Q 22.703125 -1.421875 18.890625 -0.6875 \r\n",
       "Q 15.09375 0.046875 10.984375 1.515625 \r\n",
       "z\r\n",
       "M 30.609375 32.421875 \r\n",
       "Q 37.25 32.421875 41.125 36.953125 \r\n",
       "Q 45.015625 41.5 45.015625 49.421875 \r\n",
       "Q 45.015625 57.28125 41.125 61.84375 \r\n",
       "Q 37.25 66.40625 30.609375 66.40625 \r\n",
       "Q 23.96875 66.40625 20.09375 61.84375 \r\n",
       "Q 16.21875 57.28125 16.21875 49.421875 \r\n",
       "Q 16.21875 41.5 20.09375 36.953125 \r\n",
       "Q 23.96875 32.421875 30.609375 32.421875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-57\"/>\r\n",
       "       </defs>\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-57\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"ytick_5\">\r\n",
       "     <g id=\"line2d_19\">\r\n",
       "      <path clip-path=\"url(#pf831abab33)\" d=\"M 50.14375 10.999219 \r\n",
       "L 245.44375 10.999219 \r\n",
       "\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\r\n",
       "     </g>\r\n",
       "     <g id=\"line2d_20\">\r\n",
       "      <g>\r\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"50.14375\" xlink:href=\"#m01a5cd0860\" y=\"10.999219\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "     <g id=\"text_11\">\r\n",
       "      <!-- 0.92 -->\r\n",
       "      <g transform=\"translate(20.878125 14.798437)scale(0.1 -0.1)\">\r\n",
       "       <use xlink:href=\"#DejaVuSans-48\"/>\r\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n",
       "       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-57\"/>\r\n",
       "       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-50\"/>\r\n",
       "      </g>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "    <g id=\"text_12\">\r\n",
       "     <!-- test acc -->\r\n",
       "     <g transform=\"translate(14.798437 95.531708)rotate(-90)scale(0.1 -0.1)\">\r\n",
       "      <defs>\r\n",
       "       <path d=\"M 18.3125 70.21875 \r\n",
       "L 18.3125 54.6875 \r\n",
       "L 36.8125 54.6875 \r\n",
       "L 36.8125 47.703125 \r\n",
       "L 18.3125 47.703125 \r\n",
       "L 18.3125 18.015625 \r\n",
       "Q 18.3125 11.328125 20.140625 9.421875 \r\n",
       "Q 21.96875 7.515625 27.59375 7.515625 \r\n",
       "L 36.8125 7.515625 \r\n",
       "L 36.8125 0 \r\n",
       "L 27.59375 0 \r\n",
       "Q 17.1875 0 13.234375 3.875 \r\n",
       "Q 9.28125 7.765625 9.28125 18.015625 \r\n",
       "L 9.28125 47.703125 \r\n",
       "L 2.6875 47.703125 \r\n",
       "L 2.6875 54.6875 \r\n",
       "L 9.28125 54.6875 \r\n",
       "L 9.28125 70.21875 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-116\"/>\r\n",
       "       <path d=\"M 44.28125 53.078125 \r\n",
       "L 44.28125 44.578125 \r\n",
       "Q 40.484375 46.53125 36.375 47.5 \r\n",
       "Q 32.28125 48.484375 27.875 48.484375 \r\n",
       "Q 21.1875 48.484375 17.84375 46.4375 \r\n",
       "Q 14.5 44.390625 14.5 40.28125 \r\n",
       "Q 14.5 37.15625 16.890625 35.375 \r\n",
       "Q 19.28125 33.59375 26.515625 31.984375 \r\n",
       "L 29.59375 31.296875 \r\n",
       "Q 39.15625 29.25 43.1875 25.515625 \r\n",
       "Q 47.21875 21.78125 47.21875 15.09375 \r\n",
       "Q 47.21875 7.46875 41.1875 3.015625 \r\n",
       "Q 35.15625 -1.421875 24.609375 -1.421875 \r\n",
       "Q 20.21875 -1.421875 15.453125 -0.5625 \r\n",
       "Q 10.6875 0.296875 5.421875 2 \r\n",
       "L 5.421875 11.28125 \r\n",
       "Q 10.40625 8.6875 15.234375 7.390625 \r\n",
       "Q 20.0625 6.109375 24.8125 6.109375 \r\n",
       "Q 31.15625 6.109375 34.5625 8.28125 \r\n",
       "Q 37.984375 10.453125 37.984375 14.40625 \r\n",
       "Q 37.984375 18.0625 35.515625 20.015625 \r\n",
       "Q 33.0625 21.96875 24.703125 23.78125 \r\n",
       "L 21.578125 24.515625 \r\n",
       "Q 13.234375 26.265625 9.515625 29.90625 \r\n",
       "Q 5.8125 33.546875 5.8125 39.890625 \r\n",
       "Q 5.8125 47.609375 11.28125 51.796875 \r\n",
       "Q 16.75 56 26.8125 56 \r\n",
       "Q 31.78125 56 36.171875 55.265625 \r\n",
       "Q 40.578125 54.546875 44.28125 53.078125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-115\"/>\r\n",
       "       <path id=\"DejaVuSans-32\"/>\r\n",
       "       <path d=\"M 34.28125 27.484375 \r\n",
       "Q 23.390625 27.484375 19.1875 25 \r\n",
       "Q 14.984375 22.515625 14.984375 16.5 \r\n",
       "Q 14.984375 11.71875 18.140625 8.90625 \r\n",
       "Q 21.296875 6.109375 26.703125 6.109375 \r\n",
       "Q 34.1875 6.109375 38.703125 11.40625 \r\n",
       "Q 43.21875 16.703125 43.21875 25.484375 \r\n",
       "L 43.21875 27.484375 \r\n",
       "z\r\n",
       "M 52.203125 31.203125 \r\n",
       "L 52.203125 0 \r\n",
       "L 43.21875 0 \r\n",
       "L 43.21875 8.296875 \r\n",
       "Q 40.140625 3.328125 35.546875 0.953125 \r\n",
       "Q 30.953125 -1.421875 24.3125 -1.421875 \r\n",
       "Q 15.921875 -1.421875 10.953125 3.296875 \r\n",
       "Q 6 8.015625 6 15.921875 \r\n",
       "Q 6 25.140625 12.171875 29.828125 \r\n",
       "Q 18.359375 34.515625 30.609375 34.515625 \r\n",
       "L 43.21875 34.515625 \r\n",
       "L 43.21875 35.40625 \r\n",
       "Q 43.21875 41.609375 39.140625 45 \r\n",
       "Q 35.0625 48.390625 27.6875 48.390625 \r\n",
       "Q 23 48.390625 18.546875 47.265625 \r\n",
       "Q 14.109375 46.140625 10.015625 43.890625 \r\n",
       "L 10.015625 52.203125 \r\n",
       "Q 14.9375 54.109375 19.578125 55.046875 \r\n",
       "Q 24.21875 56 28.609375 56 \r\n",
       "Q 40.484375 56 46.34375 49.84375 \r\n",
       "Q 52.203125 43.703125 52.203125 31.203125 \r\n",
       "z\r\n",
       "\" id=\"DejaVuSans-97\"/>\r\n",
       "      </defs>\r\n",
       "      <use xlink:href=\"#DejaVuSans-116\"/>\r\n",
       "      <use x=\"39.208984\" xlink:href=\"#DejaVuSans-101\"/>\r\n",
       "      <use x=\"100.732422\" xlink:href=\"#DejaVuSans-115\"/>\r\n",
       "      <use x=\"152.832031\" xlink:href=\"#DejaVuSans-116\"/>\r\n",
       "      <use x=\"192.041016\" xlink:href=\"#DejaVuSans-32\"/>\r\n",
       "      <use x=\"223.828125\" xlink:href=\"#DejaVuSans-97\"/>\r\n",
       "      <use x=\"285.107422\" xlink:href=\"#DejaVuSans-99\"/>\r\n",
       "      <use x=\"340.087891\" xlink:href=\"#DejaVuSans-99\"/>\r\n",
       "     </g>\r\n",
       "    </g>\r\n",
       "   </g>\r\n",
       "   <g id=\"line2d_21\">\r\n",
       "    <path clip-path=\"url(#pf831abab33)\" d=\"M 50.14375 137.55131 \r\n",
       "L 71.84375 80.56187 \r\n",
       "L 93.54375 43.252236 \r\n",
       "L 115.24375 48.718849 \r\n",
       "L 136.94375 27.672389 \r\n",
       "L 158.64375 24.939082 \r\n",
       "L 180.34375 17.69582 \r\n",
       "L 202.04375 22.615771 \r\n",
       "L 223.74375 40.79226 \r\n",
       "L 245.44375 14.005856 \r\n",
       "\" style=\"fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_3\">\r\n",
       "    <path d=\"M 50.14375 143.728583 \r\n",
       "L 50.14375 7.828583 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_4\">\r\n",
       "    <path d=\"M 245.44375 143.728583 \r\n",
       "L 245.44375 7.828583 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_5\">\r\n",
       "    <path d=\"M 50.14375 143.728583 \r\n",
       "L 245.44375 143.728583 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "   <g id=\"patch_6\">\r\n",
       "    <path d=\"M 50.14375 7.828583 \r\n",
       "L 245.44375 7.828583 \r\n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n",
       "   </g>\r\n",
       "  </g>\r\n",
       " </g>\r\n",
       " <defs>\r\n",
       "  <clipPath id=\"pf831abab33\">\r\n",
       "   <rect height=\"135.9\" width=\"195.3\" x=\"50.14375\" y=\"7.828583\"/>\r\n",
       "  </clipPath>\r\n",
       " </defs>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<Figure size 252x180 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 在单个GPU上训练网络\n",
    "train(net, num_gpus=1, batch_size=256, lr=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Expected a non cpu device, but got: cpu",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-767446712b65>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# 使用2个GPU进行训练\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnet\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnum_gpus\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m512\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-15-d7b4a8ff7033>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(net, num_gpus, batch_size, lr)\u001b[0m\n\u001b[0;32m     10\u001b[0m     \u001b[0mnet\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minit_weights\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[1;31m# nn.DataParallel会的是X切开并行到各个GPU上，并行算梯度，然后loss加起来，它重新定义了net的forward函数\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m     \u001b[0mnet\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataParallel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnet\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdevice_ids\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdevices\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# net会复制到每一个GPU上\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m     \u001b[0mtrainer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptim\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSGD\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnet\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mlr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m     \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCrossEntropyLoss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\nn\\parallel\\data_parallel.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, module, device_ids, output_device, dim)\u001b[0m\n\u001b[0;32m    136\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdim\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdim\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    137\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodule\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 138\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice_ids\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0m_get_device_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdevice_ids\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    139\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutput_device\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_get_device_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutput_device\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    140\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msrc_device_obj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice_ids\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\nn\\parallel\\data_parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    136\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdim\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdim\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    137\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodule\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 138\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice_ids\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0m_get_device_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdevice_ids\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    139\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutput_device\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_get_device_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutput_device\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    140\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msrc_device_obj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice_ids\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\11_Anaconda\\envs\\py3.6.3\\lib\\site-packages\\torch\\_utils.py\u001b[0m in \u001b[0;36m_get_device_index\u001b[1;34m(device, optional, allow_cpu)\u001b[0m\n\u001b[0;32m    495\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    496\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mallow_cpu\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mdevice\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'cpu'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 497\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Expected a non cpu device, but got: {}'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    498\u001b[0m         \u001b[0mdevice_idx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mdevice\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'cpu'\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mdevice\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    499\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mint\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Expected a non cpu device, but got: cpu"
     ]
    }
   ],
   "source": [
    "# 使用2个GPU进行训练\n",
    "train(net, num_gpus=2, batch_size=512, lr=0.2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.3",
   "language": "python",
   "name": "python3.6.3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "2041.23px",
    "left": "45.9792px",
    "top": "56px",
    "width": "301.875px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}