backpropagation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6f4e03cf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-31T14:12:58.849666Z",
     "start_time": "2024-10-31T14:12:58.842784Z"
    }
   },
   "outputs": [],
   "source": [
    "from copy import deepcopy\n",
    "from math import exp\n",
    "from random import random, seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f96a2774",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-31T14:12:58.879613Z",
     "start_time": "2024-10-31T14:12:58.857547Z"
    },
    "run_control": {
     "marked": false
    }
   },
   "outputs": [],
   "source": [
    "class NeuralNetwork(object):\n",
    "    \"\"\"This class is designed to implement neural network.\n",
    "    \"\"\"\n",
    "    def __init__(self, n_inputs, n_hidden, n_outputs, n_epochs, learning_rate):\n",
    "        \"\"\"Initialize a neural network.\"\"\"\n",
    "        # Algorithm Line 1-2\n",
    "        self.network = list()\n",
    "        hidden_layer = [{'weights':[random() for w in range(n_inputs + 1)]} for n in range(n_hidden)]\n",
    "        self.network.append(hidden_layer)\n",
    "        output_layer = [{'weights':[random() for w in range(n_hidden + 1)]} for n in range(n_outputs)]\n",
    "        self.network.append(output_layer)\n",
    "        self.n_outputs = n_outputs\n",
    "        self.n_epochs = n_epochs\n",
    "        self.learning_rate = learning_rate\n",
    "\n",
    "    @staticmethod\n",
    "    def weighted_sum(weights, inputs):\n",
    "        \"\"\"Calculate weighted sum for inputs.\"\"\"\n",
    "        # keep bias term alive with constant 1.0 as input\n",
    "        activation = weights[-1] * 1.0\n",
    "        for i in range(len(weights)-1):\n",
    "            activation += weights[i] * inputs[i]\n",
    "        return activation\n",
    "\n",
    "    @staticmethod\n",
    "    def sigmoid(activation):\n",
    "        \"\"\"Define a sigmoid activation function, also called squashing function,\n",
    "        whose outputs are in range of (0,1); and its gradient f'(x)=f(x)*(1-f(x)).\n",
    "        \"\"\"\n",
    "        return 1.0 / (1.0 + exp(-activation))\n",
    "\n",
    "    @staticmethod\n",
    "    def sigmoid_derivative(output):\n",
    "        \"\"\"Calculate the derivative of sigmoid function.\"\"\"\n",
    "        return output * (1.0 - output)\n",
    "\n",
    "    def forward(self, data):\n",
    "        \"\"\"Propagate the inputs forward to compute the outputs.\"\"\"\n",
    "        inputs = deepcopy(data)\n",
    "        # Algorithm Line 5-11\n",
    "        for layer in self.network:\n",
    "            new_inputs = []\n",
    "            for neuron in layer:\n",
    "                activation = self.weighted_sum(neuron['weights'], inputs)\n",
    "                neuron['output'] = self.sigmoid(activation)\n",
    "                new_inputs.append(neuron['output'])\n",
    "            inputs = new_inputs\n",
    "        return inputs\n",
    "\n",
    "    def backward(self, expected):\n",
    "        \"\"\"Propagate the deltas backward from the output layer to the input layer.\"\"\"\n",
    "        # Algorithm Line 12-17\n",
    "        for l in reversed(range(len(self.network))):\n",
    "            layer = self.network[l]\n",
    "            errors = list()\n",
    "            # Algorithm Line 13\n",
    "            if l == len(self.network)-1:\n",
    "                for j in range(len(layer)):\n",
    "                    neuron = layer[j]\n",
    "                    errors.append(neuron['output'] - expected[j])\n",
    "            else:\n",
    "                # Algorithm Line 15\n",
    "                for i in range(len(layer)):\n",
    "                    error = 0.0\n",
    "                    # Algorithm Line 16\n",
    "                    for neuron in self.network[l+1]:\n",
    "                        error += (neuron['weights'][i] * neuron['delta'])\n",
    "                    errors.append(error)\n",
    "            for i in range(len(layer)):\n",
    "                neuron = layer[i]\n",
    "                neuron['delta'] = errors[i] * self.sigmoid_derivative(neuron['output'])\n",
    "    \n",
    "    def update(self, data):\n",
    "        \"\"\"Update the weights using the deltas. Stochastic gradient descent (SGD) is used,\n",
    "        the weights are updated after every training data.\n",
    "        \"\"\"\n",
    "        # Algorithm Line 18-20\n",
    "        for l in range(len(self.network)):\n",
    "            inputs = data[:-1]\n",
    "            if l != 0:\n",
    "                inputs = [neuron['output'] for neuron in self.network[l-1]]\n",
    "            for neuron in self.network[l]:\n",
    "                for j in range(len(inputs)):\n",
    "                    neuron['weights'][j] -= self.learning_rate * neuron['delta'] * inputs[j]\n",
    "                neuron['weights'][-1] -= self.learning_rate * neuron['delta'] * 1.0 # update bias term\n",
    "\n",
    "    def train(self, dataset):\n",
    "        \"\"\"Train this neural network.\"\"\"\n",
    "        # Algorithm Line 3\n",
    "        for epoch in range(self.n_epochs):\n",
    "            squared_loss = 0\n",
    "            # Algorithm Line 4\n",
    "            for data in dataset:\n",
    "                outputs = self.forward(data)\n",
    "                expected = [0]*int(self.n_outputs)\n",
    "                expected[data[-1]] = 1\n",
    "                squared_loss += sum([0.5*(expected[i]-outputs[i])**2 for i in range(len(expected))])\n",
    "                self.backward(expected)\n",
    "                self.update(data)\n",
    "#             print('epoch=%3d, learning_rate=%.2f, squared_loss=%.2f' % (epoch+1, self.learning_rate, squared_loss))\n",
    "\n",
    "    def predict(self, data):\n",
    "        \"\"\"Make a prediction with this trained neural network.\"\"\"\n",
    "        outputs = self.forward(data)\n",
    "        return outputs.index(max(outputs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c5fd61f8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-31T14:12:58.921526Z",
     "start_time": "2024-10-31T14:12:58.882093Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected=0, Predicted=0\n",
      "Expected=0, Predicted=0\n",
      "Expected=0, Predicted=0\n",
      "Expected=0, Predicted=0\n",
      "Expected=0, Predicted=0\n",
      "Expected=1, Predicted=1\n",
      "Expected=1, Predicted=1\n",
      "Expected=1, Predicted=1\n",
      "Expected=1, Predicted=1\n",
      "Expected=1, Predicted=1\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    \"\"\"Test on training a neural network with backpropagation algorithm.\"\"\"\n",
    "    seed(2024)\n",
    "    # dataset format in [input1, input2, label]\n",
    "    dataset = [\n",
    "        [  2.7810836, 2.550537003, 0],\n",
    "        [1.465489372, 2.362125076, 0],\n",
    "        [3.396561688, 4.400293529, 0],\n",
    "        [ 1.38807019, 1.850220317, 0],\n",
    "        [ 3.06407232, 3.005305973, 0],\n",
    "        [7.627531214, 2.759262235, 1],\n",
    "        [5.332441248, 2.088626775, 1],\n",
    "        [6.922596716,  1.77106367, 1],\n",
    "        [8.675418651,-0.242068655, 1],\n",
    "        [7.673756466, 3.508563011, 1],\n",
    "        ]\n",
    "    n_inputs = len(dataset[0]) - 1\n",
    "    n_outputs = len(set([data[-1] for data in dataset]))\n",
    "    nn = NeuralNetwork(n_inputs=n_inputs, n_hidden=6, n_outputs=n_outputs, n_epochs=60, learning_rate=0.3)\n",
    "    nn.train(dataset)\n",
    "    for data in dataset:\n",
    "        prediction = nn.predict(data)\n",
    "        print('Expected=%d, Predicted=%d' % (data[-1], prediction))"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}