{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A simple neural network with one hidden layer in pure Python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A simple neural network class with ReLU activation function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A simple feed-forward neutral network with on hidden layer\n",
    "# see also https://towardsdatascience.com/how-to-build-your-own-neural-network-from-scratch-in-python-68998a08e4f6\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "class NeuralNetwork:\n",
    "    def __init__(self, x, y):\n",
    "        n1 = 4 # number of neurons in the hidden layer\n",
    "        self.input         = x\n",
    "        self.weights1      = np.random.rand(self.input.shape[1],n1)\n",
    "        self.bias1         = np.random.rand(n1)\n",
    "        self.weights2      = np.random.rand(n1,1)\n",
    "        self.bias2         = np.random.rand(1)                 \n",
    "        self.y             = y\n",
    "        self.output        = np.zeros(y.shape)\n",
    "        self.learning_rate = 0.00001\n",
    "        self.n_train       = 0\n",
    "        self.loss_history  = []\n",
    "\n",
    "    def relu(self, x):\n",
    "        return np.where(x>0, x, 0)\n",
    "    \n",
    "    def relu_derivative(self, x):\n",
    "        return np.where(x>0, 1, 0)\n",
    "\n",
    "    def feedforward(self):\n",
    "        self.layer1 = self.relu(self.input @ self.weights1 + self.bias1)\n",
    "        self.output = self.relu(self.layer1 @ self.weights2 + self.bias2)\n",
    "\n",
    "    def backprop(self):\n",
    "\n",
    "        # delta1: [m, 1], m = number of training data\n",
    "        delta1 = 2 * (self.y - self.output) * self.relu_derivative(self.output)\n",
    "\n",
    "        # Gradient w.r.t. weights from hidden to output layer: [n1, 1] matrix, n1 = # neurons in hidden layer\n",
    "        d_weights2 = self.layer1.T @ delta1\n",
    "        d_bias2    = np.sum(delta1) \n",
    "    \n",
    "        # shape of delta2: [m, n1], m = number of training data, n1 = # neurons in hidden layer\n",
    "        delta2     = (delta1 @ self.weights2.T) * self.relu_derivative(self.layer1)\n",
    "        d_weights1 = self.input.T @ delta2\n",
    "        d_bias1    = np.ones(delta2.shape[0]) @ delta2 \n",
    "        \n",
    "        # update weights and biases\n",
    "        self.weights1 += self.learning_rate * d_weights1\n",
    "        self.weights2 += self.learning_rate * d_weights2\n",
    "\n",
    "        self.bias1    += self.learning_rate * d_bias1\n",
    "        self.bias2    += self.learning_rate * d_bias2\n",
    "\n",
    "    def train(self, X, y):\n",
    "        self.output = np.zeros(y.shape)\n",
    "        self.input = X\n",
    "        self.y = y\n",
    "        self.feedforward()\n",
    "        self.backprop()\n",
    "        self.n_train += 1\n",
    "        if (self.n_train %1000 == 0):\n",
    "            loss = np.sum((self.y - self.output)**2)\n",
    "            print(\"loss: \", loss)\n",
    "            self.loss_history.append(loss)\n",
    "    \n",
    "    def predict(self, X):\n",
    "        self.output = np.zeros(y.shape)\n",
    "        self.input = X\n",
    "        self.feedforward()\n",
    "        return self.output\n",
    "    \n",
    "    def loss_history(self):\n",
    "        return self.loss_history\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create toy data\n",
    "We create three toy data sets\n",
    "1. two moon-like distributions\n",
    "2. circles\n",
    "3. linearly separable data sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html#sphx-glr-auto-examples-classification-plot-classifier-comparison-py\n",
    "import numpy as np\n",
    "from sklearn.datasets import make_moons, make_circles, make_classification\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X, y = make_classification(\n",
    "    n_features=2, n_redundant=0, n_informative=2, random_state=1, n_clusters_per_class=1\n",
    ")\n",
    "rng = np.random.RandomState(2)\n",
    "X += 2 * rng.uniform(size=X.shape)\n",
    "linearly_separable = (X, y)\n",
    "\n",
    "datasets = [\n",
    "    make_moons(n_samples=200, noise=0.1, random_state=0),\n",
    "    make_circles(n_samples=200, noise=0.1, factor=0.5, random_state=1),\n",
    "    linearly_separable,\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create training and test data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# datasets: 0 = moons, 1 = circles, 2 = linearly separable\n",
    "X, y = datasets[1]\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.4, random_state=42\n",
    ")\n",
    "\n",
    "x_min, x_max = X[:, 0].min() - 0.5, X[:, 0].max() + 0.5\n",
    "y_min, y_max = X[:, 1].min() - 0.5, X[:, 1].max() + 0.5\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = y_train.reshape(-1, 1)\n",
    "\n",
    "nn = NeuralNetwork(X_train, y_train)\n",
    "\n",
    "for i in range(1000000):\n",
    "    nn.train(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot the loss vs. the number of epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(nn.loss_history)\n",
    "plt.xlabel(\"# epochs / 1000\")\n",
    "plt.ylabel(\"loss\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.colors import ListedColormap\n",
    "\n",
    "cm = plt.cm.RdBu\n",
    "cm_bright = ListedColormap([\"#FF0000\", \"#0000FF\"])\n",
    "\n",
    "xv = np.linspace(x_min, x_max, 10)\n",
    "yv = np.linspace(y_min, y_max, 10)\n",
    "Xv, Yv = np.meshgrid(xv, yv)\n",
    "XYpairs = np.vstack([ Xv.reshape(-1), Yv.reshape(-1)])\n",
    "zv = nn.predict(XYpairs.T)\n",
    "Zv = zv.reshape(Xv.shape)\n",
    "\n",
    "fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(9, 7))\n",
    "ax.set_aspect(1)\n",
    "cn = ax.contourf(Xv, Yv, Zv, cmap=\"coolwarm_r\", alpha=0.4)\n",
    "\n",
    "ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright, edgecolors=\"k\")\n",
    "\n",
    "# Plot the testing points\n",
    "ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.4, edgecolors=\"k\")\n",
    "\n",
    "ax.set_xlim(x_min, x_max)\n",
    "ax.set_ylim(y_min, y_max)\n",
    "# ax.set_xticks(())\n",
    "# ax.set_yticks(())\n",
    "\n",
    "fig.colorbar(cn)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
