{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Simple linear regression problem\n", "\n", "y = ax + b + e" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "# Data Generation\n", "#feature x and create our labels using a = 1, b = 2 and some Gaussian noise\n", "\n", "np.random.seed(42)\n", "x = np.random.rand(100, 1)\n", "y = 1 + 2 * x + .1 * np.random.randn(100, 1)\n", "\n", "# Shuffles the indices\n", "idx = np.arange(100)\n", "np.random.shuffle(idx)\n", "\n", "# Uses first 80 random indices for train\n", "train_idx = idx[:80]\n", "# Uses the remaining indices for validation\n", "val_idx = idx[80:]\n", "\n", "# Generates train and validation sets\n", "x_train, y_train = x[train_idx], y[train_idx]\n", "x_val, y_val = x[val_idx], y[val_idx]" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Text(0.5, 1.0, 'Generated data train')" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAHH9JREFUeJzt3XGwXOV53/HvT0jC3EiAdSUXirn31jHEYM8Y27cY6tahMZMQJjX/EAf3golLqgHhjlyTSTJR6zgkSidpaheKi6OMAQspDo3jxpTBnXFqU3BqcC42YGziltpCqBZGvgIJLNsIePrHORetVmf3nN095+zu2d9nZke7e9495z260nPffd/nfV9FBGZm1iwrhl0BMzMrn4O7mVkDObibmTWQg7uZWQM5uJuZNZCDu5lZAzm4m6Uk/aqkL/dQfpekC6usU5kkfULSvx12PaweDu7WlaTLJD0g6YeSnk6fb5KkYdetnaR7JP3asOuRRVJIev0Anx/4F0lEXB0RvzfIOWx8OLhbR5KuA24A/j1wCvD3gKuBdwCra67LyjqvN27892PtHNwtk6STgOuBTRHxmYh4LhJfj4iFiPhJWu54SX8sabek76df/U9Ij10gaY+k69JW/15J72+5RpHP/qakp4BbJb1a0l2S9kl6Jn3+2rT8VuCfADdJel7STen7b5D0BUn7JX1b0ntarj8t6U5JByV9FfjpnL+TKyQ9IWlJ0pa2Y+dK+oqkZ9P7vEnS6vTYvWmxh9O6/Uq3e8m47u3ADPDf0s//hqS59NvAVZJ2A19My/6FpKckHZB0r6Q3tpznNkm/X+RnY+PPwd06OR84HvhcTrk/BM4EzgFeD5wGfLjl+CnASen7VwEfl/TqHj67DpgFNpL8e701fT0D/Ai4CSAitgD3AR+IiDUR8QFJPwV8Afgz4DXAe4H/3BLwPg78GDgV+BfpI5Oks4GbgSuAvw9MA63B+CXgXwPrSf7u3gVsSuv2zrTMm9O63dHtXtpFxBXAbuCfpZ//o5bDPwucBfxC+vrzwBnp/X4N2Nnpnuj+s7FxFxF++HHMA7gceKrtvf8FPEsSiN4JCPgh8NMtZc4Hvps+vyAtu7Ll+NPAeQU/+wLwqi51PAd4puX1PcCvtbz+FeC+ts/8CfA7wHHAYeANLcf+APhyh2t9GPjzltc/ldbvwg7lPwj815bXAby+6L1kHN/Vei1gLj3n67p85uS0zEnp69uA38/72Qz7354f5TzcT2edLAHrJa2MiBcBIuIfAUjaQ9Ly3ABMAQ+2jK+KJHC+cp7lz6cOAWsKfnZfRPz4lYPSFPAx4CJguYW5VtJxEfFSxj3MAm+X9GzLeyuB29PrrwSebDn2RPZfBZC01l8pGxE/lLTUUrczgY8C8+l9rQQe7HSyPu6lk1fqJOk4YCvwyyT393J6aD1wIOOznX421gDulrFOvgL8BLikS5kfkLT+3hgRJ6ePkyKiSIAo8tn2JUuvA34GeHtEnEjy7QGSXwpZ5Z8E/mfL+U+OpFvjGmAf8CJwekv5mS713dtaNg3O0y3Hbwb+Djgjrdtvt9QrS969tOu0fGvr+/+c5Od1IUl3y1zOOa3BHNwtU0Q8C/wuSR/1pZLWSFoh6RySLgki4mXgT4GPSXoNgKTTJP1CxxMfOX8/n11L8gvhWUnrSLpXWn0feF3L67uAM9OB0FXp4x9KOittHX8W+IikqbRP/cou1/4M8EuS/nE6UHo9R///WQscBJ6X9Abgmpy65d1Lu/bPZ1lL8gt5ieTbwx/klLcGc3C3jiIZuPsQ8Bsk/bHfJ+mz/k2S/nfS548D90s6CPw1SYu0iF4/+x+BE0ha/fcD/73t+A3ApWn2yY0R8Rzw88BlwPeAp0gGcY9Py3+ApBviKZL+6Fs7XTgivglcSzI4uxd4BtjTUuTXSVrOz5H80rqj7RQfAT6VZtO8p8C9tPt3wL9JP//rHcpsJ+la+n/At9Lz2oRShDfrMDNrGrfczcwayMHdzKyBHNzNzBrIwd3MrIGGNolp/fr1MTc3N6zLm5mNpQcffPAHEbEhr9zQgvvc3ByLi4vDuryZ2ViS1G0m9SvcLWNm1kAO7mZmDeTgbmbWQA7uZmYN5OBuZtZADu5mZjXZuRPm5mDFiuTPnd32yRqQN+swM6vBzp2wcSMcOpS8fuKJ5DXAwkL513PL3cysBlu2HAnsyw4dSt6vgoO7mVkNdu/u7f1BObibmdVgpsMmjp3eH1RucJf0KklflfSwpG9K+t2MMsdLukPS45IekDRXRWXNzMbV1q0wNXX0e1NTyftVKNJy/wnwcxHxZuAc4CJJ57WVuQp4JiJeT7Kj+x+WW00zs/G2sADbtsHsLEjJn9u2VTOYCgWyZSLZh+/59OWq9NG+N98lJHtEQrKR8E2SFN7Dz8zsFQsL1QXzdoX63CUdJ+khkk2SvxARD7QVOQ14EiAiXgQOANMZ59koaVHS4r59+waruZnZCKszpz1LoeAeES9FxDnAa4FzJb2prYiyPpZxnm0RMR8R8xs25C5HbGY2lpZz2p94AiKO5LTXGeB7ypaJiGeBe4CL2g7tAU4HkLQSOAnYX0L9zMzGTt057VmKZMtskHRy+vwE4ELg79qK3QlcmT6/FPii+9vNbFLVndOepUjL/VTgS5IeAf6WpM/9LknXS3p3WuaTwLSkx4EPAb9VTXXNzEZf3TntWYpkyzwCvCXj/Q+3PP8x8MvlVs3MbDxt3Xr0OjJQbU57Fs9QNbNGGXaWCtSf057Fq0KaWWPUvfJiN3XmtGdxy93MGqOOLJXWbwbr1yePYX5L6MQtdzNrjKqzVNq/GSwtHTk2zG8JWdxyN7PGKJKlMkiffNY3g1aHDsHmzcXPVyUHdzNrjLyVFwedOVrkG8DS0mh0zzi4m1lj5GWp9Non397KX7euWD3qnInaiYY1kXR+fj4WFxeHcm0zm0wrViQt9nYSvPzy0e+1968DrFqVlH3hhfxrzc4mLf2ZmeSbQ1n98JIejIj5vHJuuZvZxOhl5mhWK//wYVi79sg3gxUdIqg03EXDwMHdzCZIL7shdepf378fdu1KWvrbtx97PunYbwd1LxoGDu5mNkF6mTlapJWfdb5OPd11LhoGDu5mNmEWFo60vHft6twXXrSV336+2dns89W5aBg4uJuZZep3fZi6N8LuxDNUzcw66Gd9mNa0yyqyZYpycDczK9mwFw0Dd8uYmTWSg7uZjYWy12kfhXXfq+RuGTMbeWWv0z5K675XxcsPmNnIm5tLAnC72dkk/XDY56uTlx8ws8Yoe532qtd9HwUO7mY28npZE2YY5xtFDu5mNvLKnhg0KhONquTgbmYjr9fZonmZMP3OPh0nHlA1s0bJWod9aqo5wdsDqmY21vrNQ+91t6Wmcp67mY2cQfLQJyETpgi33M1s5AzS+p6ETJgiHNzNbOQM0vqehEyYIhzczSxT3Wuv7NwJ69dnb1O3rEjrexIyYYpwcDezYyz3ede1yfPOnfD+98PSUucyU1Nw8cXFfuEU3W2pyZwKaWbHqHvtlU7Xa73uxRfDpz7V3BTHopwKaWZ9qzvjpNt5peQXyt13O8WxFw7uZnaMTn3b69ZV0w/frS99+ZhTHHvj4G5mx8jKOFm9Gg4erKYffutWWLXq2PdXrz6S5eIUx944uJvZMbIyTtauhcOHjy5XVrfIwgLceitMTx95b3oabrnlSH+6Uxx74wFVMytkxYrsFEUpyUqpw86dyS+T3buTFvvWrZM1mArFB1S9/ICZFTIzk53RUme3yMLC5AXzfrlbxswKKatbpOkbU48KB3czK6SMmZ91T46aZLl97pJOB7YDpwAvA9si4oa2MhcAnwO+m7712Yi4vtt53eduNnnGeWPqUVHmJKYXgesi4izgPOBaSWdnlLsvIs5JH10Du5mNp6JdKu3lNm3qPgvVuerlyx1QjYi9wN70+XOSHgNOA75Vcd3MbIQUXWM9q9zNN3c/t3PVy9dTn7ukOeAtwAMZh8+X9LCkz0t6Y4fPb5S0KGlx3759PVfWzIan0xrrmzfnl+vGuerVKBzcJa0B/hL4YEQcbDv8NWA2It4M/Cfgr7LOERHbImI+IuY3bNjQb53NbAg6dZ0sLR3dPdNLF8ukLsdbh0LBXdIqksC+MyI+2348Ig5GxPPp87uBVZLWl1pTMxuqbl0nrbNUi3axLA+iOrBXIze4SxLwSeCxiPhohzKnpOWQdG563i4rM5vZqOo0aNqt66S1tZ6VD5/FXTHVKjJD9R3AFcA3JD2UvvfbwAxARHwCuBS4RtKLwI+Ay2JY6xqYWd/yBk03b87eUKO1tb7cEt+ypXN2zPS0W+xV89oyZvaKvDz09uAP3TfMWN5hqXXBsVWrkkXCHNz74806zKxneWum9zNLNemw7fzaquHgbjaChrX+SpE103vZn3TLFnjhhaPfe+EF755UBwd3sxEzzPVXyl4z3bsnDY+Du9mI6TRZqI7WbhmLg7Xy7knD4+BuNmKa1Nr17knD4+BuNmKG2dotu0uo7G8CVpyDu9mIqaO122nAtoouoV4GYK083mbPbMS0TgKqYq/QbhOVmtQlNOk8iclswnSbqATeTGPUeRKTmWXq1jr3AGhzOLibTZhuA7YeAG0OB3ezMVHWrNW81rkHQJvBwd1sDJSZoujW+WTwgKrZGMhbrdEmhwdUzRrEKYrWKwd3szHgNVqsVw7uZmPAKYrWKwd3szHgQVDrlZcfMBsTCwsO5lacW+5mJRjWzklmnbjlbjagbgtxuaVtw+KWu9mAhrVzkr8tWDduuZsNaBg56P62YHnccjcb0KA56P20wIe5z6qNBwd3swENkoPe75oxnrFqeRzczQY0SA56vy1wz1i1PA7uZiXod5ncflvgnrFqeRzczYao3xa4Z6xaHgd3syHKaoFLSd973uCqN9WwbhzczYaotQUOSWBf3mJhkA05zBzczYZsuQU+O3sksC9zeqP1y8HdbEQ4vdHK5OBuNiKc3mhlcnA3GxFOb7QyObibjQinN1qZHNzNKtTrujFOb7SyOLibVaTXdWO8hK+VycHdrCK9rBvT7wJiZp04uJv1oJfWdS+pjV7C18qWG9wlnS7pS5Iek/RNSZszykjSjZIel/SIpLdWU12z4em1dd1LaqNz3K1sRVruLwLXRcRZwHnAtZLObivzi8AZ6WMjcHOptTQbAb22rntJbXSOu5UtN7hHxN6I+Fr6/DngMeC0tmKXANsjcT9wsqRTS6+t2YAGGbTstXXdS2qjc9ytbD3toSppDngL8EDbodOAJ1te70nf2ztA3cxKNei+ozMzyWey3u9kYaHYuZfLbNmS/LKYmUkCu1MhrV+FB1QlrQH+EvhgRBxsP5zxkWh/Q9JGSYuSFvft29dbTc0GNOigZdWta+e4W5kKBXdJq0gC+86I+GxGkT3A6S2vXwt8r71QRGyLiPmImN+wYUM/9TXr26CDlp5BauOkSLaMgE8Cj0XERzsUuxN4X5o1cx5wICLcJWMjpYxBS7eubVwUabm/A7gC+DlJD6WPiyVdLenqtMzdwHeAx4E/BTZVU12z/nnQ0iZJ7oBqRHyZ7D711jIBXFtWpcyq4EFLmySeoWqNUDTF0d0qNikc3G3slbUuixfusiZxcLexV8a6LF64y5rGwd3GXhnrsnjhLmsaB3cbe2WkOHrhLmsaB3cbe2WkOK5b19v7ZqPOwd3GnmeOmh2rp4XDzEZV0QW6Otm/v7f3zUadW+5meD11ax4HdzO8NIE1j4O79a1Jk37cb29N4z5368ugG1+MokH77c1GiVvu1hdP+jEbbQ7u1hdP+jEbbQ7uVkh7/3qnyT2jlF3SpDEBs145uFuurEW1Dh6E1auPLtdLdknVgdcLgdmkc3C3XFn964cPw9q1xbJL2gP5pk3VB16PCdikU7KJUv3m5+djcXFxKNe23qxYkQThdlKy6UU37Vk1y5/LOt/sbLKBRhkGqbPZKJP0YETM55Vzy91ydepHX7cuv2slqwXdqT1R5mCsZ5zapHNwt1xZszdXr0763fO6VnoJ2GUGXs84tUnn4G65smZvrl2b9Lu3yurTLhqwyw68nnFqk8597taXon3aWX3u7WZnk8DuwGuWr2ifu5cfsL7MzCRdMVnvt1oO2Jdfnn0eqbxBVDM7wt0y1pde+rQXFpLWeRYPcJpVw8Hd+tJrn7YHOM3q5W4Z61svqygul9uyJcmgmZlxP7tZlRzcrTZeUtesPu6WMTNrIAd3M7MGcnA3M2sgB/cJ4bXNzSaLg/sIqSoA561t3nrd9euTh38JmI03Z8uMiCo3nM5b27z1uktLR8o0YdNrs0nltWVGxNxc9nT+MtY477YOTKdlBMqug5mVw+u5j5kqN5zutrZ5kfN702uz8ePgPiKq3Fyi29T/Iuf3+i9m48fBfURUufZKt3Vgsq7bSx2chWM2oiJiKI+3ve1tYUfbsSNidjZCSv7csaP+605PJ4/2OmTVbceOiKmpiKRHP3lMTdVXb7NJBCxGgRjrAVXLlbXhxtQUnHDC0dk1yzwAa1ad0gZUJd0i6WlJj3Y4foGkA5IeSh8f7qfCVq9eulM6pVJmBXbwAKzZKCiS534bcBOwvUuZ+yLil0qpkVWu15z6XoO1B2DNhi+35R4R9wL7a6iL1SRvUlO7TsF6etobcJiNqrKyZc6X9LCkz0t6Y0nntAxlZKf0mlPfKZPnhht6243JzGpUZNQVmAMe7XDsRGBN+vxi4P90Oc9GYBFYnJmZqXpQeezkZcuUlZ0yO3v0OZYfs7P9183M6kGZ2TKS5oC7IuJNBcruAuYj4gfdyjlb5midMlJaW8JlLVFQ5FpmNppqW35A0imSlD4/Nz1nhzyK8VPXJJ1O/eCXX37kumUtUdDr5tZmNn6KpEJ+GvgK8DOS9ki6StLVkq5Oi1wKPCrpYeBG4LIo8nVgDOQtlVumbgF6+brr1mUf7yc7ZWEhae2//HLy53Jg94xTs2bwJKYuqlypsei1Wk1Pw49+VF13irtrzEafV4UsQZUrNbbLW+MFYP/+artTek2RNLPR5c06uui01nkVk3SWA/SWLZ1b8DMzSbmqWtF1/jIzs2q55d5FlSs1ZlnuB9+xYziTg6pcdtjM6uXg3sWwskqGdd26f5mZWXUc3HN0yiopW3uWCtRz3VZOkTRrDve5D9HOnUf62KUj+5y2LuQFSZndu5Puka1bqw22Vfbpm1l9HNyHpD3tsD0j9dAh2Lz56NTHvNUbzcyWuVtmSLLSDtstLTk10cz64+A+JIOkFzo10czyOLgPYJCp+nnphVNTyYzUfj5rZubg3qdu684UCfpZaYfJ8mtHslRuuMGpiWbWHw+o9qnTVP2ig6CtM1LzMmHqzJYxs2bwwmF9WrHi2AyXbqanYc0aB2kzG4wXDqtYr/3eS0v1LB1sZgYTGNzLWq+801T9ToOg7YqkNHptdTPr10T1ubdPHBpkUlCnPnM4dk30TrqlNJZZVzObPBPV517X5hvLywosB/3nn0+6ZXq5bp0bhZjZ+HCfe4Yq1ivP6jppX2ysn5RGr61uZoOYqOBe9nrlRfdY7We1Ra+tbmaDmKjgXvZ65b1sS9fr0sFeW93MBjFRwb3s9cqr7Drx2upmNoiJypaBY7NcllvZ/QTNqvdY9drqZtavxrXc83LDi/aTF+GuEzMbVY0K7kUCd6d+8iuv7H2ykLtOzGxUNSq4Fxng7NQf/tJLvbXkl78hXHFF8vr22+vb69TMLE+jgnuRAc4i/eF5SwOU2bVjZlaFRgX3IrnhW7fCqlX55+qW8dJLCqSZ2TCMfXBvHUB9/nlYvfro4+0DnAsLcOKJ+eft1sL37FEzG3VjHdzbu0eWlpI/p6e7D3Du39/9vHkZL3nfELyao5kN21gH96zukcOHk00xus0E7dYqL5Lx0i0F0v3xZjYKxjq499s90ik479hRLOOlWwqk++PNbBSMdXDvd3GtMvLTO60V4/54MxsFYx3cs1rgkAys5nWD9LqQV1FezdHMRsFYB/flFnj71nZLS4P1cw8yIOolCcxsFIxVcO+0McaaNceW7befe9ABUS9JYGajYGy22WvfUxSSFvG2bckSAFm3ISXdLr3w9nZmNsoat81etyyUMvu5PSBqZk0wNsG9W9Ats5/bA6Jm1gRjE9y7Bd0y+7k9IGpmTTA2wT0v6JaV2ugBUTNrgtzgLukWSU9LerTDcUm6UdLjkh6R9Nbyq9l/0O0nrbGqHHgzs7oU2UP1NuAmYHuH478InJE+3g7cnP5Zul73FG3PsFlOa1w+l5lZU+W23CPiXqDbOoqXANsjcT9wsqRTy6rgILzOi5lNqjL63E8Dnmx5vSd97xiSNkpalLS4b9++Ei7dndMazWxSlRHclfFe5syoiNgWEfMRMb9hw4YSLt2d0xrNbFKVEdz3AKe3vH4t8L0SzjswpzWa2aQqI7jfCbwvzZo5DzgQEXtLOO/AnNZoZpMqN1tG0qeBC4D1kvYAvwOsAoiITwB3AxcDjwOHgPdXVdl+9JphY2bWBLnBPSLem3M8gGtLq5GZmQ1sbGaomplZcQ7uZmYN5OBuZtZADu5mZg00tJ2YJO0DMvY8yrUe+EHJ1RkHvu/J4vueLL3c92xE5M4CHVpw75ekxSJbTDWN73uy+L4nSxX37W4ZM7MGcnA3M2ugcQzu24ZdgSHxfU8W3/dkKf2+x67P3czM8o1jy93MzHI4uJuZNdBIBndJF0n6drrp9m9lHD9e0h3p8QckzdVfy/IVuO8PSfpWuhH5/5A0O4x6ViHv3lvKXSopJI19ulyRe5b0nvRn/k1Jf1Z3HatS4N/6jKQvSfp6+u/94mHUs0ySbpH0tKRHOxyXpBvTv5NHJL11oAtGxEg9gOOA/wu8DlgNPAyc3VZmE/CJ9PllwB3DrndN9/1Pgan0+TVNuO+i956WWwvcC9wPzA+73jX8vM8Avg68On39mmHXu8Z73wZckz4/G9g17HqXcN/vBN4KPNrh+MXA50l2tzsPeGCQ641iy/1c4PGI+E5EvAD8Ockm3K0uAT6VPv8M8C5JWdv9jZPc+46IL0XE8pbf95PsetUERX7mAL8H/BHw4zorV5Ei9/wvgY9HxDMAEfF0zXWsSpF7D+DE9PlJjMjuboOIiHuB/V2KXAJsj8T9wMmSTu33eqMY3ItsuP1KmYh4ETgATNdSu+oU3mg8dRXJb/kmyL13SW8BTo+Iu+qsWIWK/LzPBM6U9DeS7pd0UW21q1aRe/8IcHm6QdDdwL+qp2pD1WsM6Cp3s44hKLLhduFNucdI4XuSdDkwD/xspTWqT9d7l7QC+Bjwq3VVqAZFft4rSbpmLiD5lnafpDdFxLMV161qRe79vcBtEfEfJJ0P3J7e+8vVV29oSo1ro9hyL7Lh9itlJK0k+drW7evOOCi00bikC4EtwLsj4ic11a1qefe+FngTcI+kXST9kXeO+aBq0X/nn4uIwxHxXeDbJMF+3BW596uA/wIQEV8BXkWyuFaTFYoBRY1icP9b4AxJ/0DSapIB0zvbytwJXJk+vxT4YqQjEmMs977Trok/IQnsTel/hZx7j4gDEbE+IuYiYo5kvOHdEbE4nOqWosi/878iGURH0nqSbprv1FrLahS5993AuwAknUUS3PfVWsv63Qm8L82aOQ84EBF7+z7bsEeQu4wa/2+SEfUt6XvXk/yHhuQH/Rckm3J/FXjdsOtc033/NfB94KH0ceew61zXvbeVvYcxz5Yp+PMW8FHgW8A3gMuGXeca7/1s4G9IMmkeAn5+2HUu4Z4/DewFDpO00q8Crgaubvl5fzz9O/nGoP/GvfyAmVkDjWK3jJmZDcjB3cysgRzczcwayMHdzKyBHNzNzBrIwd3MrIEc3M3MGuj/AxPfBHiAyVEnAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.plot(x_train, y_train, 'bo')\n", "plt.title('Generated data train')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "plt.plot(x_val, y_val, 'ro')\n", "plt.title('Generated data validation')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Gradient Descent\n", "\n", "Step 1: Compute the Loss\n", "\n", "$$MSE=\\sum_{i=0}^N (y_i - \\hat{y_i})^2$$\n", "$$MSE=\\sum_{i=0}^N (y_i - a - bx_i)^2$$\n", "\n", "Step 2: Compute the Gradients\n", "\n", "$$\\frac{\\partial{MSE}} {\\partial{a}} = \\frac{\\partial{MSE}} {\\partial{\\hat{y_i}}} \\frac{\\partial{\\hat{y_i}}} {\\partial{a}} = ...$$\n", "\n", "$$\\frac{\\partial{MSE}} {\\partial{b}} = \\frac{\\partial{MSE}} {\\partial{\\hat{y_i}}} \\frac{\\partial{\\hat{y_i}}} {\\partial{b}} = ...$$\n", "\n", "Step 3: Update the Parameters\n", "\n", "$$ a = a - \\eta \\frac{\\partial{MSE}} {\\partial{a}}$$\n", "$$ b = b - \\eta \\frac{\\partial{MSE}} {\\partial{b}}$$" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[0.49671415] [-0.1382643]\n", "[1.02354094] [1.96896411]\n" ] } ], "source": [ "# Initializes parameters \"a\" and \"b\" randomly\n", "np.random.seed(42)\n", "a = np.random.randn(1)\n", "b = np.random.randn(1)\n", "\n", "print(a, b)\n", "\n", "# Sets learning rate\n", "lr = 1e-1\n", "# Defines number of epochs\n", "n_epochs = 1000\n", "\n", "for epoch in range(n_epochs):\n", " # Computes our model's predicted output\n", " yhat = a + b * x_train\n", " \n", " # How wrong is our model? That's the error! \n", " error = (y_train - yhat)\n", " # It is a regression, so it computes mean squared error (MSE)\n", " loss = (error ** 2).mean()\n", " \n", " # Computes gradients for both \"a\" and \"b\" parameters\n", " a_grad = -2 * error.mean()\n", " b_grad = -2 * (x_train * error).mean()\n", " \n", " # Updates parameters using gradients and the learning rate\n", " a = a - lr * a_grad\n", " b = b - lr * b_grad\n", " \n", "print(a, b)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Sanity Check: do we get the same results as our gradient descent?\n", "from sklearn.linear_model import LinearRegression\n", "linr = LinearRegression()\n", "linr.fit(x_train, y_train)\n", "print(linr.intercept_, linr.coef_[0])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "PyTorch it!\n", "\n", "everything is a tensor\n", "\n", "Loading Data, Devices and CUDA" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import torch\n", "import torch.optim as optim\n", "import torch.nn as nn\n", "from torchviz import make_dot\n", "\n", "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", "\n", "# Our data was in Numpy arrays, but we need to transform them into PyTorch's Tensors\n", "# and then we send them to the chosen device\n", "x_train_tensor = torch.from_numpy(x_train).float().to(device)\n", "y_train_tensor = torch.from_numpy(y_train).float().to(device)\n", "\n", "# Here we can see the difference - notice that .type() is more useful\n", "# since it also tells us WHERE the tensor is (device)\n", "print(type(x_train), type(x_train_tensor), x_train_tensor.type())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Creating Parameters and computation of gradients" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# FIRST\n", "# Initializes parameters \"a\" and \"b\" randomly, ALMOST as we did in Numpy\n", "# since we want to apply gradient descent on these parameters, we need\n", "# to set REQUIRES_GRAD = TRUE\n", "a = torch.randn(1, requires_grad=True, dtype=torch.float)\n", "b = torch.randn(1, requires_grad=True, dtype=torch.float)\n", "print(a, b)\n", "\n", "# SECOND\n", "# But what if we want to run it on a GPU? We could just send them to device, right?\n", "a = torch.randn(1, requires_grad=True, dtype=torch.float).to(device)\n", "b = torch.randn(1, requires_grad=True, dtype=torch.float).to(device)\n", "print(a, b)\n", "# Sorry, but NO! The to(device) \"shadows\" the gradient...\n", "\n", "# THIRD\n", "# We can either create regular tensors and send them to the device (as we did with our data)\n", "a = torch.randn(1, dtype=torch.float).to(device)\n", "b = torch.randn(1, dtype=torch.float).to(device)\n", "# and THEN set them as requiring gradients...\n", "\n", "#In PyTorch, every method that ends with an underscore (_) makes changes in-place, meaning, they will modify the underlying variable.\n", "a.requires_grad_()\n", "b.requires_grad_()\n", "print(a, b)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# We can specify the device at the moment of creation - RECOMMENDED!\n", "torch.manual_seed(42)\n", "a = torch.randn(1, requires_grad=True, dtype=torch.float, device=device)\n", "b = torch.randn(1, requires_grad=True, dtype=torch.float, device=device)\n", "print(a, b)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Autograd\n", "\n", "Autograd is PyTorch’s automatic differentiation package. Thanks to it, we don’t need to worry about partial derivatives, chain rule or anything like it." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "lr = 1e-1\n", "n_epochs = 1000\n", "\n", "torch.manual_seed(42)\n", "a = torch.randn(1, requires_grad=True, dtype=torch.float, device=device)\n", "b = torch.randn(1, requires_grad=True, dtype=torch.float, device=device)\n", "\n", "for epoch in range(n_epochs):\n", " yhat = a + b * x_train_tensor\n", " error = y_train_tensor - yhat\n", " loss = (error ** 2).mean()\n", "\n", " # No more manual computation of gradients! \n", " # a_grad = -2 * error.mean()\n", " # b_grad = -2 * (x_tensor * error).mean()\n", " \n", " # We just tell PyTorch to work its way BACKWARDS from the specified loss!\n", " loss.backward()\n", " # Let's check the computed gradients...\n", " print(a.grad)\n", " print(b.grad)\n", " \n", " # What about UPDATING the parameters? Not so fast...\n", " \n", " # FIRST ATTEMPT\n", " # AttributeError: 'NoneType' object has no attribute 'zero_'\n", " # lost requires_grad\n", " # a = a - lr * a.grad\n", " # b = b - lr * b.grad\n", " # print(a)\n", "\n", " # SECOND ATTEMPT\n", " # RuntimeError: a leaf Variable that requires grad has been used in an in-place operation.\n", " # dynamic computation graph \n", " # a -= lr * a.grad\n", " # b -= lr * b.grad \n", " \n", " # THIRD ATTEMPT\n", " # We need to use NO_GRAD to keep the update out of the gradient computation\n", " # Why is that? It boils down to the DYNAMIC GRAPH that PyTorch uses...\n", " with torch.no_grad():\n", " a -= lr * a.grad\n", " b -= lr * b.grad\n", " \n", " # PyTorch is \"clingy\" to its computed gradients, we need to tell it to let it go...\n", " a.grad.zero_()\n", " b.grad.zero_()\n", " \n", "print(a, b)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Dynamic Computation Graph" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "torch.manual_seed(42)\n", "a = torch.randn(1, requires_grad=True, dtype=torch.float, device=device)\n", "b = torch.randn(1, requires_grad=True, dtype=torch.float, device=device)\n", "\n", "yhat = a + b * x_train_tensor\n", "error = y_train_tensor - yhat\n", "loss = (error ** 2).mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "make_dot(yhat)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Optimizer" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "torch.manual_seed(42)\n", "a = torch.randn(1, requires_grad=True, dtype=torch.float, device=device)\n", "b = torch.randn(1, requires_grad=True, dtype=torch.float, device=device)\n", "print(a, b)\n", "\n", "lr = 1e-1\n", "n_epochs = 1000\n", "\n", "# Defines a SGD optimizer to update the parameters\n", "optimizer = optim.SGD([a, b], lr=lr)\n", "\n", "for epoch in range(n_epochs):\n", " yhat = a + b * x_train_tensor\n", " error = y_train_tensor - yhat\n", " loss = (error ** 2).mean()\n", "\n", " loss.backward() \n", " \n", " # No more manual update!\n", " # with torch.no_grad():\n", " # a -= lr * a.grad\n", " # b -= lr * b.grad\n", " optimizer.step()\n", " \n", " # No more telling PyTorch to let gradients go!\n", " # a.grad.zero_()\n", " # b.grad.zero_()\n", " optimizer.zero_grad()\n", " \n", "print(a, b)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Loss" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "torch.manual_seed(42)\n", "a = torch.randn(1, requires_grad=True, dtype=torch.float, device=device)\n", "b = torch.randn(1, requires_grad=True, dtype=torch.float, device=device)\n", "print(a, b)\n", "\n", "lr = 1e-1\n", "n_epochs = 1000\n", "\n", "# Defines a MSE loss function\n", "loss_fn = nn.MSELoss(reduction='mean')\n", "\n", "optimizer = optim.SGD([a, b], lr=lr)\n", "\n", "for epoch in range(n_epochs):\n", " yhat = a + b * x_train_tensor\n", " \n", " # No more manual loss!\n", " # error = y_tensor - yhat\n", " # loss = (error ** 2).mean()\n", " loss = loss_fn(y_train_tensor, yhat)\n", "\n", " loss.backward() \n", " optimizer.step()\n", " optimizer.zero_grad()\n", " \n", "print(a, b)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class ManualLinearRegression(nn.Module):\n", " def __init__(self):\n", " super().__init__()\n", " # To make \"a\" and \"b\" real parameters of the model, we need to wrap them with nn.Parameter\n", " self.a = nn.Parameter(torch.randn(1, requires_grad=True, dtype=torch.float))\n", " self.b = nn.Parameter(torch.randn(1, requires_grad=True, dtype=torch.float))\n", " \n", " def forward(self, x):\n", " # Computes the outputs / predictions\n", " return self.a + self.b * x" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "torch.manual_seed(42)\n", "\n", "# Now we can create a model and send it at once to the device where the data is!\n", "model = ManualLinearRegression().to(device)\n", "# We can also inspect its parameters using its state_dict\n", "print(model.state_dict())\n", "\n", "lr = 1e-1\n", "n_epochs = 1000\n", "\n", "loss_fn = nn.MSELoss(reduction='mean')\n", "optimizer = optim.SGD(model.parameters(), lr=lr)\n", "\n", "for epoch in range(n_epochs):\n", " # What is this?!?\n", " model.train()\n", "\n", " # No more manual prediction!\n", " # yhat = a + b * x_tensor\n", " yhat = model(x_train_tensor)\n", " \n", " loss = loss_fn(y_train_tensor, yhat)\n", " loss.backward() \n", " optimizer.step()\n", " optimizer.zero_grad()\n", " \n", "print(model.state_dict())" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def make_train_step(model, loss_fn, optimizer):\n", " # Builds function that performs a step in the train loop\n", " def train_step(x, y):\n", " # Sets model to TRAIN mode\n", " model.train()\n", " # Makes predictions\n", " yhat = model(x)\n", " # Computes loss\n", " loss = loss_fn(y, yhat)\n", " # Computes gradients\n", " loss.backward()\n", " # Updates parameters and zeroes gradients\n", " optimizer.step()\n", " optimizer.zero_grad()\n", " # Returns the loss\n", " return loss.item()\n", " \n", " # Returns the function that will be called inside the train loop\n", " return train_step\n", "\n", "# Creates the train_step function for our model, loss function and optimizer\n", "train_step = make_train_step(model, loss_fn, optimizer)\n", "losses = []\n", "\n", "# For each epoch...\n", "for epoch in range(n_epochs):\n", " # Performs one train step and returns the corresponding loss\n", " loss = train_step(x_train_tensor, y_train_tensor)\n", " losses.append(loss)\n", " \n", "# Checks model's parameters\n", "print(model.state_dict())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Dataset" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from torch.utils.data import Dataset, TensorDataset\n", "\n", "class CustomDataset(Dataset):\n", " def __init__(self, x_tensor, y_tensor):\n", " self.x = x_tensor\n", " self.y = y_tensor\n", " \n", " def __getitem__(self, index):\n", " return (self.x[index], self.y[index])\n", "\n", " def __len__(self):\n", " return len(self.x)\n", "\n", "# Wait, is this a CPU tensor now? Why? Where is .to(device)?\n", "x_train_tensor = torch.from_numpy(x_train).float()\n", "y_train_tensor = torch.from_numpy(y_train).float()\n", "\n", "train_data = CustomDataset(x_train_tensor, y_train_tensor)\n", "print(train_data[0])\n", "\n", "train_data = TensorDataset(x_train_tensor, y_train_tensor)\n", "print(train_data[0])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Data Loader and mini-batch gradient descent" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from torch.utils.data import DataLoader\n", "\n", "train_loader = DataLoader(dataset=train_data, batch_size=16, shuffle=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "losses = []\n", "train_step = make_train_step(model, loss_fn, optimizer)\n", "\n", "for epoch in range(n_epochs):\n", " for x_batch, y_batch in train_loader:\n", " # the dataset \"lives\" in the CPU, so do our mini-batches\n", " # therefore, we need to send those mini-batches to the\n", " # device where the model \"lives\"\n", " x_batch = x_batch.to(device)\n", " y_batch = y_batch.to(device)\n", " \n", " loss = train_step(x_batch, y_batch)\n", " losses.append(loss)\n", " \n", "print(model.state_dict())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Evaluation" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from torch.utils.data.dataset import random_split\n", "\n", "x_tensor = torch.from_numpy(x).float()\n", "y_tensor = torch.from_numpy(y).float()\n", "\n", "dataset = TensorDataset(x_tensor, y_tensor)\n", "\n", "train_dataset, val_dataset = random_split(dataset, [80, 20])\n", "\n", "train_loader = DataLoader(dataset=train_dataset, batch_size=16)\n", "val_loader = DataLoader(dataset=val_dataset, batch_size=20)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "losses = []\n", "val_losses = []\n", "train_step = make_train_step(model, loss_fn, optimizer)\n", "\n", "for epoch in range(n_epochs):\n", " model.train()\n", " for x_batch, y_batch in train_loader:\n", " x_batch = x_batch.to(device)\n", " y_batch = y_batch.to(device)\n", "\n", " loss = train_step(x_batch, y_batch)\n", " losses.append(loss)\n", " \n", " model.eval()\n", " with torch.no_grad():\n", " for x_val, y_val in val_loader:\n", " x_val = x_val.to(device)\n", " y_val = y_val.to(device)\n", " \n", " yhat = model(x_val)\n", " val_loss = loss_fn(y_val, yhat)\n", " val_losses.append(val_loss.item())\n", "\n", "print(model.state_dict())" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.0" } }, "nbformat": 4, "nbformat_minor": 2 }