{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Towards Intuition"
      ],
      "id": "8b1b3dda-9555-4042-be26-1e2069837047"
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [],
      "source": [
        "import ipywidgets as widgets\n",
        "from ipywidgets import interact\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "plt.rcParams.update({'font.size': 16})"
      ],
      "id": "32abd3e4-ab74-45ff-9d9e-05ac72770245"
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [],
      "source": [
        "def f1(x):\n",
        "    \"\"\"some function to minimize\"\"\"\n",
        "    return x * x\n",
        "def g1(x):\n",
        "    \"\"\"gradient of f(x)\"\"\"\n",
        "    return 2 * x"
      ],
      "id": "6460b379-e341-426a-a9c0-e3b9493bac9b"
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [],
      "source": [
        "def gradient_descent_step(grad, init, stepsize):\n",
        "    return init - stepsize * grad"
      ],
      "id": "bc4ed73d-e9f1-43d3-b319-90f151f21ca8"
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "outputId": "b4a29d99-9a29-495a-936c-0216dc988695",
        "quarto-private-1": {
          "key": "colab",
          "value": {
            "base_uri": "https://localhost:8080/",
            "height": 1000,
            "referenced_widgets": [
              "27afa4c58e0e48019235f3524171cfa0",
              "f87a8b466fdb45f1a0836e470d140461",
              "2bec79dc0c484a5d8957b866514e2ce2",
              "5eb3cff9d1274457ad1594fec319da58",
              "0932a1604a8f4c13a143bc9f3af3af28",
              "73491d4089364acc876622cdb7b4bb25",
              "22e31bd0a698423db1b04a01a70a6d9b"
            ]
          }
        }
      },
      "outputs": [
        {
          "output_type": "display_data",
          "metadata": {},
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d42bc0d2cf99442a91350941b0a849f5",
              "version_major": 2,
              "version_minor": 0
            }
          }
        }
      ],
      "source": [
        "@interact(\n",
        "    S = widgets.IntSlider(value=1, min=0, max=100, step=1, description = \"steps\")\n",
        ")\n",
        "def plot_gradient_descent(S):\n",
        "    stepsize = 5 # range from 0 to 2\n",
        "\n",
        "    fig, axs = plt.subplots(1, 2, figsize = (12, 5))\n",
        "    x = np.linspace(-10, 10, 301)\n",
        "    fx = f1(x)\n",
        "    axs[0].plot(x, fx)\n",
        "    xstar = 8\n",
        "    xstars = np.zeros(S)\n",
        "    ystars = np.zeros(S)\n",
        "\n",
        "    for s in range(S):\n",
        "        # store xs and ys for pretty plots\n",
        "        xstars[s] = xstar\n",
        "        ystars[s] = f1(xstar)\n",
        "\n",
        "        # evaluate gradient\n",
        "        grad = g1(xstar)\n",
        "\n",
        "        # minimization step\n",
        "        xstar = gradient_descent_step(grad, xstar, stepsize)\n",
        "\n",
        "    axs[0].plot(xstars, f1(xstars), marker = \"o\", linestyle = \"-\", c = \"orange\", markersize = 10)\n",
        "    axs[0].set_title(f\"After {S - 1} steps, x_star = {xstar:.2f}\")\n",
        "\n",
        "    axs[1].plot(np.arange(S), ystars, color = \"orange\", markersize = 10, marker = \"o\")\n",
        "    axs[1].set_xlabel(\"step\")\n",
        "    axs[1].set_ylabel(\"loss\")"
      ],
      "id": "e602f854-a570-45be-9841-9cc9dd4059e4"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Optimization is Hard"
      ],
      "id": "e54403af-9b3e-45e8-93a4-1f03f17fbbb7"
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [],
      "source": [
        "def f2(x):\n",
        "    return x ** 4 - 5 * x ** 2 - 3 * x\n",
        "def g2(x):\n",
        "    return 4 * x ** 3 - 10 * x - 3"
      ],
      "id": "d06a63cf-bdb8-4bf0-8d98-ce6854122cf3"
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "outputId": "3b90b301-8a7b-4f61-c619-283a2ed6883f",
        "quarto-private-1": {
          "key": "colab",
          "value": {
            "base_uri": "https://localhost:8080/",
            "height": 537,
            "referenced_widgets": [
              "5cbf522e2dfd4bd09771dd427b00cd7a",
              "526fad1c363b40e580938f4bdd47bdc4",
              "6664ce47441a4ac381fb2344a0fcd744",
              "b054b7f862374b8198303ce3bd42663a",
              "b9f2936f1a5340a5bc90a5312f088578",
              "1a1d610d85774f6c8bef9838a3d9b98c",
              "fc5717520873404a835a2aa0db9c5929"
            ]
          }
        }
      },
      "outputs": [
        {
          "output_type": "display_data",
          "metadata": {},
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1690784b8cb646a4b621c257630150d7",
              "version_major": 2,
              "version_minor": 0
            }
          }
        }
      ],
      "source": [
        "@interact(\n",
        "    S = widgets.IntSlider(value=1, min=0, max=100, step=1, description = \"steps\")\n",
        ")\n",
        "def plot_gradient_descent(S):\n",
        "    stepsize = 0.07 # change stepsize\n",
        "\n",
        "    fig, axs = plt.subplots(1, 2, figsize = (12, 5))\n",
        "    x = np.linspace(-3, 3, 301)\n",
        "    fx = f2(x)\n",
        "    axs[0].plot(x, fx)\n",
        "    xstar = -2.5 # change initial position, e.g. starting point\n",
        "    xstars = np.zeros(S)\n",
        "    ystars = np.zeros(S)\n",
        "\n",
        "    for s in range(S):\n",
        "        # store xs and ys for pretty plots\n",
        "        xstars[s] = xstar\n",
        "        ystars[s] = f2(xstar)\n",
        "\n",
        "        # evaluate gradient\n",
        "        grad = g2(xstar)\n",
        "\n",
        "        # minimization step\n",
        "        xstar = gradient_descent_step(grad, xstar, stepsize)\n",
        "\n",
        "    axs[0].plot(xstars, f2(xstars), marker = \"o\", linestyle = \"-\", c = \"orange\", markersize = 10)\n",
        "    axs[0].set_title(f\"After {S - 1} steps, x_star = {xstar:.2f}\")\n",
        "\n",
        "    axs[1].plot(np.arange(S), ystars, color = \"orange\", markersize = 10, marker = \"o\")\n",
        "    axs[1].set_xlabel(\"step\")\n",
        "    axs[1].set_ylabel(\"loss\")"
      ],
      "id": "00eae643-95f5-4aaf-aebf-3fc9988e8917"
    }
  ],
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3 (ipykernel)",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "codemirror_mode": {
        "name": "ipython",
        "version": "3"
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.12"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0932a1604a8f4c13a143bc9f3af3af28": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1a1d610d85774f6c8bef9838a3d9b98c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "SliderStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "SliderStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": "",
            "handle_color": null
          }
        },
        "22e31bd0a698423db1b04a01a70a6d9b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27afa4c58e0e48019235f3524171cfa0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "VBoxModel",
          "state": {
            "_dom_classes": [
              "widget-interact"
            ],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f87a8b466fdb45f1a0836e470d140461",
              "IPY_MODEL_2bec79dc0c484a5d8957b866514e2ce2"
            ],
            "layout": "IPY_MODEL_5eb3cff9d1274457ad1594fec319da58"
          }
        },
        "2bec79dc0c484a5d8957b866514e2ce2": {
          "model_module": "@jupyter-widgets/output",
          "model_module_version": "1.0.0",
          "model_name": "OutputModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/output",
            "_model_module_version": "1.0.0",
            "_model_name": "OutputModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/output",
            "_view_module_version": "1.0.0",
            "_view_name": "OutputView",
            "layout": "IPY_MODEL_22e31bd0a698423db1b04a01a70a6d9b",
            "msg_id": "",
            "outputs": []
          }
        },
        "526fad1c363b40e580938f4bdd47bdc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "IntSliderModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "IntSliderModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "IntSliderView",
            "continuous_update": true,
            "description": "steps",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_b9f2936f1a5340a5bc90a5312f088578",
            "max": 100,
            "min": 0,
            "orientation": "horizontal",
            "readout": true,
            "readout_format": "d",
            "step": 1,
            "style": "IPY_MODEL_1a1d610d85774f6c8bef9838a3d9b98c",
            "value": 1
          }
        },
        "5cbf522e2dfd4bd09771dd427b00cd7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "VBoxModel",
          "state": {
            "_dom_classes": [
              "widget-interact"
            ],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_526fad1c363b40e580938f4bdd47bdc4",
              "IPY_MODEL_6664ce47441a4ac381fb2344a0fcd744"
            ],
            "layout": "IPY_MODEL_b054b7f862374b8198303ce3bd42663a"
          }
        },
        "5eb3cff9d1274457ad1594fec319da58": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6664ce47441a4ac381fb2344a0fcd744": {
          "model_module": "@jupyter-widgets/output",
          "model_module_version": "1.0.0",
          "model_name": "OutputModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/output",
            "_model_module_version": "1.0.0",
            "_model_name": "OutputModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/output",
            "_view_module_version": "1.0.0",
            "_view_name": "OutputView",
            "layout": "IPY_MODEL_fc5717520873404a835a2aa0db9c5929",
            "msg_id": "",
            "outputs": []
          }
        },
        "73491d4089364acc876622cdb7b4bb25": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "SliderStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "SliderStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": "",
            "handle_color": null
          }
        },
        "b054b7f862374b8198303ce3bd42663a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b9f2936f1a5340a5bc90a5312f088578": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f87a8b466fdb45f1a0836e470d140461": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "IntSliderModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "IntSliderModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "IntSliderView",
            "continuous_update": true,
            "description": "steps",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_0932a1604a8f4c13a143bc9f3af3af28",
            "max": 100,
            "min": 0,
            "orientation": "horizontal",
            "readout": true,
            "readout_format": "d",
            "step": 1,
            "style": "IPY_MODEL_73491d4089364acc876622cdb7b4bb25",
            "value": 0
          }
        },
        "fc5717520873404a835a2aa0db9c5929": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  }
}