{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Superconductivty Data Set: Predict the critical temperature based on 81 material features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"https://www.physi.uni-heidelberg.de/~reygers/lectures/2021/ml/data/train_critical_temp.csv\"\n",
    "df = pd.read_csv(filename, engine='python')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>number_of_elements</th>\n",
       "      <th>mean_atomic_mass</th>\n",
       "      <th>wtd_mean_atomic_mass</th>\n",
       "      <th>gmean_atomic_mass</th>\n",
       "      <th>wtd_gmean_atomic_mass</th>\n",
       "      <th>entropy_atomic_mass</th>\n",
       "      <th>wtd_entropy_atomic_mass</th>\n",
       "      <th>range_atomic_mass</th>\n",
       "      <th>wtd_range_atomic_mass</th>\n",
       "      <th>std_atomic_mass</th>\n",
       "      <th>...</th>\n",
       "      <th>wtd_mean_Valence</th>\n",
       "      <th>gmean_Valence</th>\n",
       "      <th>wtd_gmean_Valence</th>\n",
       "      <th>entropy_Valence</th>\n",
       "      <th>wtd_entropy_Valence</th>\n",
       "      <th>range_Valence</th>\n",
       "      <th>wtd_range_Valence</th>\n",
       "      <th>std_Valence</th>\n",
       "      <th>wtd_std_Valence</th>\n",
       "      <th>critical_temp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>88.944468</td>\n",
       "      <td>57.862692</td>\n",
       "      <td>66.361592</td>\n",
       "      <td>36.116612</td>\n",
       "      <td>1.181795</td>\n",
       "      <td>1.062396</td>\n",
       "      <td>122.90607</td>\n",
       "      <td>31.794921</td>\n",
       "      <td>51.968828</td>\n",
       "      <td>...</td>\n",
       "      <td>2.257143</td>\n",
       "      <td>2.213364</td>\n",
       "      <td>2.219783</td>\n",
       "      <td>1.368922</td>\n",
       "      <td>1.066221</td>\n",
       "      <td>1</td>\n",
       "      <td>1.085714</td>\n",
       "      <td>0.433013</td>\n",
       "      <td>0.437059</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>92.729214</td>\n",
       "      <td>58.518416</td>\n",
       "      <td>73.132787</td>\n",
       "      <td>36.396602</td>\n",
       "      <td>1.449309</td>\n",
       "      <td>1.057755</td>\n",
       "      <td>122.90607</td>\n",
       "      <td>36.161939</td>\n",
       "      <td>47.094633</td>\n",
       "      <td>...</td>\n",
       "      <td>2.257143</td>\n",
       "      <td>1.888175</td>\n",
       "      <td>2.210679</td>\n",
       "      <td>1.557113</td>\n",
       "      <td>1.047221</td>\n",
       "      <td>2</td>\n",
       "      <td>1.128571</td>\n",
       "      <td>0.632456</td>\n",
       "      <td>0.468606</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>88.944468</td>\n",
       "      <td>57.885242</td>\n",
       "      <td>66.361592</td>\n",
       "      <td>36.122509</td>\n",
       "      <td>1.181795</td>\n",
       "      <td>0.975980</td>\n",
       "      <td>122.90607</td>\n",
       "      <td>35.741099</td>\n",
       "      <td>51.968828</td>\n",
       "      <td>...</td>\n",
       "      <td>2.271429</td>\n",
       "      <td>2.213364</td>\n",
       "      <td>2.232679</td>\n",
       "      <td>1.368922</td>\n",
       "      <td>1.029175</td>\n",
       "      <td>1</td>\n",
       "      <td>1.114286</td>\n",
       "      <td>0.433013</td>\n",
       "      <td>0.444697</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>88.944468</td>\n",
       "      <td>57.873967</td>\n",
       "      <td>66.361592</td>\n",
       "      <td>36.119560</td>\n",
       "      <td>1.181795</td>\n",
       "      <td>1.022291</td>\n",
       "      <td>122.90607</td>\n",
       "      <td>33.768010</td>\n",
       "      <td>51.968828</td>\n",
       "      <td>...</td>\n",
       "      <td>2.264286</td>\n",
       "      <td>2.213364</td>\n",
       "      <td>2.226222</td>\n",
       "      <td>1.368922</td>\n",
       "      <td>1.048834</td>\n",
       "      <td>1</td>\n",
       "      <td>1.100000</td>\n",
       "      <td>0.433013</td>\n",
       "      <td>0.440952</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>88.944468</td>\n",
       "      <td>57.840143</td>\n",
       "      <td>66.361592</td>\n",
       "      <td>36.110716</td>\n",
       "      <td>1.181795</td>\n",
       "      <td>1.129224</td>\n",
       "      <td>122.90607</td>\n",
       "      <td>27.848743</td>\n",
       "      <td>51.968828</td>\n",
       "      <td>...</td>\n",
       "      <td>2.242857</td>\n",
       "      <td>2.213364</td>\n",
       "      <td>2.206963</td>\n",
       "      <td>1.368922</td>\n",
       "      <td>1.096052</td>\n",
       "      <td>1</td>\n",
       "      <td>1.057143</td>\n",
       "      <td>0.433013</td>\n",
       "      <td>0.428809</td>\n",
       "      <td>23.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 82 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   number_of_elements  mean_atomic_mass  wtd_mean_atomic_mass  \\\n",
       "0                   4         88.944468             57.862692   \n",
       "1                   5         92.729214             58.518416   \n",
       "2                   4         88.944468             57.885242   \n",
       "3                   4         88.944468             57.873967   \n",
       "4                   4         88.944468             57.840143   \n",
       "\n",
       "   gmean_atomic_mass  wtd_gmean_atomic_mass  entropy_atomic_mass  \\\n",
       "0          66.361592              36.116612             1.181795   \n",
       "1          73.132787              36.396602             1.449309   \n",
       "2          66.361592              36.122509             1.181795   \n",
       "3          66.361592              36.119560             1.181795   \n",
       "4          66.361592              36.110716             1.181795   \n",
       "\n",
       "   wtd_entropy_atomic_mass  range_atomic_mass  wtd_range_atomic_mass  \\\n",
       "0                 1.062396          122.90607              31.794921   \n",
       "1                 1.057755          122.90607              36.161939   \n",
       "2                 0.975980          122.90607              35.741099   \n",
       "3                 1.022291          122.90607              33.768010   \n",
       "4                 1.129224          122.90607              27.848743   \n",
       "\n",
       "   std_atomic_mass  ...  wtd_mean_Valence  gmean_Valence  wtd_gmean_Valence  \\\n",
       "0        51.968828  ...          2.257143       2.213364           2.219783   \n",
       "1        47.094633  ...          2.257143       1.888175           2.210679   \n",
       "2        51.968828  ...          2.271429       2.213364           2.232679   \n",
       "3        51.968828  ...          2.264286       2.213364           2.226222   \n",
       "4        51.968828  ...          2.242857       2.213364           2.206963   \n",
       "\n",
       "   entropy_Valence  wtd_entropy_Valence  range_Valence  wtd_range_Valence  \\\n",
       "0         1.368922             1.066221              1           1.085714   \n",
       "1         1.557113             1.047221              2           1.128571   \n",
       "2         1.368922             1.029175              1           1.114286   \n",
       "3         1.368922             1.048834              1           1.100000   \n",
       "4         1.368922             1.096052              1           1.057143   \n",
       "\n",
       "   std_Valence  wtd_std_Valence  critical_temp  \n",
       "0     0.433013         0.437059           29.0  \n",
       "1     0.632456         0.468606           26.0  \n",
       "2     0.433013         0.444697           19.0  \n",
       "3     0.433013         0.440952           22.0  \n",
       "4     0.433013         0.428809           23.0  \n",
       "\n",
       "[5 rows x 82 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df['critical_temp'].values\n",
    "X = df[[col for col in df.columns if col!=\"critical_temp\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.9422240257263184\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPRegressor\n",
    "import time\n",
    "\n",
    "mlpr = MLPRegressor(hidden_layer_sizes=(50,50), activation='relu', random_state=1, max_iter=5000)\n",
    "\n",
    "start_time = time.time()\n",
    "mlpr.fit(X_train, y_train)\n",
    "run_time = time.time() - start_time\n",
    "\n",
    "print(run_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = mlpr.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 77752 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(y_test, y_pred, s=2)\n",
    "plt.xlabel(\"true critical temperature (K)\", fontsize=14)\n",
    "plt.ylabel(\"predicted critical temperature (K)\", fontsize=14)\n",
    "plt.savefig(\"critical_temperature.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root mean square error 22.29\n"
     ]
    }
   ],
   "source": [
    "rms = np.sqrt(mean_squared_error(y_test, y_pred))\n",
    "print(f\"root mean square error {rms:.2f}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/sklearn/model_selection/_validation.py:378: FitFailedWarning: \n",
      "10 fits failed out of a total of 35.\n",
      "The score on these train-test partitions for these parameters will be set to nan.\n",
      "If these failures are not expected, you can try to debug them by setting error_score='raise'.\n",
      "\n",
      "Below are more details about the failures:\n",
      "--------------------------------------------------------------------------------\n",
      "5 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/homebrew/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 686, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/opt/homebrew/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 762, in fit\n",
      "    return self._fit(X, y, incremental=False)\n",
      "  File \"/opt/homebrew/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 385, in _fit\n",
      "    self._validate_hyperparameters()\n",
      "  File \"/opt/homebrew/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 508, in _validate_hyperparameters\n",
      "    raise ValueError(\"learning rate %s is not supported. \" % self.learning_rate)\n",
      "ValueError: learning rate 0.0001 is not supported. \n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "5 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/homebrew/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 686, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/opt/homebrew/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 762, in fit\n",
      "    return self._fit(X, y, incremental=False)\n",
      "  File \"/opt/homebrew/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 385, in _fit\n",
      "    self._validate_hyperparameters()\n",
      "  File \"/opt/homebrew/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 508, in _validate_hyperparameters\n",
      "    raise ValueError(\"learning rate %s is not supported. \" % self.learning_rate)\n",
      "ValueError: learning rate 1e-05 is not supported. \n",
      "\n",
      "  warnings.warn(some_fits_failed_message, FitFailedWarning)\n",
      "/opt/homebrew/lib/python3.10/site-packages/sklearn/model_selection/_search.py:953: UserWarning: One or more of the test scores are non-finite: [-491.79222643 -338.81111017 -359.406494   -512.73262966 -450.1530479\n",
      "           nan           nan]\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=5, estimator=MLPRegressor(max_iter=5000, random_state=1),\n",
       "             param_grid=[{&#x27;hidden_layer_sizes&#x27;: [(5, 5), (20, 20), (50, 50)]},\n",
       "                         {&#x27;activation&#x27;: [&#x27;relu&#x27;, &#x27;logistic&#x27;]},\n",
       "                         {&#x27;learning_rate&#x27;: [0.0001, 1e-05]}],\n",
       "             scoring=&#x27;neg_mean_squared_error&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=5, estimator=MLPRegressor(max_iter=5000, random_state=1),\n",
       "             param_grid=[{&#x27;hidden_layer_sizes&#x27;: [(5, 5), (20, 20), (50, 50)]},\n",
       "                         {&#x27;activation&#x27;: [&#x27;relu&#x27;, &#x27;logistic&#x27;]},\n",
       "                         {&#x27;learning_rate&#x27;: [0.0001, 1e-05]}],\n",
       "             scoring=&#x27;neg_mean_squared_error&#x27;)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: MLPRegressor</label><div class=\"sk-toggleable__content\"><pre>MLPRegressor(max_iter=5000, random_state=1)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MLPRegressor</label><div class=\"sk-toggleable__content\"><pre>MLPRegressor(max_iter=5000, random_state=1)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=5, estimator=MLPRegressor(max_iter=5000, random_state=1),\n",
       "             param_grid=[{'hidden_layer_sizes': [(5, 5), (20, 20), (50, 50)]},\n",
       "                         {'activation': ['relu', 'logistic']},\n",
       "                         {'learning_rate': [0.0001, 1e-05]}],\n",
       "             scoring='neg_mean_squared_error')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid = [\n",
    "    {'hidden_layer_sizes': [(5,5), (20,20), (50,50)]},\n",
    "    {'activation': ['relu', 'logistic']},\n",
    "    {'learning_rate': [0.001, 0.01]}\n",
    "]\n",
    "\n",
    "rgr = GridSearchCV(\n",
    "    MLPRegressor(random_state=1, max_iter=5000), param_grid, scoring='neg_mean_squared_error', cv=5)\n",
    "\n",
    "rgr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([9.26026688e+00, 5.38674998e+00, 3.01752081e+00, 2.39675040e+00,\n",
       "        6.93448777e+00, 1.16915703e-03, 1.12295151e-03]),\n",
       " 'std_fit_time': array([1.62185829e+00, 1.45875884e+00, 3.64975502e-01, 8.09020523e-01,\n",
       "        2.24471612e+00, 4.18326171e-05, 2.05156382e-05]),\n",
       " 'mean_score_time': array([0.00132494, 0.00277996, 0.00223546, 0.00275273, 0.0047976 ,\n",
       "        0.        , 0.        ]),\n",
       " 'std_score_time': array([9.74773887e-05, 2.08459164e-03, 2.29868718e-04, 7.39559789e-04,\n",
       "        1.71814052e-03, 0.00000000e+00, 0.00000000e+00]),\n",
       " 'param_hidden_layer_sizes': masked_array(data=[(5, 5), (20, 20), (50, 50), --, --, --, --],\n",
       "              mask=[False, False, False,  True,  True,  True,  True],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_activation': masked_array(data=[--, --, --, 'relu', 'logistic', --, --],\n",
       "              mask=[ True,  True,  True, False, False,  True,  True],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_learning_rate': masked_array(data=[--, --, --, --, --, 0.0001, 1e-05],\n",
       "              mask=[ True,  True,  True,  True,  True, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'hidden_layer_sizes': (5, 5)},\n",
       "  {'hidden_layer_sizes': (20, 20)},\n",
       "  {'hidden_layer_sizes': (50, 50)},\n",
       "  {'activation': 'relu'},\n",
       "  {'activation': 'logistic'},\n",
       "  {'learning_rate': 0.0001},\n",
       "  {'learning_rate': 1e-05}],\n",
       " 'split0_test_score': array([-337.26593326, -374.62590294, -320.12378376, -622.21350077,\n",
       "        -441.23453102,           nan,           nan]),\n",
       " 'split1_test_score': array([-352.98713915, -308.15961551, -320.85973223, -322.17792968,\n",
       "        -406.32690689,           nan,           nan]),\n",
       " 'split2_test_score': array([-1131.85755686,  -413.69988392,  -376.10052041,  -456.73858549,\n",
       "         -458.85602326,            nan,            nan]),\n",
       " 'split3_test_score': array([-325.8868918 , -289.44625711, -424.39045007, -841.68007514,\n",
       "        -534.05373457,           nan,           nan]),\n",
       " 'split4_test_score': array([-310.9636111 , -308.12389139, -355.55798352, -320.85305721,\n",
       "        -410.29404375,           nan,           nan]),\n",
       " 'mean_test_score': array([-491.79222643, -338.81111017, -359.406494  , -512.73262966,\n",
       "        -450.1530479 ,           nan,           nan]),\n",
       " 'std_test_score': array([320.32871333,  47.34793777,  38.84687957, 198.22042277,\n",
       "         46.26894811,          nan,          nan]),\n",
       " 'rank_test_score': array([4, 1, 2, 5, 3, 6, 7], dtype=int32)}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgr.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([22.17638894, 18.40682238, 18.95801925, 22.64360019, 21.2168105 ,\n",
       "               nan,         nan])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sqrt(-rgr.cv_results_['mean_test_score'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
