14 useful features in scikit-learn library.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clone Estimator "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.datasets import make_classification\n",
    "\n",
    "# create dataset`\n",
    "X1, y1 = make_classification(n_classes=2, n_features=5, random_state=1)\n",
    "\n",
    "# create estimators \n",
    "logistic_classifier_1 = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='auto', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_classifier_1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='auto', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.base import clone, is_classifier, is_regressor \n",
    "\n",
    "#  duplicae the first classifier with clone function \n",
    "logistic_classifier_2 = clone(logistic_classifier_1)\n",
    "\n",
    "logistic_classifier_2 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Identify model if is a Classifier or Regressior "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# create estimators \n",
    "model_1 = LinearRegression()\n",
    "model_2 = RandomForestClassifier() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check if it classifier \n",
    "from sklearn.base import is_classifier \n",
    "\n",
    "is_classifier(model_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check if it regressor \n",
    "from sklearn.base import is_regressor \n",
    "\n",
    "is_regressor(model_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## make column selector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.6464639 ,  0.        ,  1.        ],\n",
       "       [-0.98787834,  1.        ,  0.        ],\n",
       "       [-0.10976426,  1.        ,  0.        ],\n",
       "       [-0.5488213 ,  0.        ,  1.        ]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
    "from sklearn.compose import make_column_transformer\n",
    "from sklearn.compose import make_column_selector\n",
    "\n",
    "# create a dataframe with different data types\n",
    "data = pd.DataFrame(\n",
    "    {\"gender\": [\"male\", \"female\", \"female\", \"male\"],\n",
    "     \"age\": [23, 5, 11, 8]}\n",
    ")\n",
    "\n",
    "\n",
    "# create a column transformer with make_column_selector\n",
    "\n",
    "ct = make_column_transformer(\n",
    "    (StandardScaler(), make_column_selector(dtype_include=np.number)),  # ages\n",
    "    (OneHotEncoder(), make_column_selector(dtype_include=object)),  # genders\n",
    ") \n",
    "\n",
    "transformed_data = ct.fit_transform(data)\n",
    "\n",
    "transformed_data "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 2000x1000 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# import libraries\n",
    "import matplotlib.pyplot as plt \n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import plot_confusion_matrix\n",
    "from sklearn.tree import DecisionTreeClassifier, plot_tree, export_text \n",
    "from sklearn.datasets import load_iris\n",
    "\n",
    "#load data \n",
    "iris = load_iris()\n",
    "\n",
    "# create our instances\n",
    "model = DecisionTreeClassifier()\n",
    "\n",
    "# train test split\n",
    "X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, random_state = 0)\n",
    "\n",
    "# fit and predict\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# plot the tree\n",
    "plt.figure(figsize = (20, 10))\n",
    "plot_tree(model,feature_names=iris.feature_names, filled = True) \n",
    "plt.show() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fetch dataset from Openml."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\datasets\\_openml.py:372: UserWarning: Multiple active versions of the dataset matching the name bank-marketing exist. Versions may be fundamentally different, returning version 1.\n",
      "  \" {version}.\".format(name=name, version=res[0]['version']))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_openml \n",
    "\n",
    "#fetch by using data name\n",
    "bank_marketing = fetch_openml(name=\"bank-marketing\")\n",
    "\n",
    "# seperate independent variables and target variable \n",
    "x = bank_marketing.data \n",
    "y = bank_marketing.target "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 5.800e+01,  4.000e+00,  1.000e+00,  2.000e+00,  0.000e+00,\n",
       "         2.143e+03,  1.000e+00,  0.000e+00,  2.000e+00,  5.000e+00,\n",
       "         8.000e+00,  2.610e+02,  1.000e+00, -1.000e+00,  0.000e+00,\n",
       "         3.000e+00],\n",
       "       [ 4.400e+01,  9.000e+00,  2.000e+00,  1.000e+00,  0.000e+00,\n",
       "         2.900e+01,  1.000e+00,  0.000e+00,  2.000e+00,  5.000e+00,\n",
       "         8.000e+00,  1.510e+02,  1.000e+00, -1.000e+00,  0.000e+00,\n",
       "         3.000e+00]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['1', '1'], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fetch by using id from this link https://www.openml.org/d/1461\n",
    "bank_marketing = fetch_openml(data_id=1461)\n",
    "\n",
    "# seperate independent variables and target variable \n",
    "x = bank_marketing.data \n",
    "y  = bank_marketing.target "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 5.800e+01,  4.000e+00,  1.000e+00,  2.000e+00,  0.000e+00,\n",
       "         2.143e+03,  1.000e+00,  0.000e+00,  2.000e+00,  5.000e+00,\n",
       "         8.000e+00,  2.610e+02,  1.000e+00, -1.000e+00,  0.000e+00,\n",
       "         3.000e+00]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[:1] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[1] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_classification\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import learning_curve\n",
    "\n",
    "# create dataset`\n",
    "X, y = make_classification(n_classes=2, n_features=10, n_samples=5000, random_state=1)\n",
    "\n",
    "# create estimator\n",
    "KNN_classifier = KNeighborsClassifier() \n",
    "\n",
    "\n",
    "#\n",
    "train_sizes, train_scores, test_scores = learning_curve(\n",
    "estimator = KNN_classifier,\n",
    "X = X, y = y,train_sizes=np.linspace(0.1, 1.0, 5), shuffle=True, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 400, 1300, 2200, 3100, 4000])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the train size to generate learning curve\n",
    "train_sizes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.8575    , 0.8975    , 0.9       , 0.8775    , 0.8475    ],\n",
       "       [0.90153846, 0.88846154, 0.88230769, 0.89153846, 0.88692308],\n",
       "       [0.88863636, 0.89409091, 0.89181818, 0.88818182, 0.88636364],\n",
       "       [0.8983871 , 0.89387097, 0.89032258, 0.88903226, 0.88580645],\n",
       "       [0.893     , 0.894     , 0.88825   , 0.893     , 0.88625   ]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Scores on training sets\n",
    "train_scores "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.828, 0.815, 0.835, 0.839, 0.814],\n",
       "       [0.823, 0.823, 0.843, 0.848, 0.837],\n",
       "       [0.833, 0.821, 0.854, 0.851, 0.843],\n",
       "       [0.837, 0.825, 0.854, 0.857, 0.834],\n",
       "       [0.834, 0.828, 0.849, 0.86 , 0.835]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show validation scores\n",
    "test_scores "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Scores mean:[0.876      0.89015385 0.88981818 0.89148387 0.8909    ]\n",
      "Test Scores mean:[0.8262 0.8348 0.8404 0.8414 0.8412]\n"
     ]
    }
   ],
   "source": [
    "# find the mean of training scores and validation scores \n",
    "train_scores_mean = train_scores.mean(axis = 1)\n",
    "print(\"Training Scores mean:{}\".format(train_scores_mean))\n",
    "test_scores_mean = test_scores.mean(axis = 1)\n",
    "print(\"Test Scores mean:{}\".format(test_scores_mean))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cross Validation and Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets, linear_model\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "#load dataet\n",
    "diabetes = datasets.load_diabetes()\n",
    "\n",
    "X = diabetes.data\n",
    "y = diabetes.target\n",
    "\n",
    "RF_regressor = RandomForestRegressor()\n",
    "\n",
    "# perfrom cross validation and prediction\n",
    "y_pred = cross_val_predict(estimator=RF_regressor, X= X, y=y, cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([229.04,  93.12, 195.18, 171.14,  86.27, 115.27,  75.69, 155.59,\n",
       "       153.06, 168.94])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#show prediction\n",
    "y_pred[:10] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Select From Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.01631211, -0.04448689, -0.01041713, ..., -0.03925967,\n",
       "        -0.02122777, -0.03405436],\n",
       "       [ 0.00188878, -0.04444519, -0.00816801, ..., -0.03918144,\n",
       "        -0.06436135, -0.05463903],\n",
       "       [-0.02699287, -0.04433151, -0.06285579, ..., -0.0756844 ,\n",
       "        -0.05557734, -0.06683906],\n",
       "       ...,\n",
       "       [ 0.03415162,  0.05040128,  0.11077166, ..., -0.00292399,\n",
       "         0.027618  ,  0.07302442],\n",
       "       [ 0.03416799,  0.05030017,  0.12469165, ...,  0.10747183,\n",
       "        -0.00019805,  0.02747969],\n",
       "       [-0.04907612, -0.04462806,  0.16038187, ...,  0.0340123 ,\n",
       "         0.02773604,  0.01114488]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import datasets, linear_model\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "#load dataet\n",
    "diabetes = datasets.load_diabetes()\n",
    "\n",
    "X = diabetes.data\n",
    "y = diabetes.target\n",
    "\n",
    "lg_regressor = LogisticRegression()\n",
    "\n",
    "# identify and select important fatures by using SelectFromModel\n",
    "selector = SelectFromModel(estimator=lg_regressor).fit(X, y)\n",
    "\n",
    "#show estimator coefficient \n",
    "selector.estimator_.coef_\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45211, 16)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12.197550946960686"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show the treshold value\n",
    "selector.threshold_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.05068012,  0.06169621,  0.02187235, -0.04340085, -0.00259226,\n",
       "         0.01990842],\n",
       "       [-0.04464164, -0.05147406, -0.02632783,  0.07441156, -0.03949338,\n",
       "        -0.06832974],\n",
       "       [ 0.05068012,  0.04445121, -0.00567061, -0.03235593, -0.00259226,\n",
       "         0.00286377]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# transform data to selected features \n",
    "transformed = selector.transform(X)\n",
    "transformed[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(442, 6)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformed.shape "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FunctionTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4.48863637, 3.52636052, 2.19722458, 0.        , 1.60943791,\n",
       "        4.46590812, 3.98898405, 3.09104245, 4.20469262, 3.78418963],\n",
       "       [2.48490665, 4.14313473, 4.20469262, 0.69314718, 2.19722458,\n",
       "        3.80666249, 4.39444915, 3.98898405, 3.09104245, 4.29045944]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "\n",
    "X = np.array([[89,34,9, 1,5,87,54,22,67,44], [12, 63,67,2,9,45,81,54,22,73]])\n",
    "\n",
    "#create FunctionTransformer\n",
    "log_transformer = FunctionTransformer(np.log)\n",
    "\n",
    "#transform the data\n",
    "log_transformer.transform(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Determine the target data type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'multiclass'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.utils.multiclass import type_of_target\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "#load dataet\n",
    "diabetes = datasets.load_diabetes()\n",
    "\n",
    "X = diabetes.data\n",
    "y = diabetes.target\n",
    "\n",
    "type_of_target(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## add dummy feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 5., 89., 34.],\n",
       "       [ 5., 12., 63.]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np \n",
    "from sklearn.preprocessing import add_dummy_feature\n",
    "\n",
    "p = np.array([[89,34], [12, 63]])\n",
    "\n",
    "add_dummy_feature(p, value=5)\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Impute Missing Values with Iterative Imputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.experimental import enable_iterative_imputer  \n",
    "from sklearn.impute import IterativeImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create dataset with missing values\n",
    "data = [[61, 22, 43,np.nan,67],\n",
    "        [np.nan, 6, 27, 8, 11],\n",
    "        [83, 51, np.nan, 32, 9],\n",
    "        [74, np.nan, 35, 26, 97],\n",
    "        [np.nan, 4, 13,45, 33]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\impute\\_iterative.py:638: ConvergenceWarning: [IterativeImputer] Early stopping criterion not reached.\n",
      "  \" reached.\", ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[61.        , 22.        , 43.        , 27.74898065, 67.        ],\n",
       "       [50.92363568,  6.        , 27.        ,  8.        , 11.        ],\n",
       "       [83.        , 51.        , 28.62176528, 32.        ,  9.        ],\n",
       "       [74.        , 20.72107515, 35.        , 26.        , 97.        ],\n",
       "       [67.54222006,  4.        , 13.        , 45.        , 33.        ]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Impute missing values using iterative imputer\n",
    "iter_imp = IterativeImputer(random_state= 42)\n",
    "iter_imp.fit_transform(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter Tuning Using Random Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import linear_model, datasets\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from xgboost import XGBClassifier\n",
    "from scipy.stats import randint\n",
    "\n",
    "# Load data\n",
    "iris = datasets.load_iris()\n",
    "X = iris.data\n",
    "y = iris.target\n",
    "\n",
    "# create model\n",
    "classifier = XGBClassifier()\n",
    "\n",
    "# Create Hyperparameter Search Space\n",
    "param_dist = {\n",
    "     # randomly sample numbers from 50 to 400 estimators\n",
    "    \"n_estimators\":  randint(50,400),\n",
    "    \"learning_rate\": [0.01, 0.03, 0.05],\n",
    "    \"subsample\": [0.5, 0.7],\n",
    "    \"max_depth\": [3, 4, 5],\n",
    "    \"min_child_weight\": [1, 2, 3],\n",
    "}\n",
    "\n",
    "# create random search\n",
    "\n",
    "# Create randomized search 5-fold cross validation and 100 iterations\n",
    "clf = RandomizedSearchCV(\n",
    "    estimator=classifier,\n",
    "    param_distributions=param_dist,\n",
    "    random_state=1,\n",
    "    n_iter=100,\n",
    "    cv=5,\n",
    "    verbose=0,\n",
    "    n_jobs=-1,\n",
    ")\n",
    "\n",
    "# Fit randomized search\n",
    "best_model = clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best n_estimator: 259\n",
      "Best learning_rate: 0.03\n",
      "Best subsample: 0.5\n",
      "Best max_depth: 3\n",
      "Best min_child_weight: 1\n"
     ]
    }
   ],
   "source": [
    "# View best hyperparameters\n",
    "print('Best n_estimator:', best_model.best_estimator_.get_params()['n_estimators'])\n",
    "print('Best learning_rate:', best_model.best_estimator_.get_params()['learning_rate'])\n",
    "print('Best subsample:', best_model.best_estimator_.get_params()['subsample'])\n",
    "print('Best max_depth:', best_model.best_estimator_.get_params()['max_depth'])\n",
    "print('Best min_child_weight:', best_model.best_estimator_.get_params()['min_child_weight'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Text files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_files\n",
    "\n",
    "news_reports = load_files(\n",
    "    container_path=\"news_report/\",\n",
    "    description=\"News reports in 2020\",\n",
    "    load_content=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['business', 'healthy', 'international', 'sport']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show target names \n",
    "news_reports.target_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# specify independent variable and target variable \n",
    "X = news_reports.data\n",
    "y  = news_reports.target "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}