unsupervised.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import mglearn\n",
    "import matplotlib.pyplot as plt\n",
    "mglearn.plots.plot_scaling()\n",
    "plt.suptitle(\"scaleing_data\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "from sklearn.model_selection import train_test_split\n",
    "cancer = load_breast_cancer()\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, random_state=1)\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "scaler.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.set_printoptions(suppress=True, precision=2)\n",
    "X_train_scaled = scaler.transform(X_train)\n",
    "print(\"transformed shape: %s\" % (X_train_scaled.shape,))\n",
    "print(\"per-feature minimum before scaling:\\n %s\" % X_train.min(axis=0))\n",
    "print(\"per-feature maximum before scaling:\\n %s\" % X_train.max(axis=0))\n",
    "print(\"per-feature minimum after scaling:\\n %s\" % X_train_scaled.min(axis=0))\n",
    "print(\"per-feature maximum after scaling:\\n %s\" % X_train_scaled.max(axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_test_scaled = scaler.transform(X_test)\n",
    "print(\"per-feature minimum after scaling: %s\" % X_test_scaled.min(axis=0))\n",
    "print(\"per-feature maximum after scaling: %s\" % X_test_scaled.max(axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_blobs\n",
    "X, _ = make_blobs(n_samples=50, centers=5, random_state=4, cluster_std=2)\n",
    "X_train, X_test = train_test_split(X, random_state=5, test_size=.1)\n",
    "\n",
    "fig, axes = plt.subplots(1,3, figsize=(13,4))\n",
    "axes[0].scatter(X_train[:,0], X_train[:,1], c='b', label=\"training set\", s=60)\n",
    "axes[0].scatter(X_test[:,0], X_test[:,1], c='r', label=\"test set\", s=60, marker='^')\n",
    "axes[0].legend(loc='upper left')\n",
    "axes[0].set_title(\"original data\")\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "scaler.fit(X_train)\n",
    "X_train_scaled = scaler.transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "\n",
    "axes[1].scatter(X_train_scaled[:, 0], X_train_scaled[:, 1],\n",
    "                c='b', label=\"training set\", s=60)\n",
    "axes[1].scatter(X_test_scaled[:, 0], X_test_scaled[:, 1], marker='^',\n",
    "                c='r', label=\"test set\", s=60)\n",
    "axes[1].set_title(\"scaled data\")\n",
    "\n",
    "test_scaler = MinMaxScaler()\n",
    "test_scaler.fit(X_test)\n",
    "X_test_scaled_badly = test_scaler.transform(X_test)\n",
    "\n",
    "axes[2].scatter(X_train_scaled[:, 0], X_train_scaled[:, 1],\n",
    "                c='b', label=\"training set\", s=60)\n",
    "axes[2].scatter(X_test_scaled_badly[:, 0], X_test_scaled_badly[:, 1], marker='^',\n",
    "                c='r', label=\"test set\", s=60)\n",
    "axes[2].set_title(\"improperly scaled data\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, random_state=0)\n",
    "svm = SVC(C=100)\n",
    "svm.fit(X_train, y_train)\n",
    "print(svm.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "scaler = MinMaxScaler()\n",
    "scaler.fit(X_train)\n",
    "X_train_scaled = scaler.transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "svm.fit(X_train_scaled, y_train)\n",
    "svm.score(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X_train)\n",
    "X_train_scaled = scaler.transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "svm.fit(X_train_scaled, y_train)\n",
    "svm.score(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_pca_illustration()\n",
    "plt.suptitle(\"pca illustration\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(15, 2, figsize=(10, 20))\n",
    "malignant = cancer.data[cancer.target == 0]\n",
    "benign = cancer.data[cancer.target == 1]\n",
    "ax = axes.ravel()\n",
    "\n",
    "for i in range(30):\n",
    "    _, bins = np.histogram(cancer.data[:, i], bins=50)\n",
    "    ax[i].hist(malignant[:, i], bins=bins, color='b', alpha=.5)\n",
    "    ax[i].hist(benign[:, i], bins=bins, color='r', alpha=.5)\n",
    "    ax[i].set_title(cancer.feature_names[i])\n",
    "    ax[i].set_yticks(())\n",
    "fig.tight_layout()\n",
    "plt.suptitle(\"cancer_histograms\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "cancer = load_breast_cancer()\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(cancer.data)\n",
    "X_scaled = scaler.transform(cancer.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "pca = PCA(n_components=2)\n",
    "pca.fit(X_scaled)\n",
    "\n",
    "X_pca = pca.transform(X_scaled)\n",
    "print(\"Original shape: %s\" % str(X_scaled.shape))\n",
    "print(\"Reduced shape: %s\" % str(X_pca.shape))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(8, 8))\n",
    "plt.scatter(X_pca[:, 0], X_pca[:, 1], c=cancer.target, cmap=mglearn.tools.cm, s=60)\n",
    "plt.gca().set_aspect(\"equal\")\n",
    "plt.xlabel(\"First principal component\")\n",
    "plt.ylabel(\"Second principal component\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pca.components_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(pca.components_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.matshow(pca.components_, cmap='viridis')\n",
    "plt.yticks([0, 1], [\"first component\", \"second component\"])\n",
    "plt.colorbar()\n",
    "plt.xticks(range(len(cancer.feature_names)),\n",
    "           cancer.feature_names, rotation=60, ha='left');\n",
    "plt.suptitle(\"pca_components_cancer\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_lfw_people\n",
    "people = fetch_lfw_people(min_faces_per_person=20, resize=0.7)\n",
    "image_shape = people.images[0].shape\n",
    "\n",
    "fix, axes = plt.subplots(2, 5, figsize=(15, 8), subplot_kw={'xticks': (), 'yticks': ()})\n",
    "for target, image, ax in zip(people.target, people.images, axes.ravel()):\n",
    "    ax.imshow(image)\n",
    "    ax.set_title(people.target_names[target])\n",
    "plt.suptitle(\"some_faces\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(people.images.shape)\n",
    "print(len(people.target_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "counts = np.bincount(people.target)\n",
    "for i, (count, name) in enumerate(zip(counts, people.target_names)):\n",
    "    print(\"{0:25} {1:3}\".format(name, count), end='   ')\n",
    "    if (i + 1) % 3 == 0:\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mask = np.zeros(people.target.shape, dtype=np.bool)\n",
    "for target in np.unique(people.target):\n",
    "    mask[np.where(people.target == target)[0][:50]] = 1\n",
    "\n",
    "X_people = people.data[mask]\n",
    "y_people = people.target[mask]\n",
    "\n",
    "X_people = X_people / 255.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X_people, y_people, stratify=y_people, random_state=0)\n",
    "knn = KNeighborsClassifier(n_neighbors=1)\n",
    "knn.fit(X_train, y_train)\n",
    "knn.score(X_test, y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_pca_whitening()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pca = PCA(n_components=100, whiten=True).fit(X_train)\n",
    "X_train_pca = pca.transform(X_train)\n",
    "X_test_pca = pca.transform(X_test)\n",
    "\n",
    "print(X_train_pca.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors=1)\n",
    "knn.fit(X_train_pca, y_train)\n",
    "knn.score(X_test_pca, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pca.components_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fix, axes = plt.subplots(3, 5, figsize=(15, 12),\n",
    "                         subplot_kw={'xticks': (), 'yticks': ()})\n",
    "fig.suptitle(\"pca_face_components\")\n",
    "for i, (component, ax) in enumerate(zip(pca.components_, axes.ravel())):\n",
    "    ax.imshow(component.reshape(image_shape),\n",
    "              cmap='viridis')\n",
    "    ax.set_title(\"%d. component\" % (i + 1))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from matplotlib.offsetbox import OffsetImage, AnnotationBbox\n",
    "\n",
    "image_shape = people.images[0].shape\n",
    "plt.figure(figsize=(20, 3))\n",
    "ax = plt.gca()\n",
    "\n",
    "imagebox = OffsetImage(people.images[0], zoom=7, cmap=\"gray\")\n",
    "ab = AnnotationBbox(imagebox, (.05, 0.4), pad=0.0, xycoords='data')\n",
    "ax.add_artist(ab)\n",
    "\n",
    "for i in range(4):\n",
    "    imagebox = OffsetImage(pca.components_[i].reshape(image_shape), zoom=7, cmap=\"viridis\")\n",
    "\n",
    "    ab = AnnotationBbox(imagebox, (.3 + .2 * i, 0.4),\n",
    "                        pad=0.0,\n",
    "                        xycoords='data'\n",
    "                        )\n",
    "    ax.add_artist(ab)\n",
    "    if i == 0:\n",
    "        plt.text(.18, .25, 'x_%d *' % i, fontdict={'fontsize': 50})\n",
    "    else:\n",
    "        plt.text(.15 + .2 * i, .25, '+ x_%d *' % i, fontdict={'fontsize': 50})\n",
    "\n",
    "plt.text(.95, .25, '+ ...', fontdict={'fontsize': 50})\n",
    "\n",
    "plt.rc('text', usetex=False)\n",
    "plt.text(.13, .3, r'\\approx', fontdict={'fontsize': 50})\n",
    "plt.axis(\"off\")\n",
    "plt.title(\"decomposition\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_pca_faces(X_train, X_test, image_shape)\n",
    "plt.suptitle(\"pca_reconstructions\");\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.scatter(X_train_pca[:, 0], X_train_pca[:, 1], c=y_train, cmap='Paired', s=60)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_nmf_illustration()\n",
    "plt.suptitle(\"nmf_illustration\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_nmf_faces(X_train, X_test, image_shape)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import NMF\n",
    "nmf = NMF(n_components=15, random_state=0)\n",
    "nmf.fit(X_train)\n",
    "X_train_nmf = nmf.transform(X_train)\n",
    "X_test_nmf = nmf.transform(X_test)\n",
    "\n",
    "fix, axes = plt.subplots(3, 5, figsize=(15, 12),\n",
    "                         subplot_kw={'xticks': (), 'yticks': ()})\n",
    "for i, (component, ax) in enumerate(zip(nmf.components_, axes.ravel())):\n",
    "    ax.imshow(component.reshape(image_shape))\n",
    "    ax.set_title(\"%d. component\" % i)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "compn = 3\n",
    "inds = np.argsort(X_train_nmf[:, compn])[::-1]\n",
    "fig, axes = plt.subplots(2, 5, figsize=(15, 8),\n",
    "                         subplot_kw={'xticks': (), 'yticks': ()})\n",
    "fig.suptitle(\"Large component 3\")\n",
    "for i, (ind, ax) in enumerate(zip(inds, axes.ravel())):\n",
    "    ax.imshow(X_train[ind].reshape(image_shape))\n",
    "\n",
    "compn = 7\n",
    "inds = np.argsort(X_train_nmf[:, compn])[::-1]\n",
    "fig.suptitle(\"Large component 7\")\n",
    "fig, axes = plt.subplots(2, 5, figsize=(15, 8),\n",
    "                         subplot_kw={'xticks': (), 'yticks': ()})\n",
    "for i, (ind, ax) in enumerate(zip(inds, axes.ravel())):\n",
    "    ax.imshow(X_train[ind].reshape(image_shape))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_digits\n",
    "digits = load_digits()\n",
    "\n",
    "fig, axes = plt.subplots(2, 5, figsize=(10, 5),\n",
    "                         subplot_kw={'xticks':(), 'yticks': ()})\n",
    "for ax, img in zip(axes.ravel(), digits.images):\n",
    "    ax.imshow(img)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pca = PCA(n_components=2)\n",
    "pca.fit(digits.data)\n",
    "digits_pca = pca.transform(digits.data)\n",
    "colors = [\"#476A2A\", \"#7851B8\", \"#BD3430\", \"#4A2D4E\", \"#875525\",\n",
    "          \"#A83683\", \"#4E655E\", \"#853541\", \"#3A3120\",\"#535D8E\"]\n",
    "plt.figure(figsize=(10, 10))\n",
    "plt.xlim(digits_pca[:, 0].min(), digits_pca[:, 0].max())\n",
    "plt.ylim(digits_pca[:, 1].min(), digits_pca[:, 1].max())\n",
    "for i in range(len(digits.data)):\n",
    "    plt.text(digits_pca[i, 0], digits_pca[i, 1], str(digits.target[i]),\n",
    "             color = colors[digits.target[i]],\n",
    "             fontdict={'weight': 'bold', 'size': 9})\n",
    "plt.xlabel(\"first principal component\")\n",
    "plt.ylabel(\"second principal component\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "tsne = TSNE(random_state=42)\n",
    "digits_tsne = tsne.fit_transform(digits.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 10))\n",
    "plt.xlim(digits_tsne[:, 0].min(), digits_tsne[:, 0].max() + 1)\n",
    "plt.ylim(digits_tsne[:, 1].min(), digits_tsne[:, 1].max() + 1)\n",
    "for i in range(len(digits.data)):\n",
    "    plt.text(digits_tsne[i, 0], digits_tsne[i, 1], str(digits.target[i]),\n",
    "             color = colors[digits.target[i]],\n",
    "             fontdict={'weight': 'bold', 'size': 9})\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_kmeans_algorithm()\n",
    "plt.suptitle(\"kmeans_algorithm\");\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_blobs\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "X, y = make_blobs(random_state=1)\n",
    "kmeans = KMeans(n_clusters=3)\n",
    "kmeans.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(kmeans.labels_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(kmeans.predict(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.scatter(X[:, 0], X[:, 1], c=kmeans.labels_, cmap=mglearn.cm3, s=60)\n",
    "plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1],\n",
    "            marker='^', s=100, linewidth=2, c=[0, 1, 2], cmap=mglearn.cm3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 2)\n",
    "\n",
    "kmeans = KMeans(n_clusters=2)\n",
    "kmeans.fit(X)\n",
    "assignments = kmeans.labels_\n",
    "\n",
    "axes[0].scatter(X[:, 0], X[:, 1], c=assignments, cmap=mglearn.cm2, s=60)\n",
    "\n",
    "kmeans = KMeans(n_clusters=5)\n",
    "kmeans.fit(X)\n",
    "assignments = kmeans.labels_\n",
    "\n",
    "axes[1].scatter(X[:, 0], X[:, 1], c=assignments, cmap='jet', s=60);\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X, y = make_blobs(random_state=0)\n",
    "plt.scatter(X[:, 0], X[:, 1]);\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X, y = make_blobs(random_state=170, n_samples=600)\n",
    "rng = np.random.RandomState(74)\n",
    "\n",
    "transformation = rng.normal(size=(2, 2))\n",
    "X = np.dot(X, transformation)\n",
    "\n",
    "kmeans = KMeans(n_clusters=3)\n",
    "kmeans.fit(X)\n",
    "y_pred = kmeans.predict(X)\n",
    "\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y_pred, cmap=mglearn.cm3)\n",
    "plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1],\n",
    "            marker='^', c=['b', 'r', 'g'], s=60, linewidth=2);\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_moons\n",
    "X, y = make_moons(n_samples=200, noise=0.05, random_state=0)\n",
    "\n",
    "kmeans = KMeans(n_clusters=2)\n",
    "kmeans.fit(X)\n",
    "y_pred = kmeans.predict(X)\n",
    "\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y_pred, cmap=mglearn.cm3, s=60)\n",
    "plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1],\n",
    "            marker='^', c=['b', 'g'], s=60, linewidth=2);\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_people, y_people, stratify=y_people, random_state=0)\n",
    "nmf = NMF(n_components=100)\n",
    "nmf.fit(X_train)\n",
    "pca = PCA(n_components=100)\n",
    "pca.fit(X_train)\n",
    "kmeans = KMeans(n_clusters=100)\n",
    "kmeans.fit(X_train)\n",
    "\n",
    "X_reconstructed_pca = pca.inverse_transform(pca.transform(X_test))\n",
    "X_reconstructed_kmeans = kmeans.cluster_centers_[kmeans.predict(X_test)]\n",
    "X_reconstructed_nmf = np.dot(nmf.transform(X_test), nmf.components_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(3, 5, figsize=(8, 8)) #, subplot_kw={'xticks': (), 'yticks': ()}\n",
    "fig.suptitle(\"Extracted Components\")\n",
    "for ax, comp_kmeans, comp_pca, comp_nmf in zip(axes.T, kmeans.cluster_centers_, pca.components_, nmf.components_):\n",
    "    ax[0].imshow(comp_kmeans.reshape(image_shape))\n",
    "    ax[1].imshow(comp_pca.reshape(image_shape), cmap='viridis')\n",
    "    ax[2].imshow(comp_nmf.reshape(image_shape))\n",
    "\n",
    "axes[0, 0].set_ylabel(\"kmeans\")\n",
    "axes[1, 0].set_ylabel(\"pca\")\n",
    "axes[2, 0].set_ylabel(\"nmf\")\n",
    "\n",
    "fig, axes = plt.subplots(4, 5, subplot_kw={'xticks': (), 'yticks': ()}, figsize=(8, 8))\n",
    "fig.suptitle(\"Reconstructions\")\n",
    "for ax, orig, rec_kmeans, rec_pca, rec_nmf in zip(axes.T, X_test, X_reconstructed_kmeans,\n",
    "                                         X_reconstructed_pca, X_reconstructed_nmf):\n",
    "    ax[0].imshow(orig.reshape(image_shape))\n",
    "    ax[1].imshow(rec_kmeans.reshape(image_shape))\n",
    "    ax[2].imshow(rec_pca.reshape(image_shape))\n",
    "    ax[3].imshow(rec_nmf.reshape(image_shape))\n",
    "\n",
    "axes[0, 0].set_ylabel(\"original\")\n",
    "axes[1, 0].set_ylabel(\"kmeans\")\n",
    "axes[2, 0].set_ylabel(\"pca\")\n",
    "axes[3, 0].set_ylabel(\"nmf\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X, y = make_moons(n_samples=200, noise=0.05, random_state=0)\n",
    "\n",
    "kmeans = KMeans(n_clusters=10)\n",
    "kmeans.fit(X)\n",
    "y_pred = kmeans.predict(X)\n",
    "\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y_pred, s=60, cmap='Paired')\n",
    "plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1],\n",
    "            marker='^', c=range(kmeans.n_clusters), s=60, linewidth=2, cmap='Paired')\n",
    "print(y_pred)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "distance_features = kmeans.transform(X)\n",
    "print(distance_features.shape)\n",
    "print(distance_features)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_agglomerative_algorithm()\n",
    "plt.suptitle(\"agglomerative_algorithm\");\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import AgglomerativeClustering\n",
    "X, y = make_blobs(random_state=1)\n",
    "\n",
    "agg = AgglomerativeClustering(n_clusters=3)\n",
    "assignment = agg.fit_predict(X)\n",
    "\n",
    "plt.scatter(X[:, 0], X[:, 1], c=assignment, cmap=mglearn.cm3, s=60)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.cluster.hierarchy import dendrogram, ward\n",
    "\n",
    "X, y = make_blobs(random_state=0, n_samples=12)\n",
    "linkage_array = ward(X)\n",
    "dendrogram(linkage_array);\n",
    "\n",
    "ax = plt.gca()\n",
    "bounds = ax.get_xbound()\n",
    "ax.plot(bounds, [7.25, 7.25], '--', c='k')\n",
    "ax.plot(bounds, [4, 4], '--', c='k')\n",
    "\n",
    "ax.text(bounds[1], 7.25, ' two clusters', verticalalignment='center', fontdict={'size': 15})\n",
    "ax.text(bounds[1], 4, ' three clusters', verticalalignment='center', fontdict={'size': 15})\n",
    "plt.title(\"dendrogram\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import DBSCAN\n",
    "X, y = make_blobs(random_state=0, n_samples=12)\n",
    "\n",
    "dbscan = DBSCAN()\n",
    "clusters = dbscan.fit_predict(X)\n",
    "clusters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(3, 4, figsize=(11, 8), subplot_kw={'xticks': (), 'yticks': ()})\n",
    "colors = np.array(['r', 'g', 'b', 'w'])\n",
    "\n",
    "for i, min_samples in enumerate([2, 3, 5]):\n",
    "    for j, eps in enumerate([1, 1.5, 2, 3]):\n",
    "        dbscan = DBSCAN(min_samples=min_samples, eps=eps)\n",
    "        clusters = dbscan.fit_predict(X)\n",
    "        print(\"min_samples: %d eps: %f  cluster: %s\" % (min_samples, eps, clusters))\n",
    "        sizes = 60 * np.ones(X.shape[0])\n",
    "        sizes[dbscan.core_sample_indices_] *= 4\n",
    "        axes[i, j].scatter(X[:, 0], X[:, 1], c=colors[clusters], s=sizes)\n",
    "        axes[i, j].set_title(\"min_samples: %d eps: %.1f\" % (min_samples, eps))\n",
    "fig.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X, y = make_moons(n_samples=200, noise=0.05, random_state=0)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X)\n",
    "X_scaled = scaler.transform(X)\n",
    "\n",
    "dbscan = DBSCAN()\n",
    "clusters = dbscan.fit_predict(X_scaled)\n",
    "plt.scatter(X_scaled[:, 0], X_scaled[:, 1], c=clusters, cmap=mglearn.cm2, s=60)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.cluster import adjusted_rand_score\n",
    "X, y = make_moons(n_samples=200, noise=0.05, random_state=0)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X)\n",
    "X_scaled = scaler.transform(X)\n",
    "\n",
    "fig, axes = plt.subplots(1, 4, figsize=(15, 3), subplot_kw={'xticks': (), 'yticks': ()})\n",
    "\n",
    "algorithms = [KMeans(n_clusters=2), AgglomerativeClustering(n_clusters=2), DBSCAN()]\n",
    "\n",
    "random_state = np.random.RandomState(seed=0)\n",
    "random_clusters = random_state.randint(low=0, high=2, size=len(X))\n",
    "\n",
    "axes[0].scatter(X_scaled[:, 0], X_scaled[:, 1], c=random_clusters, cmap=mglearn.cm3, s=60)\n",
    "axes[0].set_title(\"Random assignment - ARI: %.2f\" % adjusted_rand_score(y, random_clusters))\n",
    "\n",
    "for ax, algorithm in zip(axes[1:], algorithms):\n",
    "    clusters = algorithm.fit_predict(X_scaled)\n",
    "    ax.scatter(X_scaled[:, 0], X_scaled[:, 1], c=clusters, cmap=mglearn.cm3, s=60)\n",
    "    ax.set_title(\"%s - ARI: %.2f\" % (algorithm.__class__.__name__, adjusted_rand_score(y, clusters)))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "clusters1 = [0, 0, 1, 1, 0]\n",
    "clusters2 = [1, 1, 0, 0, 1]\n",
    "print(\"Accuracy: %.2f\" % accuracy_score(clusters1, clusters2))\n",
    "print(\"ARI: %.2f\" % adjusted_rand_score(clusters1, clusters2))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.cluster import silhouette_score\n",
    "\n",
    "X, y = make_moons(n_samples=200, noise=0.05, random_state=0)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X)\n",
    "X_scaled = scaler.transform(X)\n",
    "\n",
    "fig, axes = plt.subplots(1, 4, figsize=(15, 3), subplot_kw={'xticks': (), 'yticks': ()})\n",
    "\n",
    "random_state = np.random.RandomState(seed=0)\n",
    "random_clusters = random_state.randint(low=0, high=2, size=len(X))\n",
    "\n",
    "axes[0].scatter(X_scaled[:, 0], X_scaled[:, 1], c=random_clusters, cmap=mglearn.cm3, s=60)\n",
    "axes[0].set_title(\"Random assignment: %.2f\" % silhouette_score(X_scaled, random_clusters))\n",
    "\n",
    "algorithms = [KMeans(n_clusters=2), AgglomerativeClustering(n_clusters=2), DBSCAN()]\n",
    "\n",
    "for ax, algorithm in zip(axes[1:], algorithms):\n",
    "    clusters = algorithm.fit_predict(X_scaled)\n",
    "    ax.scatter(X_scaled[:, 0], X_scaled[:, 1], c=clusters, cmap=mglearn.cm3, s=60)\n",
    "    ax.set_title(\"%s : %.2f\" % (algorithm.__class__.__name__, silhouette_score(X_scaled, clusters)))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "pca = PCA(n_components=100, whiten=True)\n",
    "pca.fit_transform(X_people)\n",
    "X_pca = pca.transform(X_people)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dbscan = DBSCAN()\n",
    "labels = dbscan.fit_predict(X_pca)\n",
    "np.unique(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dbscan = DBSCAN(min_samples=3, eps=15)\n",
    "labels = dbscan.fit_predict(X_pca)\n",
    "np.unique(labels)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.bincount(labels + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "noise = X_people[labels==-1]\n",
    "\n",
    "fig, axes = plt.subplots(3, 9, subplot_kw={'xticks': (), 'yticks': ()}, figsize=(12, 4))\n",
    "for image, ax in zip(noise, axes.ravel()):\n",
    "    ax.imshow(image.reshape(image_shape), vmin=0, vmax=1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for eps in [1, 3, 5, 7, 9, 11, 13]:\n",
    "    print(\"\\neps=%d\" % eps)\n",
    "    dbscan = DBSCAN(eps=eps, min_samples=3)\n",
    "    labels = dbscan.fit_predict(X_pca)\n",
    "    print(\"Number of clusters: %s\" % np.unique(labels))\n",
    "    print(\"Clusters: %s\" % np.bincount(labels + 1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labels = dbscan.fit_predict(X_pca)\n",
    "\n",
    "for cluster in range(max(labels)):\n",
    "    mask = labels == cluster\n",
    "    n_images =  np.sum(mask)\n",
    "    fig, axes = plt.subplots(1, n_images, subplot_kw={'xticks': (), 'yticks': ()}, figsize=(n_images * 1.5, 4))\n",
    "    for image, label, ax in zip(X_people[mask], y_people[mask], axes):\n",
    "\n",
    "        ax.imshow(image.reshape(image_shape), vmin=0, vmax=1)\n",
    "        ax.set_title(people.target_names[label].split()[-1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n_clusters = 10\n",
    "km = KMeans(n_clusters=n_clusters, random_state=0)\n",
    "labels_km = km.fit_predict(X_pca)\n",
    "print(\"cluster sizes k-Means: %s\" % np.bincount(labels_km))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(2, 5, subplot_kw={'xticks': (), 'yticks': ()}, figsize=(12, 4))\n",
    "for center, ax in zip(km.cluster_centers_, axes.ravel()):\n",
    "    ax.imshow(pca.inverse_transform(center).reshape(image_shape), vmin=0, vmax=1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n_clusters = 10\n",
    "for cluster in range(n_clusters):\n",
    "    center = km.cluster_centers_[cluster]\n",
    "    mask = km.labels_ == cluster\n",
    "    dists = np.sum((X_pca - center) ** 2, axis=1)\n",
    "    dists[~mask] = np.inf\n",
    "    inds = np.argsort(dists)[:5]\n",
    "    dists[~mask] = -np.inf\n",
    "    inds = np.r_[inds, np.argsort(dists)[-5:]]\n",
    "    fig, axes = plt.subplots(1, 11, subplot_kw={'xticks': (), 'yticks': ()}, figsize=(10, 8))\n",
    "    axes[0].imshow(pca.inverse_transform(center).reshape(image_shape), vmin=0, vmax=1)\n",
    "    for image, label, asdf, ax in zip(X_people[inds], y_people[inds], labels_km[inds], axes[1:]):\n",
    "        ax.imshow(image.reshape(image_shape), vmin=0, vmax=1)\n",
    "        ax.set_title(\"%s\" % (people.target_names[label].split()[-1]), fontdict={'fontsize': 9})\n",
    "print(\"kmeans_face_clusters\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "agglomerative = AgglomerativeClustering(n_clusters=10)\n",
    "labels_agg = agglomerative.fit_predict(X_pca)\n",
    "print(\"cluster sizes agglomerative clustering: %s\" % np.bincount(labels_agg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "adjusted_rand_score(labels_agg, labels_km)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.cluster.hierarchy import dendrogram, ward\n",
    "linkage_array = ward(X_pca)\n",
    "plt.figure(figsize=(20, 5))\n",
    "dendrogram(linkage_array, p=7, truncate_mode='level', no_labels=True);\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}