00_generate_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "choice-subscription",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np \n",
    "import os, sys\n",
    "from tqdm.notebook import tqdm\n",
    "import itertools as it\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "from rdkit import Chem\n",
    "import sys\n",
    "sys.path.append(\"reinvent/\")\n",
    "import pandas as pd\n",
    "import numpy as np \n",
    "import scipy.stats as sps\n",
    "from tqdm.auto import tqdm, trange\n",
    "from models import dataset\n",
    "from models.model import Model\n",
    "from utils.smiles import standardize_smiles\n",
    "from pandarallel import pandarallel\n",
    "pandarallel.initialize(progress_bar=False, verbose=1)\n",
    "chembl_prior = Model.load_from_file(\"reinvent/data/augmented.prior\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "close-solution",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_in = pd.read_table(\"data/pubchem_reinvent_assay_low_hitrate_human_interactions.tsv.gz\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "painted-legislature",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>aid</th>\n",
       "      <th>sid</th>\n",
       "      <th>gi_number</th>\n",
       "      <th>result</th>\n",
       "      <th>cid</th>\n",
       "      <th>NostereoAromaticSMILES</th>\n",
       "      <th>Entry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>268</td>\n",
       "      <td>1779</td>\n",
       "      <td>842121</td>\n",
       "      <td>224586929</td>\n",
       "      <td>0</td>\n",
       "      <td>6603008</td>\n",
       "      <td>CCOCCCNCC(=O)Nc1ccc(cc1)OC(F)(F)F</td>\n",
       "      <td>Q9Y2R2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>269</td>\n",
       "      <td>1779</td>\n",
       "      <td>842122</td>\n",
       "      <td>224586929</td>\n",
       "      <td>0</td>\n",
       "      <td>6602571</td>\n",
       "      <td>COCCn1c(nnn1)CN2CCC(CC2)Cc3ccccc3</td>\n",
       "      <td>Q9Y2R2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>270</td>\n",
       "      <td>1779</td>\n",
       "      <td>842123</td>\n",
       "      <td>224586929</td>\n",
       "      <td>0</td>\n",
       "      <td>6602616</td>\n",
       "      <td>COCCn1c(nnn1)CN2CCC(CC2)(c3cccc(c3)C(F)(F)F)O</td>\n",
       "      <td>Q9Y2R2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>271</td>\n",
       "      <td>1779</td>\n",
       "      <td>842124</td>\n",
       "      <td>224586929</td>\n",
       "      <td>0</td>\n",
       "      <td>644371</td>\n",
       "      <td>c1cc(ccc1CCNC(=O)CN2CCCCCC2)F</td>\n",
       "      <td>Q9Y2R2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>272</td>\n",
       "      <td>1779</td>\n",
       "      <td>842125</td>\n",
       "      <td>224586929</td>\n",
       "      <td>0</td>\n",
       "      <td>6603132</td>\n",
       "      <td>COc1ccc(cc1)C(=O)C(c2ccccc2)N3CCOCC3</td>\n",
       "      <td>Q9Y2R2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80838259</th>\n",
       "      <td>160788292</td>\n",
       "      <td>743192</td>\n",
       "      <td>162211043</td>\n",
       "      <td>291575128</td>\n",
       "      <td>0</td>\n",
       "      <td>3474778</td>\n",
       "      <td>COc1ccc(cc1)n2cnc3c2ccc(c3)C(=O)N4CCCCC4</td>\n",
       "      <td>P07195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80838260</th>\n",
       "      <td>160788372</td>\n",
       "      <td>743374</td>\n",
       "      <td>172236216</td>\n",
       "      <td>119613085</td>\n",
       "      <td>0</td>\n",
       "      <td>72737738</td>\n",
       "      <td>COc1ccc(cc1)CN2CCOC(C2)CCc3ccccc3</td>\n",
       "      <td>P21918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80838261</th>\n",
       "      <td>160788373</td>\n",
       "      <td>743374</td>\n",
       "      <td>172236217</td>\n",
       "      <td>119613085</td>\n",
       "      <td>0</td>\n",
       "      <td>72737723</td>\n",
       "      <td>c1ccc(cc1)CCC2CN(CCO2)Cc3ccc(cc3)Cl</td>\n",
       "      <td>P21918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80838262</th>\n",
       "      <td>160788376</td>\n",
       "      <td>743378</td>\n",
       "      <td>172236216</td>\n",
       "      <td>49456799</td>\n",
       "      <td>0</td>\n",
       "      <td>72737738</td>\n",
       "      <td>COc1ccc(cc1)CN2CCOC(C2)CCc3ccccc3</td>\n",
       "      <td>Q6FH34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80838263</th>\n",
       "      <td>160788377</td>\n",
       "      <td>743378</td>\n",
       "      <td>172236217</td>\n",
       "      <td>49456799</td>\n",
       "      <td>0</td>\n",
       "      <td>72737723</td>\n",
       "      <td>c1ccc(cc1)CCC2CN(CCO2)Cc3ccc(cc3)Cl</td>\n",
       "      <td>Q6FH34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80838264 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Unnamed: 0     aid        sid  gi_number  result       cid  \\\n",
       "0                268    1779     842121  224586929       0   6603008   \n",
       "1                269    1779     842122  224586929       0   6602571   \n",
       "2                270    1779     842123  224586929       0   6602616   \n",
       "3                271    1779     842124  224586929       0    644371   \n",
       "4                272    1779     842125  224586929       0   6603132   \n",
       "...              ...     ...        ...        ...     ...       ...   \n",
       "80838259   160788292  743192  162211043  291575128       0   3474778   \n",
       "80838260   160788372  743374  172236216  119613085       0  72737738   \n",
       "80838261   160788373  743374  172236217  119613085       0  72737723   \n",
       "80838262   160788376  743378  172236216   49456799       0  72737738   \n",
       "80838263   160788377  743378  172236217   49456799       0  72737723   \n",
       "\n",
       "                                 NostereoAromaticSMILES   Entry  \n",
       "0                     CCOCCCNCC(=O)Nc1ccc(cc1)OC(F)(F)F  Q9Y2R2  \n",
       "1                     COCCn1c(nnn1)CN2CCC(CC2)Cc3ccccc3  Q9Y2R2  \n",
       "2         COCCn1c(nnn1)CN2CCC(CC2)(c3cccc(c3)C(F)(F)F)O  Q9Y2R2  \n",
       "3                         c1cc(ccc1CCNC(=O)CN2CCCCCC2)F  Q9Y2R2  \n",
       "4                  COc1ccc(cc1)C(=O)C(c2ccccc2)N3CCOCC3  Q9Y2R2  \n",
       "...                                                 ...     ...  \n",
       "80838259       COc1ccc(cc1)n2cnc3c2ccc(c3)C(=O)N4CCCCC4  P07195  \n",
       "80838260              COc1ccc(cc1)CN2CCOC(C2)CCc3ccccc3  P21918  \n",
       "80838261            c1ccc(cc1)CCC2CN(CCO2)Cc3ccc(cc3)Cl  P21918  \n",
       "80838262              COc1ccc(cc1)CN2CCOC(C2)CCc3ccccc3  Q6FH34  \n",
       "80838263            c1ccc(cc1)CCC2CN(CCO2)Cc3ccc(cc3)Cl  Q6FH34  \n",
       "\n",
       "[80838264 rows x 8 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "south-notebook",
   "metadata": {},
   "outputs": [],
   "source": [
    "assays_per_target = data_in.pivot_table(index=[\"NostereoAromaticSMILES\", \"Entry\"],\n",
    "                                        columns=\"result\",\n",
    "                                        values=\"aid\",\n",
    "                                        aggfunc=\"nunique\",\n",
    "                                        fill_value=0)\n",
    "assays_per_target.rename(columns={0: \"inactive\", 1: \"active\"},inplace=True)\n",
    "\n",
    "target_count = data_in.pivot_table(index=\"NostereoAromaticSMILES\", \n",
    "                                   columns=\"result\", \n",
    "                                   values=\"Entry\", \n",
    "                                   aggfunc=\"nunique\", \n",
    "                                   fill_value=0)\n",
    "target_count.rename(columns={0: \"inactive\", 1: \"active\"},inplace=True)\n",
    "\n",
    "\n",
    "assay_count = data_in.pivot_table(index=\"NostereoAromaticSMILES\", \n",
    "                                   columns=\"result\", \n",
    "                                   values=\"aid\", \n",
    "                                   aggfunc=\"nunique\", \n",
    "                                   fill_value=0)\n",
    "assay_count.rename(columns={0: \"inactive\", 1: \"active\"},inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "classical-applicant",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_assay_count = target_count.join(assay_count, lsuffix= \" targets\", rsuffix= \" assays\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "waiting-recall",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert(len(assays_per_target.query(\"active >= 1 and inactive >= 1\")) == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "distant-excess",
   "metadata": {},
   "outputs": [],
   "source": [
    "def encodable(smiles, voc=chembl_prior.vocabulary, tokenizer=chembl_prior.tokenizer):\n",
    "    try:\n",
    "        tokens = tokenizer.tokenize(smiles)\n",
    "        encoded = voc.encode(tokens)\n",
    "        if len(encoded) > 2:\n",
    "            return True\n",
    "        else:\n",
    "            return False\n",
    "    except:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "corrected-vanilla",
   "metadata": {},
   "outputs": [],
   "source": [
    "smiles_cid = data_in.groupby(\"NostereoAromaticSMILES\")\n",
    "smiles_cid = pd.DataFrame(smiles_cid[\"cid\"].agg(set))\n",
    "smiles_cid[\"NostereoAromaticSMILES\"] = smiles_cid.index\n",
    "smiles_cid[\"rdkit_smiles\"] = smiles_cid[\"NostereoAromaticSMILES\"].parallel_map(standardize_smiles)\n",
    "smiles_cid[\"encodable\"] = smiles_cid[\"rdkit_smiles\"].parallel_map(encodable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "satellite-passion",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert(len(smiles_cid) == len(smiles_cid.groupby(\"rdkit_smiles\")))\n",
    "assert(len(smiles_cid) == len(smiles_cid.query(\"encodable == True\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hungry-assets",
   "metadata": {},
   "outputs": [],
   "source": [
    "smiles_aid = data_in.groupby(\"NostereoAromaticSMILES\")\n",
    "smiles_aid = pd.DataFrame(smiles_aid[\"aid\"].agg(set))\n",
    "smiles_Entry = data_in.groupby(\"NostereoAromaticSMILES\")\n",
    "smiles_Entry = pd.DataFrame(smiles_Entry[\"Entry\"].agg(set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "composite-supervision",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_count = target_count.join(smiles_cid[[\"rdkit_smiles\", \"cid\", \"encodable\"]]).join(smiles_aid).join(smiles_Entry)\n",
    "assay_count = assay_count.join(smiles_cid[[\"rdkit_smiles\", \"cid\", \"encodable\"]]).join(smiles_aid).join(smiles_Entry)\n",
    "target_assay_count = target_assay_count.join(smiles_cid[[\"rdkit_smiles\", \"cid\", \"encodable\"]]).join(smiles_aid).join(smiles_Entry)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "macro-simon",
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_sets_available():\n",
    "    paths = [\"data/pubchem_assay_compounds_processed_training.smi\", \"data/pubchem_assay_compounds_processed.tsv\", \"data/all_extracted_pubchem_mt_st_nt_compounds.tsv.gz\"]\n",
    "    for path in paths:\n",
    "        if not os.path.exists(path):\n",
    "            return False\n",
    "    return True \n",
    "\n",
    "if not data_sets_available():\n",
    "    st = target_count.query(\"active == 1 & inactive >= 4\").copy()\n",
    "    mt = target_count.query(\"active >= 5\").copy()\n",
    "    nt = target_count.query(\"active == 0 & inactive >= 5\").sample(n=len(st)).copy()\n",
    "\n",
    "    st[\"class\"] = \"singletarget\"\n",
    "    st[\"set\"] = \"test\"\n",
    "    mt[\"class\"] = \"multitarget\"\n",
    "    mt[\"set\"] = \"test\"\n",
    "    nt[\"class\"] = \"notarget\"\n",
    "    nt[\"set\"] = \"test\"\n",
    "\n",
    "    training_mt = mt.sample(n=1000).index\n",
    "    mt.loc[training_mt,\"set\"] = \"training\"\n",
    "    dataset = pd.concat([mt, st, nt])\n",
    "\n",
    "    dataset[\"cids\"] = dataset[\"cid\"].map(lambda x: \",\".join(sorted(map(str,x))))\n",
    "    dataset[\"aids\"] = dataset[\"aid\"].map(lambda x: \",\".join(sorted(map(str,x))))\n",
    "    dataset[\"Targets\"] = dataset[\"Entry\"].map(lambda x: \",\".join(sorted(map(str,x))))\n",
    "\n",
    "    if not os.path.exists(\"data/pubchem_assay_compounds_processed_training.smi\"):\n",
    "        dataset.query(\"set == 'training'\").sample(frac=1.)[[\"rdkit_smiles\"]].to_csv(\"data/pubchem_assay_compounds_processed_training.smi\",index=None,header=None)\n",
    "\n",
    "    if not os.path.exists(\"data/pubchem_assay_compounds_processed.tsv\"):\n",
    "        dataset = dataset.reset_index()[['cids', 'NostereoAromaticSMILES', 'rdkit_smiles', 'class', 'set', 'inactive', 'active', 'encodable']]\n",
    "        dataset.to_csv(\"data/pubchem_assay_compounds_processed.tsv\",index=None, sep=\"\\t\")\n",
    "        \n",
    "    if not os.path.exists(\"data/all_extracted_pubchem_mt_st_nt_compounds.tsv.gz\"):\n",
    "        nt_all = target_count.query(\"active == 0 & inactive >= 5 & index not in @nt\").copy()\n",
    "        nt_all[\"class\"] = \"notarget\"\n",
    "        nt_all[\"set\"] = \"unused\"\n",
    "\n",
    "        dataset_all = pd.concat([mt, st, nt, nt_all])\n",
    "        dataset_all['index_num'] = range(len(dataset_all))\n",
    "        dataset_all = dataset_all[[\"class\", \"set\", \"index_num\"]].join(target_assay_count)\n",
    "        dataset_all = dataset_all[[\"rdkit_smiles\", \"cid\", \"class\", \"set\", \"aid\", \"active assays\", \"inactive assays\", \"Entry\", \"active targets\", \"inactive targets\", \"index_num\"]].copy()\n",
    "\n",
    "        dataset_all[\"cid\"] = dataset_all[\"cid\"].map(lambda x: \",\".join(sorted(map(str,x))))\n",
    "        dataset_all[\"aid\"] = dataset_all[\"aid\"].map(lambda x: \",\".join(sorted(map(str,x))))\n",
    "        dataset_all[\"Entry\"] = dataset_all[\"Entry\"].map(lambda x: \",\".join(sorted(map(str,x))))\n",
    "        \n",
    "        dataset_all = dataset_all.sort_values(\"index_num\")\n",
    "        del dataset_all[\"index_num\"]\n",
    "        dataset_all = dataset_all.rename(mapper={\"cid\": \"cids\", \"aid\": \"validated assay ids\",\"Entry\": \"validated target ids\"}, axis=1).reset_index()\n",
    "        \n",
    "        dataset_all.to_csv(\"data/all_extracted_pubchem_mt_st_nt_compounds.tsv.gz\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "living-decade",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}