combineAndCurate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from os import walk\n",
    "import json\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ahmed/Dropbox/DFCI/14_zoo/scrap-science/pubmed/out/pubmed_result.csv\n",
      "(508, 19)\n",
      "/home/ahmed/Dropbox/DFCI/14_zoo/scrap-science/miccai/out/miccai_result.csv\n",
      "(246, 10)\n",
      "/home/ahmed/Dropbox/DFCI/14_zoo/scrap-science/ieee/out/ieee_result.csv\n",
      "(564, 12)\n",
      "(1318, 21)\n",
      "(1149, 21)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\n/home/ahmed/Dropbox/DFCI/14_zoo/scrap-science/pubmed/out/pubmed_result.csv\\n(509, 19)\\n/home/ahmed/Dropbox/DFCI/14_zoo/scrap-science/miccai/out/miccai_result.csv\\n(246, 10)\\n(755, 19)\\n(676, 19)\\n\\n'"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "folders = [\"pubmed\",\"miccai\",\"ieee\"]\n",
    "dfs = []\n",
    "for folder in folders:\n",
    "    path = '/home/ahmed/Dropbox/DFCI/14_zoo/scrap-science/' + folder + '/out'\n",
    "    for (_, _, filenames) in walk(path):\n",
    "        for filename in filenames:\n",
    "            fullPath = path+'/'+filename\n",
    "            print fullPath\n",
    "            df = pd.read_csv(fullPath, index_col=None)\n",
    "            print df.shape\n",
    "            dfs.append( df )\n",
    "df = pd.concat(dfs)\n",
    "print df.shape\n",
    "# reset index\n",
    "df = df.reset_index(drop=True)\n",
    "# all titles to lower, strip, remove periods and remove double spaces\n",
    "df['Title'] = df.Title.str.lower()\n",
    "df['Title'] = df.Title.str.strip()\n",
    "df['Title'] = df.Title.str.replace('.','')\n",
    "df['Title'] = df.Title.str.replace('  ',' ')\n",
    "# remove duplicates by title\n",
    "df = df.drop_duplicates(subset='Title', keep=\"first\")\n",
    "# remove use==0\n",
    "df = df[df['use']==1]\n",
    "# sort by date\n",
    "#####\n",
    "# reset index\n",
    "df = df.reset_index(drop=True)\n",
    "print df.shape\n",
    "df.head(n=5)\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# returns the dataframe with a clean email columns and delets the old one\n",
    "# email ---> emails\n",
    "# works for miccai,pubmed\n",
    "def clean_email_list(dfa):\n",
    "    new_email_list = []\n",
    "    for i in range(dfa.shape[0]):\n",
    "        # condition1 - null\n",
    "        if pd.isnull ( dfa.iloc[i].email  ):\n",
    "            new_email_list.append([])\n",
    "            print \"cond1\"\n",
    "        # if string\n",
    "        elif type ( dfa.iloc[i].email ) == np.str:\n",
    "            #\n",
    "            string = dfa.iloc[i].email\n",
    "            # if it starts with [ and end with ]\n",
    "            if string[0] == '[' and string[-1] == ']' and string != '[]':\n",
    "                #\n",
    "                string_split = string.split(',')\n",
    "                if len(string_split) == 1:\n",
    "                    new_email_list.append( [ str(string_split[0][3:-2]) ] )\n",
    "                    print \"cond2\"\n",
    "\n",
    "                elif len(string_split) > 1:\n",
    "                    new_email_list.append(string_split)\n",
    "                    print \"cond3\"\n",
    "\n",
    "                else:\n",
    "                    print \"uncaught1: \" ,dfa.iloc[i].email\n",
    "\n",
    "            elif string == '[]':\n",
    "                new_email_list.append([])\n",
    "                print \"cond4\"\n",
    "\n",
    "\n",
    "            elif string[0] != '[' and string[-1] != ']' and string != '[]':\n",
    "                string_split = string.split(',')\n",
    "                new_email_list.append(string_split)\n",
    "                print \"cond5\"\n",
    "\n",
    "        else:\n",
    "            print \"uncaught3: \" ,dfa.iloc[i].email\n",
    "\n",
    "    dfa['emails'] = new_email_list\n",
    "    del dfa['email']\n",
    "\n",
    "    return dfa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond5\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n",
      "cond1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>URL</th>\n",
       "      <th>Description</th>\n",
       "      <th>Details</th>\n",
       "      <th>ShortDetails</th>\n",
       "      <th>Resource</th>\n",
       "      <th>Type</th>\n",
       "      <th>Identifiers</th>\n",
       "      <th>Db</th>\n",
       "      <th>EntrezUID</th>\n",
       "      <th>Properties</th>\n",
       "      <th>abstract</th>\n",
       "      <th>keywords</th>\n",
       "      <th>fullURL</th>\n",
       "      <th>source</th>\n",
       "      <th>year</th>\n",
       "      <th>key</th>\n",
       "      <th>use</th>\n",
       "      <th>emails</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>deepbipolar: identifying genomic mutations for...</td>\n",
       "      <td>/pubmed/28600868</td>\n",
       "      <td>Laksshman S, Bhat RR, Viswanath V, Li X.</td>\n",
       "      <td>Hum Mutat. 2017 Sep;38(9):1217-1224. doi: 10.1...</td>\n",
       "      <td>Hum Mutat.  2017</td>\n",
       "      <td>PubMed</td>\n",
       "      <td>citation</td>\n",
       "      <td>PMID:28600868</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>28600868</td>\n",
       "      <td>create date:2017/06/11 | first author:Laksshman S</td>\n",
       "      <td>&lt;div class='abstr'&gt;&lt;h3&gt;Abstract&lt;/h3&gt;&lt;div class...</td>\n",
       "      <td>bipolar disorder; convolutional neural network...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov//pubmed/28600868</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>2017</td>\n",
       "      <td>c3510786-340b-45d7-9272-eee0c8960971</td>\n",
       "      <td>1</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>precision medicine for heart failure with pres...</td>\n",
       "      <td>/pubmed/28585183</td>\n",
       "      <td>Shah SJ.</td>\n",
       "      <td>J Cardiovasc Transl Res. 2017 Jun;10(3):233-24...</td>\n",
       "      <td>J Cardiovasc Transl Res.  2017</td>\n",
       "      <td>PubMed</td>\n",
       "      <td>citation</td>\n",
       "      <td>PMID:28585183 | PMCID:PMC5540576</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>28585183</td>\n",
       "      <td>create date:2017/06/07 | first author:Shah SJ</td>\n",
       "      <td>&lt;div class='abstr'&gt;&lt;h3&gt;Abstract&lt;/h3&gt;&lt;div class...</td>\n",
       "      <td>Cluster analysis; Heart failure with preserved...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov//pubmed/28585183</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>2017</td>\n",
       "      <td>640584a9-ad0c-46cf-97f9-575c47f80b99</td>\n",
       "      <td>1</td>\n",
       "      <td>[sanjiv.shah@northwestern.edu]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>deep learning for healthcare: review, opportun...</td>\n",
       "      <td>/pubmed/28481991</td>\n",
       "      <td>Miotto R, Wang F, Wang S, Jiang X, Dudley JT.</td>\n",
       "      <td>Brief Bioinform. 2017 May 6. doi: 10.1093/bib/...</td>\n",
       "      <td>Brief Bioinform.  2017</td>\n",
       "      <td>PubMed</td>\n",
       "      <td>citation</td>\n",
       "      <td>PMID:28481991</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>28481991</td>\n",
       "      <td>create date:2017/05/10 | first author:Miotto R</td>\n",
       "      <td>&lt;div class='abstr'&gt;&lt;h3&gt;Abstract&lt;/h3&gt;&lt;div class...</td>\n",
       "      <td>biomedical informatics; deep learning; electro...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov//pubmed/28481991</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>2017</td>\n",
       "      <td>bae357bb-3bec-43da-8cc4-5095bc4d9bbe</td>\n",
       "      <td>1</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>hla class i binding prediction via convolution...</td>\n",
       "      <td>/pubmed/28444127</td>\n",
       "      <td>Vang YS, Xie X.</td>\n",
       "      <td>Bioinformatics. 2017 Sep 1;33(17):2658-2665. d...</td>\n",
       "      <td>Bioinformatics.  2017</td>\n",
       "      <td>PubMed</td>\n",
       "      <td>citation</td>\n",
       "      <td>PMID:28444127</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>28444127</td>\n",
       "      <td>create date:2017/04/27 | first author:Vang YS</td>\n",
       "      <td>&lt;div class='abstr'&gt;&lt;h3&gt;Abstract&lt;/h3&gt;&lt;div class...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov//pubmed/28444127</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>2017</td>\n",
       "      <td>de37062a-9cfd-460b-9c90-e94689fefd04</td>\n",
       "      <td>1</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>deepcpg: accurate prediction of single-cell dn...</td>\n",
       "      <td>/pubmed/28395661</td>\n",
       "      <td>Angermueller C, Lee HJ, Reik W, Stegle O.</td>\n",
       "      <td>Genome Biol. 2017 Apr 11;18(1):67. doi: 10.118...</td>\n",
       "      <td>Genome Biol.  2017</td>\n",
       "      <td>PubMed</td>\n",
       "      <td>citation</td>\n",
       "      <td>PMID:28395661 | PMCID:PMC5387360</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>28395661</td>\n",
       "      <td>create date:2017/04/12 | first author:Angermue...</td>\n",
       "      <td>&lt;div class='abstr'&gt;&lt;h3&gt;Abstract&lt;/h3&gt;&lt;div class...</td>\n",
       "      <td>Artificial neural network; DNA methylation; De...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov//pubmed/28395661</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>2017</td>\n",
       "      <td>bb3fc9dd-5fc2-4cd5-9173-485ab5481c7e</td>\n",
       "      <td>1</td>\n",
       "      <td>[cangermueller@ebi.ac.uk]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Title               URL  \\\n",
       "0  deepbipolar: identifying genomic mutations for...  /pubmed/28600868   \n",
       "1  precision medicine for heart failure with pres...  /pubmed/28585183   \n",
       "2  deep learning for healthcare: review, opportun...  /pubmed/28481991   \n",
       "3  hla class i binding prediction via convolution...  /pubmed/28444127   \n",
       "4  deepcpg: accurate prediction of single-cell dn...  /pubmed/28395661   \n",
       "\n",
       "                                     Description  \\\n",
       "0       Laksshman S, Bhat RR, Viswanath V, Li X.   \n",
       "1                                       Shah SJ.   \n",
       "2  Miotto R, Wang F, Wang S, Jiang X, Dudley JT.   \n",
       "3                                Vang YS, Xie X.   \n",
       "4      Angermueller C, Lee HJ, Reik W, Stegle O.   \n",
       "\n",
       "                                             Details  \\\n",
       "0  Hum Mutat. 2017 Sep;38(9):1217-1224. doi: 10.1...   \n",
       "1  J Cardiovasc Transl Res. 2017 Jun;10(3):233-24...   \n",
       "2  Brief Bioinform. 2017 May 6. doi: 10.1093/bib/...   \n",
       "3  Bioinformatics. 2017 Sep 1;33(17):2658-2665. d...   \n",
       "4  Genome Biol. 2017 Apr 11;18(1):67. doi: 10.118...   \n",
       "\n",
       "                     ShortDetails Resource      Type  \\\n",
       "0                Hum Mutat.  2017   PubMed  citation   \n",
       "1  J Cardiovasc Transl Res.  2017   PubMed  citation   \n",
       "2          Brief Bioinform.  2017   PubMed  citation   \n",
       "3           Bioinformatics.  2017   PubMed  citation   \n",
       "4              Genome Biol.  2017   PubMed  citation   \n",
       "\n",
       "                        Identifiers      Db  EntrezUID  \\\n",
       "0                     PMID:28600868  pubmed   28600868   \n",
       "1  PMID:28585183 | PMCID:PMC5540576  pubmed   28585183   \n",
       "2                     PMID:28481991  pubmed   28481991   \n",
       "3                     PMID:28444127  pubmed   28444127   \n",
       "4  PMID:28395661 | PMCID:PMC5387360  pubmed   28395661   \n",
       "\n",
       "                                          Properties  \\\n",
       "0  create date:2017/06/11 | first author:Laksshman S   \n",
       "1      create date:2017/06/07 | first author:Shah SJ   \n",
       "2     create date:2017/05/10 | first author:Miotto R   \n",
       "3      create date:2017/04/27 | first author:Vang YS   \n",
       "4  create date:2017/04/12 | first author:Angermue...   \n",
       "\n",
       "                                            abstract  \\\n",
       "0  <div class='abstr'><h3>Abstract</h3><div class...   \n",
       "1  <div class='abstr'><h3>Abstract</h3><div class...   \n",
       "2  <div class='abstr'><h3>Abstract</h3><div class...   \n",
       "3  <div class='abstr'><h3>Abstract</h3><div class...   \n",
       "4  <div class='abstr'><h3>Abstract</h3><div class...   \n",
       "\n",
       "                                            keywords  \\\n",
       "0  bipolar disorder; convolutional neural network...   \n",
       "1  Cluster analysis; Heart failure with preserved...   \n",
       "2  biomedical informatics; deep learning; electro...   \n",
       "3                                                NaN   \n",
       "4  Artificial neural network; DNA methylation; De...   \n",
       "\n",
       "                                         fullURL  source  year  \\\n",
       "0  https://www.ncbi.nlm.nih.gov//pubmed/28600868  pubmed  2017   \n",
       "1  https://www.ncbi.nlm.nih.gov//pubmed/28585183  pubmed  2017   \n",
       "2  https://www.ncbi.nlm.nih.gov//pubmed/28481991  pubmed  2017   \n",
       "3  https://www.ncbi.nlm.nih.gov//pubmed/28444127  pubmed  2017   \n",
       "4  https://www.ncbi.nlm.nih.gov//pubmed/28395661  pubmed  2017   \n",
       "\n",
       "                                    key  use                          emails  \n",
       "0  c3510786-340b-45d7-9272-eee0c8960971    1                              []  \n",
       "1  640584a9-ad0c-46cf-97f9-575c47f80b99    1  [sanjiv.shah@northwestern.edu]  \n",
       "2  bae357bb-3bec-43da-8cc4-5095bc4d9bbe    1                              []  \n",
       "3  de37062a-9cfd-460b-9c90-e94689fefd04    1                              []  \n",
       "4  bb3fc9dd-5fc2-4cd5-9173-485ab5481c7e    1       [cangermueller@ebi.ac.uk]  "
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfa = pd.read_csv('/home/ahmed/Dropbox/DFCI/14_zoo/scrap-science/pubmed/out/pubmed_result.csv', index_col=None)\n",
    "dfa = clean_email_list(dfa)\n",
    "dfa.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(246, 10)"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfa = pd.read_csv('/home/ahmed/Dropbox/DFCI/14_zoo/scrap-science/miccai/out/miccai_result.csv', index_col=None)\n",
    "dfa.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\n"
     ]
    }
   ],
   "source": [
    "counter = 0\n",
    "for i in range(dfa.shape[0]):# \n",
    "    if dfa.iloc[i].email == '[]':\n",
    "        counter+=1\n",
    "print counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[nan,\n",
       " 'sanjiv.shah@northwestern.edu',\n",
       " nan,\n",
       " nan,\n",
       " 'cangermueller@ebi.ac.uk',\n",
       " 'pfbaldi@ics.uci.edu',\n",
       " 'miyano@ims.u-tokyo.ac.jp',\n",
       " 'pastur90@gmail.com',\n",
       " nan,\n",
       " 'jhuang@yorku.ca',\n",
       " 'hbshen@sjtu.edu.cn',\n",
       " 'leopold.parts@sanger.ac.uk',\n",
       " 'anantm@case.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'R.Welikala@kingston.ac.uk',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'german.gonzalez.serrano@gmail.com',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'fabian.theis@helmholtz-muenchen.de',\n",
       " 'richard.torres@yale.edu',\n",
       " nan,\n",
       " 'aru@np.edu.sg',\n",
       " 'xwang.cv@gmail.com',\n",
       " 'mxie@uestc.edu.cn',\n",
       " 'lina.m.sulieman@vanderbilt.edu',\n",
       " 'refael.vivanti@mail.huji.ac.il',\n",
       " nan,\n",
       " 'zhiweiwang@hust.edu.cn',\n",
       " nan,\n",
       " 'tliu@uga.edu',\n",
       " 'mkiranyaz@qu.edu.qa',\n",
       " 'tanaitou-tky@umin.ac.jp',\n",
       " nan,\n",
       " 'phillip.cheng@med.usc.edu',\n",
       " nan,\n",
       " 'gkoc2@cam.ac.uk',\n",
       " nan,\n",
       " 'joao.valiati@ai-engineers.com',\n",
       " nan,\n",
       " 'shampool@ufl.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'bje@mayo.edu',\n",
       " nan,\n",
       " nan,\n",
       " 'minizon@sina.com',\n",
       " nan,\n",
       " nan,\n",
       " 'dkong@zju.edu.cn',\n",
       " 'amirj@robots.ox.ac.uk',\n",
       " nan,\n",
       " 'jalaldeensit@gmail.com',\n",
       " 'david.cunefare@duke.edu',\n",
       " nan,\n",
       " 'mori.shinichiro@qst.go.jp',\n",
       " nan,\n",
       " nan,\n",
       " 'youdong_mao@dfci.harvard.edu',\n",
       " 'mli@uwaterloo.ca',\n",
       " nan,\n",
       " nan,\n",
       " 'yywang@fudan.edu.cn',\n",
       " nan,\n",
       " 'Hugo_Aerts@dfci.harvard.edu',\n",
       " 'mohsen.ghafoorian@radboudumc.nl',\n",
       " nan,\n",
       " 'chi.jianning@gmail.com',\n",
       " 'ksuzuki@iit.edu',\n",
       " 'di.dong@ia.ac.cn',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'harshita.sharma@campus.tu-berlin.de',\n",
       " nan,\n",
       " 'hl87c@mst.edu',\n",
       " nan,\n",
       " 'Eldad@zebra-med.com',\n",
       " nan,\n",
       " 'sdo@mgh.harvard.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'russ.altman@stanford.edu',\n",
       " nan,\n",
       " 'tung.tran@uky.edu',\n",
       " nan,\n",
       " nan,\n",
       " 'Paras.lakhani@jefferson.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'leizhang@scu.edu.cn',\n",
       " 'bje@mayo.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'Sebastien.Adam@univ-rouen.fr',\n",
       " nan,\n",
       " 'xuyan04@gmail.com',\n",
       " nan,\n",
       " 'harry.qin@polyu.edu.hk',\n",
       " 'imonb@stanford.edu',\n",
       " nan,\n",
       " 'anthony.rios1@uky.edu',\n",
       " 'anibali@students.latrobe.edu.au',\n",
       " 'yunzhi.wang-1@ou.edu',\n",
       " 'lukeoakdenrayner@gmail.com',\n",
       " 'asif@pieas.edu.pk',\n",
       " nan,\n",
       " 'wqian@utep.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'jose.dolz.upv@gmail.com',\n",
       " 'hchen@cse.cuhk.edu.hk',\n",
       " nan,\n",
       " 'liewym@um.edu.my',\n",
       " nan,\n",
       " 'svalverde@eia.udg.edu',\n",
       " nan,\n",
       " 'charlie.boone@utoronto.ca',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'bingoon@ieee.org',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'lisiqi881020@163.com',\n",
       " nan,\n",
       " 'jordenh@ece.ubc.ca',\n",
       " nan,\n",
       " 'sahu@zib.de',\n",
       " 'samavi@mcmaster.ca',\n",
       " nan,\n",
       " nan,\n",
       " 'sz2338@columbia.edu',\n",
       " 'bje@Mayo.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'sdo@mgh.harvard.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'zhaoyongjia@buaa.edu.cn',\n",
       " 'pangshuchao1212@sina.com',\n",
       " nan,\n",
       " nan,\n",
       " 'gbilgin@yildiz.edu.tr',\n",
       " 'xypan172436@gmail.com',\n",
       " nan,\n",
       " 'christian.wachinger@med.uni-muenchen.de',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'jeremy.fairbank@ndorms.ox.ac.uk',\n",
       " 'David.Freund@jhuapl.edu',\n",
       " 'hisuk@korea.ac.kr',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'yulijuan2003@126.com',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'pavle.prentasic@fer.hr',\n",
       " nan,\n",
       " 'xiangliu09@fudan.edu.cn',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'jjl5sw@virginia.edu',\n",
       " 'phillip.cheng@med.usc.edu',\n",
       " nan,\n",
       " 'chisa@fjt.info.gifu-u.ac.jp',\n",
       " 'x.gao@mdx.ac.uk',\n",
       " 'dkong@zju.edu.cn',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'konstantinos.kamnitsas12@imperial.ac.uk',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'huangxiwei@hdu.edu.cn',\n",
       " 'cspampin@dieei.unict.it',\n",
       " nan,\n",
       " nan,\n",
       " 'santi.segui@ub.edu',\n",
       " 'chy1000@gmail.com',\n",
       " 'hamarneh@sfu.ca',\n",
       " nan,\n",
       " 'dkong@zju.edu.cn',\n",
       " 'Alvin.rajkomar@ucsf.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'dkong@zju.edu.cn',\n",
       " 'dwarikanath.mahapatra@inf.ethz.ch',\n",
       " 'qianjinfeng08@gmail.com',\n",
       " nan,\n",
       " nan,\n",
       " 'thijs.kooi@radboudumc.nl',\n",
       " nan,\n",
       " 'gisbert.schneider@pharma.ethz.ch',\n",
       " 'wqian@utep.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'seyed.mohammad.havaei@usherbrooke.ca',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'gapou@unistra.fr',\n",
       " nan,\n",
       " nan,\n",
       " 'j.m.wolterink@umcutrecht.nl',\n",
       " 'wangsheng@uchicago.edu',\n",
       " 'xhx@uci.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'oliver.duerr@zhaw.ch',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'jearevaloo@unal.edu.co',\n",
       " nan,\n",
       " 'kleesiek@uni-heidelberg.de',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'nikolaus.kriegeskorte@mrc-cbu.cam.ac.uk',\n",
       " 'francesco.ciompi@radboudumc.nl',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'wangsheng@uchicago.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'jearevaloo@unal.edu.co',\n",
       " 'tzeng@cs.odu.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'sji@cs.odu.edu',\n",
       " 'brosch.tom@gmail.com',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'heekon@med.umich.edu',\n",
       " 'giovanni.montana@kcl.ac.uk',\n",
       " 'arnaud.setio@gmail.com',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'kline.timothy@mayo.edu',\n",
       " 'dgshen@med.unc.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'b.vanginneken@radboudumc.nl',\n",
       " nan,\n",
       " 'daniel.forsberg@sectra.com',\n",
       " 'aselba01@louisville.edu',\n",
       " nan,\n",
       " 'qm.hu@siat.ac.cn',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'hisuk@korea.ac.kr',\n",
       " nan,\n",
       " 'hisuk@korea.ac.kr',\n",
       " nan,\n",
       " 'dgshen@med.unc.edu',\n",
       " 'hsuk@med.unc.edu',\n",
       " 'lee.cooper@emory.edu',\n",
       " nan,\n",
       " nan,\n",
       " 'geert.litjens@radboudumc.nl',\n",
       " nan,\n",
       " 'jmluna@us.es',\n",
       " nan,\n",
       " nan,\n",
       " 'shazizi@ece.ubc.ca',\n",
       " 'gwenole.quellec@inserm.fr',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'andrew.janowczyk@case.edu',\n",
       " nan,\n",
       " 'shazizi@ece.ubc.ca',\n",
       " 'premaladha@sastra.edu',\n",
       " nan,\n",
       " 'zhenyouw@gdut.edu.cn',\n",
       " nan,\n",
       " nan,\n",
       " 'ithapu@wisc.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'zc.li@siat.ac.cn',\n",
       " 'abhijit.guha-roy@tum.de',\n",
       " nan,\n",
       " 'Lars.ebert@virtopsy.com',\n",
       " nan,\n",
       " nan,\n",
       " 'wbz99@sina.com',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'eriksson.monteiro@ua.pt',\n",
       " 'tedleng@stanford.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'rasaoka-tky@umin.ac.jp',\n",
       " 'd.rueckert@imperial.ac.uk',\n",
       " 'slishuo@gmail.com',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'shuo.li@ge.com',\n",
       " 'm.avendi@uci.edu',\n",
       " 'gunther.helms@med.lu.se',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'szalkai@pitgroup.org',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'yishi@sjtu.edu.cn',\n",
       " 'danaeep@oregonstate.edu',\n",
       " 'schaterj@purdue.edu',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'luc17@pitt.edu',\n",
       " nan,\n",
       " 'chao.cheng@dartmouth.edu',\n",
       " nan,\n",
       " 'zhangsw@nwpu.edu.cn',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'leopold.parts@sanger.ac.uk',\n",
       " nan,\n",
       " nan,\n",
       " 'jens.rittscher@eng.ox.ac.uk',\n",
       " 'viktoria.shpacovitch@isas.de',\n",
       " 'anatole.chessel@polytechnique.edu',\n",
       " nan,\n",
       " nan,\n",
       " 'isutian@gmail.com',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'dyao@uestc.edu.cn',\n",
       " 'irene.sturm@tu-berlin.de',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'y.guo@imperial.ac.uk',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " 'h.ohsugi@tsukazaki-eye.net',\n",
       " 'qaisarabbasphd@gmail.com',\n",
       " nan,\n",
       " nan,\n",
       " 'marco.visentiniscarzanella@gmail.com',\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan,\n",
       " nan]"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfa = pd.read_csv('/home/ahmed/Dropbox/DFCI/14_zoo/scrap-science/pubmed/out/pubmed_result.csv', index_col=None)\n",
    "dfa.email.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'float'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n",
      "<type 'str'>\n"
     ]
    }
   ],
   "source": [
    "for i in range(df.shape[0]):\n",
    "    print type(df.iloc[i].email)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_json( 'Data.json' ,\"records\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(509, 19)\n"
     ]
    }
   ],
   "source": [
    "ddf = pd.read_csv('/home/ahmed/Dropbox/DFCI/14_zoo/scrap-science/pubmed/archive/pubmed_clean.csv')\n",
    "ddf.shape\n",
    "\n",
    "ddf['Title'] = df.Title.str.lower()\n",
    "ddf['Title'] = df.Title.str.strip()\n",
    "ddf['Title'] = df.Title.str.replace('.','')\n",
    "ddf['Title'] = df.Title.str.replace('  ',' ')\n",
    "# remove duplicates by title\n",
    "# ddf = ddf.drop_duplicates(subset='Title', keep=\"first\")\n",
    "# remove use==0\n",
    "ddf = ddf[ddf['use']==1]\n",
    "# sort by date\n",
    "#####\n",
    "# reset index\n",
    "ddf = ddf.reset_index(drop=True)\n",
    "print ddf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venvradiomicsio",
   "language": "python",
   "name": "venvradiomicsio"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}