notebooks/01-preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess Data\n",
    "\n",
    "We will work with the text of papers from the Neural Information Processing Systems (NIPS). The data was collected by Ben Hamner in 2015 and shared as a SQLite3 Database file at [https://www.kaggle.com/benhamner/nips-papers] containing 3 tables, papers, authors and paper_authors. There are 7237 academic papers in this dataset, spanning from 1987-2015.\n",
    "\n",
    "This data has been manually exported from SQLite3 into 3 CSV files papers.csv, authors.csv and paper_authors.csv, using [commands described here](http://www.sqlitetutorial.net/sqlite-export-csv/).\n",
    "\n",
    "In this notebook, we will process these three CSV files into our first search index, as well as convert into text files that can be processed more easily with downstream tools."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import pandas as pd\n",
    "import re\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = \"../data\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Papers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>year</th>\n",
       "      <th>title</th>\n",
       "      <th>event_type</th>\n",
       "      <th>pdf_name</th>\n",
       "      <th>abstract</th>\n",
       "      <th>paper_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1987</td>\n",
       "      <td>Self-Organization of Associative Database and ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1-self-organization-of-associative-database-an...</td>\n",
       "      <td>Abstract Missing</td>\n",
       "      <td>767\\n\\nSELF-ORGANIZATION OF ASSOCIATIVE DATABA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1987</td>\n",
       "      <td>The Capacity of the Kanerva Associative Memory...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2-the-capacity-of-the-kanerva-associative-memo...</td>\n",
       "      <td>Abstract Missing</td>\n",
       "      <td>184\\n\\nTHE CAPACITY OF THE KANERVA ASSOCIATIVE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1987</td>\n",
       "      <td>Supervised Learning of Probability Distributio...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3-supervised-learning-of-probability-distribut...</td>\n",
       "      <td>Abstract Missing</td>\n",
       "      <td>52\\n\\nSupervised Learning of Probability Distr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1987</td>\n",
       "      <td>Constrained Differential Optimization</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4-constrained-differential-optimization.pdf</td>\n",
       "      <td>Abstract Missing</td>\n",
       "      <td>612\\n\\nConstrained Differential Optimization\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1987</td>\n",
       "      <td>Towards an Organizing Principle for a Layered ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5-towards-an-organizing-principle-for-a-layere...</td>\n",
       "      <td>Abstract Missing</td>\n",
       "      <td>485\\n\\nTOWARDS AN ORGANIZING PRINCIPLE FOR\\nA ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  year                                              title event_type  \\\n",
       "0   1  1987  Self-Organization of Associative Database and ...        NaN   \n",
       "1   2  1987  The Capacity of the Kanerva Associative Memory...        NaN   \n",
       "2   3  1987  Supervised Learning of Probability Distributio...        NaN   \n",
       "3   4  1987              Constrained Differential Optimization        NaN   \n",
       "4   5  1987  Towards an Organizing Principle for a Layered ...        NaN   \n",
       "\n",
       "                                            pdf_name          abstract  \\\n",
       "0  1-self-organization-of-associative-database-an...  Abstract Missing   \n",
       "1  2-the-capacity-of-the-kanerva-associative-memo...  Abstract Missing   \n",
       "2  3-supervised-learning-of-probability-distribut...  Abstract Missing   \n",
       "3        4-constrained-differential-optimization.pdf  Abstract Missing   \n",
       "4  5-towards-an-organizing-principle-for-a-layere...  Abstract Missing   \n",
       "\n",
       "                                          paper_text  \n",
       "0  767\\n\\nSELF-ORGANIZATION OF ASSOCIATIVE DATABA...  \n",
       "1  184\\n\\nTHE CAPACITY OF THE KANERVA ASSOCIATIVE...  \n",
       "2  52\\n\\nSupervised Learning of Probability Distr...  \n",
       "3  612\\n\\nConstrained Differential Optimization\\n...  \n",
       "4  485\\n\\nTOWARDS AN ORGANIZING PRINCIPLE FOR\\nA ...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "papers_df = pd.read_csv(os.path.join(DATA_DIR, \"papers.csv\"))\n",
    "papers_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Authors\n",
    "\n",
    "We will create an in-memory dictionary of author ID to author name, which we will use in the paper_authors dataframe below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Hisashi Suzuki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Suguru Arimoto</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Philip A. Chou</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>John C. Platt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Alan H. Barr</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id            name\n",
       "0   1  Hisashi Suzuki\n",
       "1   2  Suguru Arimoto\n",
       "2   3  Philip A. Chou\n",
       "3   4   John C. Platt\n",
       "4   5    Alan H. Barr"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authors_df = pd.read_csv(os.path.join(DATA_DIR, \"authors.csv\"))\n",
    "authors_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "author_ids = authors_df[\"id\"].values\n",
    "author_names = authors_df[\"name\"].values\n",
    "author_id2name = dict(zip(author_ids, author_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Paper Authors\n",
    "\n",
    "We want to denormalize the authors and attach them to the papers dataframe. To do this, we first group by paper_id, then use our author id to name lookup to create a list of author names for each paper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>paper_id</th>\n",
       "      <th>author_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>63</td>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>80</td>\n",
       "      <td>124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>80</td>\n",
       "      <td>125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>80</td>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>80</td>\n",
       "      <td>127</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  paper_id  author_id\n",
       "0   1        63         94\n",
       "1   2        80        124\n",
       "2   3        80        125\n",
       "3   4        80        126\n",
       "4   5        80        127"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paper_authors_df = pd.read_csv(os.path.join(DATA_DIR, \"paper_authors.csv\"))\n",
    "paper_authors_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author_names</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>paper_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[Hisashi Suzuki, Suguru Arimoto]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[Philip A. Chou]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[Eric B. Baum, Frank Wilczek]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[John C. Platt, Alan H. Barr]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[Ralph Linsker]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              author_names\n",
       "paper_id                                  \n",
       "1         [Hisashi Suzuki, Suguru Arimoto]\n",
       "2                         [Philip A. Chou]\n",
       "3            [Eric B. Baum, Frank Wilczek]\n",
       "4            [John C. Platt, Alan H. Barr]\n",
       "5                          [Ralph Linsker]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paper_authors_df[\"author_names\"] = paper_authors_df[\"author_id\"].apply(\n",
    "    lambda x: [author_id2name[x]])\n",
    "pa_grouped_df = paper_authors_df.groupby([\"paper_id\"]).agg({\"author_names\": \"sum\"})\n",
    "pa_grouped_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Denormalize papers dataframe\n",
    "\n",
    "We join our paper dataframe with the processed paper_authors dataframe to produce our denormalized papers dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>year</th>\n",
       "      <th>title</th>\n",
       "      <th>event_type</th>\n",
       "      <th>pdf_name</th>\n",
       "      <th>abstract</th>\n",
       "      <th>paper_text</th>\n",
       "      <th>author_names</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1987</td>\n",
       "      <td>Self-Organization of Associative Database and ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1-self-organization-of-associative-database-an...</td>\n",
       "      <td>Abstract Missing</td>\n",
       "      <td>767\\n\\nSELF-ORGANIZATION OF ASSOCIATIVE DATABA...</td>\n",
       "      <td>[Hisashi Suzuki, Suguru Arimoto]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1987</td>\n",
       "      <td>The Capacity of the Kanerva Associative Memory...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2-the-capacity-of-the-kanerva-associative-memo...</td>\n",
       "      <td>Abstract Missing</td>\n",
       "      <td>184\\n\\nTHE CAPACITY OF THE KANERVA ASSOCIATIVE...</td>\n",
       "      <td>[Philip A. Chou]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1987</td>\n",
       "      <td>Supervised Learning of Probability Distributio...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3-supervised-learning-of-probability-distribut...</td>\n",
       "      <td>Abstract Missing</td>\n",
       "      <td>52\\n\\nSupervised Learning of Probability Distr...</td>\n",
       "      <td>[Eric B. Baum, Frank Wilczek]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1987</td>\n",
       "      <td>Constrained Differential Optimization</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4-constrained-differential-optimization.pdf</td>\n",
       "      <td>Abstract Missing</td>\n",
       "      <td>612\\n\\nConstrained Differential Optimization\\n...</td>\n",
       "      <td>[John C. Platt, Alan H. Barr]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1987</td>\n",
       "      <td>Towards an Organizing Principle for a Layered ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5-towards-an-organizing-principle-for-a-layere...</td>\n",
       "      <td>Abstract Missing</td>\n",
       "      <td>485\\n\\nTOWARDS AN ORGANIZING PRINCIPLE FOR\\nA ...</td>\n",
       "      <td>[Ralph Linsker]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  year                                              title event_type  \\\n",
       "0      1  1987  Self-Organization of Associative Database and ...        NaN   \n",
       "1      2  1987  The Capacity of the Kanerva Associative Memory...        NaN   \n",
       "2      3  1987  Supervised Learning of Probability Distributio...        NaN   \n",
       "3      4  1987              Constrained Differential Optimization        NaN   \n",
       "4      5  1987  Towards an Organizing Principle for a Layered ...        NaN   \n",
       "\n",
       "                                            pdf_name          abstract  \\\n",
       "0  1-self-organization-of-associative-database-an...  Abstract Missing   \n",
       "1  2-the-capacity-of-the-kanerva-associative-memo...  Abstract Missing   \n",
       "2  3-supervised-learning-of-probability-distribut...  Abstract Missing   \n",
       "3        4-constrained-differential-optimization.pdf  Abstract Missing   \n",
       "4  5-towards-an-organizing-principle-for-a-layere...  Abstract Missing   \n",
       "\n",
       "                                          paper_text  \\\n",
       "0  767\\n\\nSELF-ORGANIZATION OF ASSOCIATIVE DATABA...   \n",
       "1  184\\n\\nTHE CAPACITY OF THE KANERVA ASSOCIATIVE...   \n",
       "2  52\\n\\nSupervised Learning of Probability Distr...   \n",
       "3  612\\n\\nConstrained Differential Optimization\\n...   \n",
       "4  485\\n\\nTOWARDS AN ORGANIZING PRINCIPLE FOR\\nA ...   \n",
       "\n",
       "                       author_names  \n",
       "0  [Hisashi Suzuki, Suguru Arimoto]  \n",
       "1                  [Philip A. Chou]  \n",
       "2     [Eric B. Baum, Frank Wilczek]  \n",
       "3     [John C. Platt, Alan H. Barr]  \n",
       "4                   [Ralph Linsker]  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "papers_denorm_df = (papers_df.set_index(\"id\")\n",
    "                    .join(pa_grouped_df.reset_index().set_index(\"paper_id\"), \n",
    "                          rsuffix=\"_pa\", how=\"inner\")\n",
    "                    .reset_index())\n",
    "papers_denorm_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing Solr to receive data\n",
    "\n",
    "In your Solr directory, execute following commands to build the first index to hold the papers (nips0index).\n",
    "\n",
    "    bin/solr create -c nips0index\n",
    "    \n",
    "Then run the following command to set up the index for your schema ([code](https://github.com/sujitpal/content-engineering-tutorial/blob/master/scripts/create_schema_0.sh)).\n",
    "\n",
    "    scripts/create_schema_0.sh\n",
    "    \n",
    "More information:\n",
    "\n",
    "* [Solr Field Types and Properties](https://lucene.apache.org/solr/guide/7_2/field-type-definitions-and-properties.html)\n",
    "* [Solr Analyzers, Tokenizers and Tokenfilters](https://lucene.apache.org/solr/guide/7_2/understanding-analyzers-tokenizers-and-filters.html)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upload data to Solr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Uploaded 0 records to Solr\n",
      "Uploaded 1000 records to Solr\n",
      "Uploaded 2000 records to Solr\n",
      "Uploaded 3000 records to Solr\n",
      "Uploaded 4000 records to Solr\n",
      "Uploaded 5000 records to Solr\n",
      "Uploaded 6000 records to Solr\n",
      "Uploaded 7000 records to Solr\n",
      "Uploaded 7237 records to Solr\n"
     ]
    }
   ],
   "source": [
    "# solr url\n",
    "solr_url = \"http://localhost:8983/solr\"\n",
    "headers = {\n",
    "    \"content-type\": \"application/json\",\n",
    "    \"accept\": \"application/json\"\n",
    "}\n",
    "# create subdirectory for text files\n",
    "textfiles_dir = os.path.join(DATA_DIR, \"textfiles\")\n",
    "if not os.path.exists(textfiles_dir):\n",
    "    os.mkdir(textfiles_dir)\n",
    "# create metadata file\n",
    "fmeta = open(os.path.join(DATA_DIR, \"papers_metadata.tsv\"), \"w\")\n",
    "fmeta.write(\"#id\\tyear\\ttitle\\tabstract\\tauthor_names\\n\")\n",
    "for paper_id, row in papers_denorm_df.iterrows():\n",
    "    year = row[\"year\"]\n",
    "    title = row[\"title\"]\n",
    "    abstract = row[\"abstract\"]\n",
    "    # squeeze out newlines from abstract\n",
    "    abstract = abstract.replace(\"\\n\", \" \")\n",
    "    abstract = re.sub(\"\\s+\", \" \", abstract)\n",
    "    paper_text = row[\"paper_text\"]\n",
    "    author_names = row[\"author_names\"]\n",
    "    # write out metadata\n",
    "    fmeta.write(\"{:d}\\t{:d}\\t{:s}\\t{:s}\\t{:s}\\n\"\n",
    "                .format(paper_id, year, title, abstract, \":\".join(author_names)))\n",
    "    # write out text file\n",
    "    ftext = open(os.path.join(textfiles_dir, \"{:d}.txt\".format(paper_id)), \"w\")\n",
    "    ftext.write(paper_text)\n",
    "    ftext.close()\n",
    "    # post to index\n",
    "    data = json.dumps({\n",
    "        \"add\": {\n",
    "            \"doc\": {\n",
    "                \"id\": paper_id,\n",
    "                \"year\": year,\n",
    "                \"title\": title,\n",
    "                \"abstract\": abstract,\n",
    "                \"text\": paper_text,\n",
    "                \"author_names\": [ author_names ]\n",
    "            }\n",
    "        }\n",
    "    })\n",
    "    if paper_id % 1000 == 0:\n",
    "        print(\"Uploaded {:d} records to Solr\".format(paper_id))\n",
    "        params = {\"commit\": \"true\"}\n",
    "    else:\n",
    "        params = {\"commit\": \"false\"}\n",
    "    requests.post(solr_url + \"/nips0index/update\", data=data, params=params, headers=headers)\n",
    "fmeta.close()\n",
    "requests.post(solr_url + \"/nips0index/update\", headers=headers, params={\"commit\": \"true\"})\n",
    "print(\"Uploaded {:d} records to Solr\".format(paper_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}