Merge pull request #4 from klrkdekira/master

Merge updates from upstream

billwatcher/__init__.py

@@ -8,49 +8,71 @@
 import pymongo
 import bson
 
+import elasticsearch
+
 def main(global_config, **settings):
     """ This function returns a Pyramid WSGI application.
     """
     config = Configurator(settings=settings)
+
+    # Path for locale files
     config.add_translation_dirs('locale/')
 
     # Custom JSON renderer
     json_renderer = JSON()
+
+    # Support date to json serialization 
     def datetime_adapter(obj, request):
         return obj.isoformat()
     json_renderer.add_adapter(datetime.datetime, datetime_adapter)
 
+    # Support objectid to json serialization
     def objectid_adapter(obj, request):
         return unicode(obj)
     json_renderer.add_adapter(bson.objectid.ObjectId, objectid_adapter)
 
     config.add_renderer('json', json_renderer)
 
+    # Jinja2
     config.include('pyramid_jinja2')
+    # Set jinja2 as renderer for `html` file
     config.add_renderer('.html', 'pyramid_jinja2.renderer_factory')
+    # Relative path for jinja2 templates
     config.add_jinja2_search_path("billwatcher:templates")
 
     config.add_static_view('static', 'billwatcher:static',
                            cache_max_age=3600)
 
+    # Parse mongodb uri
     db_url = urlparse(settings['mongo_uri'])
     config.registry.db = pymongo.Connection(
         host=db_url.hostname,
         port=db_url.port
     )
 
+    # Make MongoDB available via request
     def add_db(request):
         db = config.registry.db[db_url.path[1:]]
         if db_url.username and db_url.password:
             db.authenticate(db_url.username, db_url.password)
         return db
 
+    config.add_request_method(add_db, 'db', reify=True)
+
+    # Make GridFS available via request
     def add_fs(request):
         return GridFS(request.db)
 
-    config.add_request_method(add_db, 'db', reify=True)
     config.add_request_method(add_fs, 'fs', reify=True)
+
+    # Make elasticsearch available via request
+    def add_es(request):
+        es = elasticsearch.Elasticsearch(settings['es_uri'])
+        return es
 
+    config.add_request_method(add_es, 'es', reify=True)
+
+    # View and routes setup
     config.include('billwatcher.views.views_include')
     config.scan()
     return config.make_wsgi_app()

billwatcher/scripts/pdf_parse.py → billwatcher/scripts/bill_parse.py

@@ -82,8 +82,9 @@ def worker():
             item = q.get()
             try:
                 pdf_to_text(item)
-            finally:
-                q.task_done()
+            except Exception as e:
+                log.error(e)
+            q.task_done()
 
     for i in xrange(4):
         gevent.spawn(worker)

billwatcher/scripts/bill_to_es.py

@@ -0,0 +1,91 @@
+from gevent import monkey; monkey.patch_all()
+import gevent
+from gevent import queue
+import os
+import sys
+import logging
+import multiprocessing
+
+from urlparse import urlparse
+import pymongo
+
+import elasticsearch
+
+from pyramid.paster import (
+    get_appsettings,
+    setup_logging,
+    )
+
+log = logging.getLogger(__name__)
+
+def usage(argv):
+    cmd = os.path.basename(argv[0])
+    print('usage: %s <config_uri>\n'
+          '(example: "%s development.ini")' % (cmd, cmd))
+    sys.exit(1)
+
+
+def main(argv=sys.argv):
+    if len(argv) != 2:
+        usage(argv)
+    config_uri = argv[1]
+    setup_logging(config_uri)
+    settings = get_appsettings(config_uri)
+
+    db_url = urlparse(settings['mongo_uri'])
+    conn = pymongo.Connection(
+        host=db_url.hostname,
+        port=db_url.port
+    )
+
+    db = conn.billwatcher
+
+    q = queue.JoinableQueue()
+
+    es = elasticsearch.Elasticsearch(settings['es_uri'])
+    def index_bill(bill):
+        body = {'name': bill['name'],
+                'description': bill['description'],
+                'year': bill['year'],
+                'status': bill['status']}
+
+        document = bill.get('document')
+        if document:
+            content = document.get('content')
+            if content:
+                body['content'] = content
+
+        res = es.index(index='billwatcher',
+                       doc_type='bill',
+                       id=bill['_id'],
+                       body=body)
+        log.info(res)
+
+    def worker():
+        while True:
+            item = q.get()
+            try:
+                index_bill(item)
+            except Exception as e:
+                log.error(e)
+            q.task_done()
+
+    worker_count = multiprocessing.cpu_count() + 1
+    log.info("Initialising {worker_count} workers...".format(worker_count=worker_count))
+    for i in xrange(worker_count):
+        gevent.spawn(worker)
+    log.info("Workers started...")
+
+    log.info("Inserting records to workers...")
+    record_count = 0
+    for bill in db.bills.find():
+        q.put(bill)
+        record_count += 1
+    log.info("{record_count} inserted...".format(record_count=record_count))
+
+    log.info("Processing...")
+    q.join()
+    log.info('Refreshing index...')
+    es.indices.refresh(index='billwatcher')
+    log.info('Done!')
+    sys.exit(1)

billwatcher/static/css/main.css

@@ -6,18 +6,18 @@ html {
 }
 body {
     /* Margin bottom by footer height */
-    margin-bottom: 180px;
+    margin-bottom: 200px;
 }
 #footer {
     position: absolute;
     bottom: 0;
     width: 100%;
     /* Set the fixed height of the footer here */
-    height: 180px;
-    /* height: 100%; */
-    background-color: #f5f5f5;
+    height: 200px;
 }
 
+
+
 #jumbo {
     height: 300px;    font-size: 20px;
 
@@ -59,3 +59,29 @@ body {
     display: none;
     margin: -20px 0;
 }
+
+@media screen and (min-width: 1024px) {
+    form[role=search] > .input-group {
+        max-width: 250px;
+    }
+
+    #footer {
+        background-color: #f5f5f5;
+    }
+
+    #bill-list th:first-child {
+        width: 120px;
+    }
+
+    #bill-list th:last-child {
+        width: 100px;
+    }
+
+    #bill-list th:nth-child(3) {
+        width: 200px;
+    }
+}
+
+.panel-heading > a {
+    color: #fff;
+}

billwatcher/templates/bill/detail.html

@@ -45,6 +45,8 @@
       <div class="col-md-3" id="social">
         <a href="{{request.url}}" class="twitter-share-button" data-via="twitterapi" data-lang="en">Tweet</a>
         <div class="fb-like" data-href="{{request.url}}" data-layout="standard" data-action="like" data-show-faces="true" data-share="true"></div>
+        <h3>Comments</h3>
+        <hr />
         <div class="fb-comments" data-href="{{request.url}}" data-numposts="20" data-colorscheme="light"></div>        
       </div>
     </div>

billwatcher/templates/bill/list.html

@@ -4,45 +4,53 @@
   <div class="row">
     <div class="col-lg-12">
       <div class="col-md-2 sidebar">
-        <div class="panel panel-primary">
-          <div class="panel-heading">
-            Year
+        <div class="panel-group" id="accordion">          
+          <div class="panel panel-primary">
+            <div class="panel-heading">
+              Year
+              <a data-toggle="collapse" data-parent="#accordion" href="#year-panel">
+                <i class="fa fa-chevron-down pull-right"></i>
+              </a>
+            </div>
+            <div id="year-panel" class="list-group panel-collapse collapse out">
+              {% for y in years %}
+              <a href="{{request.route_url('bill.list', _query={'year': y['_id'], 'page': data.page})}}" class="list-group-item">
+                <span class="badge">{{y['count']}}</span>
+                {{y['_id']}}
+              </a>
+              {% endfor %}
+            </div>
           </div>
-          <div class="list-group">
-            {% for y in years %}
-            <a href="{{request.route_url('bill.list', _query={'year': y['_id'], 'page': data.page})}}" class="list-group-item">
-              <span class="badge">{{y['count']}}</span>
-              {{y['_id']}}
-            </a>
-            {% endfor %}
-          </div>
-        </div>
-        <div class="panel panel-primary">
-          <div class="panel-heading">
-            Status
-          </div>
-          <div class="list-group">
-            {% for s in statuses %}
-            <a href="{{request.route_url('bill.list', _query={'status': s['_id'], 'page': data.page})}}" class="list-group-item">
-              <span class="badge">{{s['count']}}</span>
-              {{s['_id'].capitalize()}}
-            </a>
-            {% endfor %}
+          <div class="panel panel-primary">
+            <div class="panel-heading">
+              Status
+              <a data-toggle="collapse" data-parent="#accordion" href="#status-panel">
+                <i class="fa fa-chevron-down pull-right"></i>
+              </a>
+            </div>
+            <div id="status-panel" class="list-group panel-collapse collapse out">
+              {% for s in statuses %}
+              <a href="{{request.route_url('bill.list', _query={'status': s['_id'], 'page': data.page})}}" class="list-group-item">
+                <span class="badge">{{s['count']}}</span>
+                {{s['_id'].capitalize()}}
+              </a>
+              {% endfor %}
+            </div>            
           </div>
         </div>
       </div>
-      <div class="col-md-10">
+      <div class="col-md-10 col-sm-12 col-xs-12">
         <h4 class="pull-left">Parliament Bills</h4>
         <div id="pager" class="pull-right">
           {{data.pager(format='$link_first $link_previous ~2~ $link_next $link_last')}}
         </div>
-        <table class="table table-striped table-responsive">
+        <table id="bill-list" class="table table-striped table-responsive">
 	  <thead>
 	    <tr>
-              <th style="width: 120px">Code</th>
+              <th>Code</th>
 	      <th>Bill</th>
-	      <th style="width: 200px">Status</th>
-	      <th style="width: 100px">Year</th>
+	      <th>Status</th>
+	      <th>Year</th>
 	    </tr>
 	  </thead>
 	  <tbody>

billwatcher/templates/search.html

@@ -17,15 +17,15 @@ <h4 class="pull-left">Search results for <strong>{{request.params.get('search')}
 	  {% for bill in bills %}
 	  <tr>
             <td>
-              {{bill.name}}
+              {{bill['_source'].name}}
             </td>
 	    <td>
 	      <a href="{{request.route_url('bill.detail', bill_id=bill._id)}}">
-		{{bill.description}}
+		{{bill['_source'].description}}
 	      </a>
 	    </td>
-	    <td>{{bill.status.capitalize() if bill.status}}</td>
-	    <td>{{bill.year or ''}}</td>
+	    <td>{{bill['_source'].status.capitalize() if bill['_source'].status}}</td>
+	    <td>{{bill['_source'].year or ''}}</td>
 	  </tr>
 	  {% endfor %}
 	</tbody>