BERT Summarization.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.3"},"colab":{"name":"BERT Summarization.ipynb","provenance":[{"file_id":"1ynLrVtjbIFhNemCBQoxlrHlI8jOCBT_v","timestamp":1603574485731}],"collapsed_sections":[]},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"562ba03c07194f618520cc022f5113df":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_ee0b64d4d0d64781973137410b127dc5","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_755d7b8ba5ae4e5090ff675bb8088870","IPY_MODEL_8672ba887c804665b27412d21b92e3b9"]}},"ee0b64d4d0d64781973137410b127dc5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"755d7b8ba5ae4e5090ff675bb8088870":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_73eb109574344bbbbe9cd0f0336bda1d","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":434,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":434,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_d20c77100fb04921b8fed034b1c10845"}},"8672ba887c804665b27412d21b92e3b9":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_2c206c0bc48145349181e4a8f7e96e6b","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 434/434 [00:00&lt;00:00, 675B/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_26da5f4b0b1e48988f4900472656cb89"}},"73eb109574344bbbbe9cd0f0336bda1d":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"d20c77100fb04921b8fed034b1c10845":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"2c206c0bc48145349181e4a8f7e96e6b":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"26da5f4b0b1e48988f4900472656cb89":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"39465780c2674be0ac9b70760d35e203":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_88871ba0042e400a8e339b056ac0b02f","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_dc0cc46266a04e0591d9d6422059a08d","IPY_MODEL_bb67796dfa46462cae9a39957d5cfdd0"]}},"88871ba0042e400a8e339b056ac0b02f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"dc0cc46266a04e0591d9d6422059a08d":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_31c0f460c3644f5986d2059f8ae49c6b","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":1344997306,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1344997306,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_1bfc647df4424c27b57f75d25a8e1eb8"}},"bb67796dfa46462cae9a39957d5cfdd0":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_0bf3b26a51be48168957a46f785ee83f","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1.34G/1.34G [00:33&lt;00:00, 40.2MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_0426389c060d4b2097f3d11e1528d81b"}},"31c0f460c3644f5986d2059f8ae49c6b":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"1bfc647df4424c27b57f75d25a8e1eb8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"0bf3b26a51be48168957a46f785ee83f":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"0426389c060d4b2097f3d11e1528d81b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"8957ac057b4c4ed4a9f53adecc91dd88":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_816be59fbd2c48689434af195b3bd762","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_3706e7d12d85421d8f41706d061f753a","IPY_MODEL_ebfe4bab1aba4a7fbf8ede1870cfbe25"]}},"816be59fbd2c48689434af195b3bd762":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3706e7d12d85421d8f41706d061f753a":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_579acdfece344e1cb560f53dccd3eea4","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":231508,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":231508,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_f76c18ceca3d41169285f44fb5d626de"}},"ebfe4bab1aba4a7fbf8ede1870cfbe25":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_d91c7af9c8a6403ba37d18b51fc842d0","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 232k/232k [00:00&lt;00:00, 786kB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_d55b42bd81a447c686c1528feef95b47"}},"579acdfece344e1cb560f53dccd3eea4":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"f76c18ceca3d41169285f44fb5d626de":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d91c7af9c8a6403ba37d18b51fc842d0":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"d55b42bd81a447c686c1528feef95b47":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"code","metadata":{"id":"sIOBnr8EiMKu"},"source":["from google.colab import drive\n","drive.mount('/content/drive')\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"r-d7BSA_icyU","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1603924844953,"user_tz":240,"elapsed":340,"user":{"displayName":"Shivika K Bisen","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgXYEJHMnxhEzLAhEnFo8Lhi33GVbNQAgVg14QwUQ=s64","userId":"00044211735934653548"}},"outputId":"27b73bff-5716-4fb6-dd17-71ddc4cef6a0"},"source":["%cd \"/content/drive/My Drive//\""],"execution_count":null,"outputs":[{"output_type":"stream","text":["/content/drive/My Drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Dm908g3lqGD0"},"source":["import sqlite3\n","import pandas as pd\n","# Create your connection.\n","conn = sqlite3.connect('TestDB1.db')\n","q = \"SELECT * FROM sqlite_master WHERE type='table';\"\n","db = pd.read_sql_query(q, conn)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"AXa2HoYxKTp3"},"source":["query = \"SELECT * FROM PDFS\"\n","init_train = pd.read_sql_query(query,conn)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"l6BK4xIcKTp6","colab":{"base_uri":"https://localhost:8080/","height":402},"executionInfo":{"status":"ok","timestamp":1603924854961,"user_tz":240,"elapsed":254,"user":{"displayName":"Shivika K Bisen","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgXYEJHMnxhEzLAhEnFo8Lhi33GVbNQAgVg14QwUQ=s64","userId":"00044211735934653548"}},"outputId":"46d51709-a8fd-4468-98d9-bbf1b2507609"},"source":["init_train"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>ID</th>\n","      <th>Title</th>\n","      <th>Date</th>\n","      <th>Content</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>Microsoft Word - health-policy.doc</td>\n","      <td>2006-10-20T11:53:20Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1</td>\n","      <td>Health Promotion Glossary</td>\n","      <td>1998-01-06T11:10:08Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2</td>\n","      <td>The Use of Structured Observations in the Stud...</td>\n","      <td>2000-12-06T13:57:01Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>3</td>\n","      <td>RC_Health_message_Standard Operational_Procedu...</td>\n","      <td>2016-10-12T07:43:56Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>4</td>\n","      <td>Microsoft Word - JP7CIMCIProcessGuideFormat.doc</td>\n","      <td>2004-08-27T18:49:28Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>248</th>\n","      <td>248</td>\n","      <td>Climate change, water and food security - ODI ...</td>\n","      <td>2009-03-11T09:04:07Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>249</th>\n","      <td>249</td>\n","      <td>IFRS - Group, Report (new structure)</td>\n","      <td>2018-04-18T07:02:36Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>250</th>\n","      <td>250</td>\n","      <td>Food Security and Nutrition Indicator Basics</td>\n","      <td>2017-11-29T13:20:33Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>251</th>\n","      <td>251</td>\n","      <td>Background document about the Clusters and WAS...</td>\n","      <td>2008-03-28T08:12:19Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>252</th>\n","      <td>252</td>\n","      <td>null</td>\n","      <td>2019-06-03T07:27:58Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>253 rows × 4 columns</p>\n","</div>"],"text/plain":["      ID  ...                                            Content\n","0      0  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","1      1  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","2      2  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","3      3  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","4      4  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","..   ...  ...                                                ...\n","248  248  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","249  249  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","250  250  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","251  251  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","252  252  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","\n","[253 rows x 4 columns]"]},"metadata":{"tags":[]},"execution_count":6}]},{"cell_type":"code","metadata":{"id":"g-ndeRsfgo0p","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1603924865115,"user_tz":240,"elapsed":6590,"user":{"displayName":"Shivika K Bisen","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgXYEJHMnxhEzLAhEnFo8Lhi33GVbNQAgVg14QwUQ=s64","userId":"00044211735934653548"}},"outputId":"78391795-750c-469e-a441-d56861d82cd2"},"source":["!pip install bert-extractive-summarizer"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Collecting bert-extractive-summarizer\n","  Downloading https://files.pythonhosted.org/packages/7b/e3/c8b820d8c0a96a9318a423a38242275f2e862425793b5b8287e982324ffc/bert_extractive_summarizer-0.5.1-py3-none-any.whl\n","Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from bert-extractive-summarizer) (0.22.2.post1)\n","Requirement already satisfied: spacy in /usr/local/lib/python3.6/dist-packages (from bert-extractive-summarizer) (2.2.4)\n","Collecting transformers\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2c/4e/4f1ede0fd7a36278844a277f8d53c21f88f37f3754abf76a5d6224f76d4a/transformers-3.4.0-py3-none-any.whl (1.3MB)\n","\u001b[K     |████████████████████████████████| 1.3MB 6.7MB/s \n","\u001b[?25hRequirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->bert-extractive-summarizer) (0.17.0)\n","Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->bert-extractive-summarizer) (1.4.1)\n","Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->bert-extractive-summarizer) (1.18.5)\n","Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy->bert-extractive-summarizer) (0.4.1)\n","Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy->bert-extractive-summarizer) (2.0.3)\n","Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.6/dist-packages (from spacy->bert-extractive-summarizer) (1.0.0)\n","Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.6/dist-packages (from spacy->bert-extractive-summarizer) (4.41.1)\n","Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy->bert-extractive-summarizer) (7.4.0)\n","Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy->bert-extractive-summarizer) (2.23.0)\n","Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy->bert-extractive-summarizer) (1.0.2)\n","Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy->bert-extractive-summarizer) (0.8.0)\n","Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy->bert-extractive-summarizer) (3.0.2)\n","Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy->bert-extractive-summarizer) (1.0.2)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from spacy->bert-extractive-summarizer) (50.3.0)\n","Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy->bert-extractive-summarizer) (1.1.3)\n","Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from transformers->bert-extractive-summarizer) (3.12.4)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers->bert-extractive-summarizer) (2019.12.20)\n","Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers->bert-extractive-summarizer) (0.7)\n","Collecting sacremoses\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n","\u001b[K     |████████████████████████████████| 890kB 19.6MB/s \n","\u001b[?25hCollecting sentencepiece!=0.1.92\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/2d/6d4ca4bef9a67070fa1cac508606328329152b1df10bdf31fb6e4e727894/sentencepiece-0.1.94-cp36-cp36m-manylinux2014_x86_64.whl (1.1MB)\n","\u001b[K     |████████████████████████████████| 1.1MB 31.6MB/s \n","\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers->bert-extractive-summarizer) (3.0.12)\n","Collecting tokenizers==0.9.2\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7c/a5/78be1a55b2ac8d6a956f0a211d372726e2b1dd2666bb537fea9b03abd62c/tokenizers-0.9.2-cp36-cp36m-manylinux1_x86_64.whl (2.9MB)\n","\u001b[K     |████████████████████████████████| 2.9MB 41.4MB/s \n","\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers->bert-extractive-summarizer) (20.4)\n","Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy->bert-extractive-summarizer) (2.0.0)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy->bert-extractive-summarizer) (2.10)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy->bert-extractive-summarizer) (2020.6.20)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy->bert-extractive-summarizer) (1.24.3)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy->bert-extractive-summarizer) (3.0.4)\n","Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.6/dist-packages (from protobuf->transformers->bert-extractive-summarizer) (1.15.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers->bert-extractive-summarizer) (7.1.2)\n","Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers->bert-extractive-summarizer) (2.4.7)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy->bert-extractive-summarizer) (3.3.1)\n","Building wheels for collected packages: sacremoses\n","  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=cdf056f6f6fd2ee9d07bf1383ad66fabe9c2af2f22a0990da6eae247e51e5777\n","  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n","Successfully built sacremoses\n","Installing collected packages: sacremoses, sentencepiece, tokenizers, transformers, bert-extractive-summarizer\n","Successfully installed bert-extractive-summarizer-0.5.1 sacremoses-0.0.43 sentencepiece-0.1.94 tokenizers-0.9.2 transformers-3.4.0\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"a2q-UfW8gpRk","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1603924959144,"user_tz":240,"elapsed":88111,"user":{"displayName":"Shivika K Bisen","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgXYEJHMnxhEzLAhEnFo8Lhi33GVbNQAgVg14QwUQ=s64","userId":"00044211735934653548"}},"outputId":"b0275cc7-1e78-4f94-d0f1-e83bb153d7b6"},"source":["!pip install spacy==2.1.3\n","!pip install transformers==2.2.2\n","!pip install neuralcoref\n","!python -m spacy download en_core_web_md\n"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Collecting spacy==2.1.3\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/52/da/3a1c54694c2d2f40df82f38a19ae14c6eb24a5a1a0dae87205ebea7a84d8/spacy-2.1.3-cp36-cp36m-manylinux1_x86_64.whl (27.7MB)\n","\u001b[K     |████████████████████████████████| 27.7MB 111kB/s \n","\u001b[?25hRequirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from spacy==2.1.3) (1.18.5)\n","Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy==2.1.3) (1.0.2)\n","Collecting plac<1.0.0,>=0.9.6\n","  Downloading https://files.pythonhosted.org/packages/9e/9b/62c60d2f5bc135d2aa1d8c8a86aaf84edb719a59c7f11a4316259e61a298/plac-0.9.6-py2.py3-none-any.whl\n","Requirement already satisfied: jsonschema<3.0.0,>=2.6.0 in /usr/local/lib/python3.6/dist-packages (from spacy==2.1.3) (2.6.0)\n","Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy==2.1.3) (2.23.0)\n","Collecting preshed<2.1.0,>=2.0.1\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/20/93/f222fb957764a283203525ef20e62008675fd0a14ffff8cc1b1490147c63/preshed-2.0.1-cp36-cp36m-manylinux1_x86_64.whl (83kB)\n","\u001b[K     |████████████████████████████████| 92kB 14.1MB/s \n","\u001b[?25hCollecting thinc<7.1.0,>=7.0.2\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/18/a5/9ace20422e7bb1bdcad31832ea85c52a09900cd4a7ce711246bfb92206ba/thinc-7.0.8-cp36-cp36m-manylinux1_x86_64.whl (2.1MB)\n","\u001b[K     |████████████████████████████████| 2.1MB 50.9MB/s \n","\u001b[?25hRequirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy==2.1.3) (2.0.3)\n","Requirement already satisfied: wasabi<1.1.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from spacy==2.1.3) (0.8.0)\n","Requirement already satisfied: srsly<1.1.0,>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from spacy==2.1.3) (1.0.2)\n","Collecting blis<0.3.0,>=0.2.2\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/34/46/b1d0bb71d308e820ed30316c5f0a017cb5ef5f4324bcbc7da3cf9d3b075c/blis-0.2.4-cp36-cp36m-manylinux1_x86_64.whl (3.2MB)\n","\u001b[K     |████████████████████████████████| 3.2MB 52.7MB/s \n","\u001b[?25hRequirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy==2.1.3) (2020.6.20)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy==2.1.3) (1.24.3)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy==2.1.3) (2.10)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy==2.1.3) (3.0.4)\n","Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<7.1.0,>=7.0.2->spacy==2.1.3) (4.41.1)\n","\u001b[31mERROR: en-core-web-sm 2.2.5 has requirement spacy>=2.2.2, but you'll have spacy 2.1.3 which is incompatible.\u001b[0m\n","Installing collected packages: plac, preshed, blis, thinc, spacy\n","  Found existing installation: plac 1.1.3\n","    Uninstalling plac-1.1.3:\n","      Successfully uninstalled plac-1.1.3\n","  Found existing installation: preshed 3.0.2\n","    Uninstalling preshed-3.0.2:\n","      Successfully uninstalled preshed-3.0.2\n","  Found existing installation: blis 0.4.1\n","    Uninstalling blis-0.4.1:\n","      Successfully uninstalled blis-0.4.1\n","  Found existing installation: thinc 7.4.0\n","    Uninstalling thinc-7.4.0:\n","      Successfully uninstalled thinc-7.4.0\n","  Found existing installation: spacy 2.2.4\n","    Uninstalling spacy-2.2.4:\n","      Successfully uninstalled spacy-2.2.4\n","Successfully installed blis-0.2.4 plac-0.9.6 preshed-2.0.1 spacy-2.1.3 thinc-7.0.8\n","Collecting transformers==2.2.2\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/08/4a6768ca1a7a4fa37e5ee08077c5d02b8d83876bd36caa5fc24d98992ac2/transformers-2.2.2-py3-none-any.whl (387kB)\n","\u001b[K     |████████████████████████████████| 389kB 4.9MB/s \n","\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==2.2.2) (2.23.0)\n","Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from transformers==2.2.2) (2019.12.20)\n","Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers==2.2.2) (0.1.94)\n","Collecting boto3\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/28/76/1f853a1ff319c173c638f38c34ebb389389253bf828e18fc4de52a2f4288/boto3-1.16.7-py2.py3-none-any.whl (129kB)\n","\u001b[K     |████████████████████████████████| 133kB 20.0MB/s \n","\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==2.2.2) (1.18.5)\n","Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers==2.2.2) (0.0.43)\n","Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from transformers==2.2.2) (4.41.1)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.2.2) (2.10)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.2.2) (3.0.4)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.2.2) (1.24.3)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.2.2) (2020.6.20)\n","Collecting botocore<1.20.0,>=1.19.7\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/58/64883046f9c98d9f94cc81174d0b83e0be35b8f6c252e255b709b9024ef1/botocore-1.19.7-py2.py3-none-any.whl (6.7MB)\n","\u001b[K     |████████████████████████████████| 6.7MB 17.4MB/s \n","\u001b[?25hCollecting s3transfer<0.4.0,>=0.3.0\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/69/79/e6afb3d8b0b4e96cefbdc690f741d7dd24547ff1f94240c997a26fa908d3/s3transfer-0.3.3-py2.py3-none-any.whl (69kB)\n","\u001b[K     |████████████████████████████████| 71kB 11.9MB/s \n","\u001b[?25hCollecting jmespath<1.0.0,>=0.7.1\n","  Downloading https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl\n","Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.2.2) (7.1.2)\n","Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.2.2) (1.15.0)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.2.2) (0.17.0)\n","Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.20.0,>=1.19.7->boto3->transformers==2.2.2) (2.8.1)\n","\u001b[31mERROR: botocore 1.19.7 has requirement urllib3<1.26,>=1.25.4; python_version != \"3.4\", but you'll have urllib3 1.24.3 which is incompatible.\u001b[0m\n","Installing collected packages: jmespath, botocore, s3transfer, boto3, transformers\n","  Found existing installation: transformers 3.4.0\n","    Uninstalling transformers-3.4.0:\n","      Successfully uninstalled transformers-3.4.0\n","Successfully installed boto3-1.16.7 botocore-1.19.7 jmespath-0.10.0 s3transfer-0.3.3 transformers-2.2.2\n","Collecting neuralcoref\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ea/24/0ec7845a5b73b637aa691ff4d1b9b48f3a0f3369f4002a59ffd7a7462fdb/neuralcoref-4.0-cp36-cp36m-manylinux1_x86_64.whl (287kB)\n","\u001b[K     |████████████████████████████████| 296kB 4.3MB/s \n","\u001b[?25hRequirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from neuralcoref) (1.18.5)\n","Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from neuralcoref) (1.16.7)\n","Requirement already satisfied: spacy>=2.1.0 in /usr/local/lib/python3.6/dist-packages (from neuralcoref) (2.1.3)\n","Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from neuralcoref) (2.23.0)\n","Requirement already satisfied: botocore<1.20.0,>=1.19.7 in /usr/local/lib/python3.6/dist-packages (from boto3->neuralcoref) (1.19.7)\n","Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->neuralcoref) (0.3.3)\n","Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->neuralcoref) (0.10.0)\n","Requirement already satisfied: preshed<2.1.0,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.1.0->neuralcoref) (2.0.1)\n","Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.1.0->neuralcoref) (1.0.2)\n","Requirement already satisfied: srsly<1.1.0,>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.1.0->neuralcoref) (1.0.2)\n","Requirement already satisfied: plac<1.0.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.1.0->neuralcoref) (0.9.6)\n","Requirement already satisfied: wasabi<1.1.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.1.0->neuralcoref) (0.8.0)\n","Requirement already satisfied: thinc<7.1.0,>=7.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.1.0->neuralcoref) (7.0.8)\n","Requirement already satisfied: jsonschema<3.0.0,>=2.6.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.1.0->neuralcoref) (2.6.0)\n","Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.1.0->neuralcoref) (2.0.3)\n","Requirement already satisfied: blis<0.3.0,>=0.2.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.1.0->neuralcoref) (0.2.4)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->neuralcoref) (2020.6.20)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->neuralcoref) (1.24.3)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->neuralcoref) (3.0.4)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->neuralcoref) (2.10)\n","Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.20.0,>=1.19.7->boto3->neuralcoref) (2.8.1)\n","Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<7.1.0,>=7.0.2->spacy>=2.1.0->neuralcoref) (4.41.1)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.20.0,>=1.19.7->boto3->neuralcoref) (1.15.0)\n","Installing collected packages: neuralcoref\n","Successfully installed neuralcoref-4.0\n","Collecting en_core_web_md==2.1.0\n","\u001b[?25l  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.1.0/en_core_web_md-2.1.0.tar.gz (95.4MB)\n","\u001b[K     |████████████████████████████████| 95.4MB 1.1MB/s \n","\u001b[?25hBuilding wheels for collected packages: en-core-web-md\n","  Building wheel for en-core-web-md (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for en-core-web-md: filename=en_core_web_md-2.1.0-cp36-none-any.whl size=97126236 sha256=2f57431cfb495486f159946cc00576bf60c61f3d2958f4bd9608d38ae66a036b\n","  Stored in directory: /tmp/pip-ephem-wheel-cache-2njlnns6/wheels/c1/2c/5f/fd7f3ec336bf97b0809c86264d2831c5dfb00fc2e239d1bb01\n","Successfully built en-core-web-md\n","Installing collected packages: en-core-web-md\n","Successfully installed en-core-web-md-2.1.0\n","\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n","You can now load the model via spacy.load('en_core_web_md')\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"y8UA3Igtk2pZ","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1603924967217,"user_tz":240,"elapsed":2602,"user":{"displayName":"Shivika K Bisen","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgXYEJHMnxhEzLAhEnFo8Lhi33GVbNQAgVg14QwUQ=s64","userId":"00044211735934653548"}},"outputId":"38242f5b-60d7-40c1-8696-d13716cbd88a"},"source":["!pip install urllib3==1.25.5"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Collecting urllib3==1.25.5\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/81/b7/cef47224900ca67078ed6e2db51342796007433ad38329558f56a15255f5/urllib3-1.25.5-py2.py3-none-any.whl (125kB)\n","\r\u001b[K     |██▋                             | 10kB 24.0MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 20kB 2.9MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 30kB 3.9MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 40kB 3.0MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 51kB 3.6MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 61kB 4.0MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 71kB 4.3MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 81kB 4.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████▌        | 92kB 5.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 102kB 5.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 112kB 5.0MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 122kB 5.0MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 133kB 5.0MB/s \n","\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n","\u001b[?25hInstalling collected packages: urllib3\n","  Found existing installation: urllib3 1.24.3\n","    Uninstalling urllib3-1.24.3:\n","      Successfully uninstalled urllib3-1.24.3\n","Successfully installed urllib3-1.25.5\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"XbuxngsfgpMa","colab":{"base_uri":"https://localhost:8080/","height":200,"referenced_widgets":["562ba03c07194f618520cc022f5113df","ee0b64d4d0d64781973137410b127dc5","755d7b8ba5ae4e5090ff675bb8088870","8672ba887c804665b27412d21b92e3b9","73eb109574344bbbbe9cd0f0336bda1d","d20c77100fb04921b8fed034b1c10845","2c206c0bc48145349181e4a8f7e96e6b","26da5f4b0b1e48988f4900472656cb89","39465780c2674be0ac9b70760d35e203","88871ba0042e400a8e339b056ac0b02f","dc0cc46266a04e0591d9d6422059a08d","bb67796dfa46462cae9a39957d5cfdd0","31c0f460c3644f5986d2059f8ae49c6b","1bfc647df4424c27b57f75d25a8e1eb8","0bf3b26a51be48168957a46f785ee83f","0426389c060d4b2097f3d11e1528d81b","8957ac057b4c4ed4a9f53adecc91dd88","816be59fbd2c48689434af195b3bd762","3706e7d12d85421d8f41706d061f753a","ebfe4bab1aba4a7fbf8ede1870cfbe25","579acdfece344e1cb560f53dccd3eea4","f76c18ceca3d41169285f44fb5d626de","d91c7af9c8a6403ba37d18b51fc842d0","d55b42bd81a447c686c1528feef95b47"]},"executionInfo":{"status":"ok","timestamp":1603925035162,"user_tz":240,"elapsed":63371,"user":{"displayName":"Shivika K Bisen","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgXYEJHMnxhEzLAhEnFo8Lhi33GVbNQAgVg14QwUQ=s64","userId":"00044211735934653548"}},"outputId":"fefc902f-6a16-4b05-9304-120a481d0239"},"source":["from summarizer import Summarizer\n","\n","body = '''\n","The Chrysler Building, the famous art deco New York skyscraper, will be sold for a small fraction of its previous sales price.\n","The deal, first reported by The Real Deal, was for $150 million, according to a source familiar with the deal.\n","Mubadala, an Abu Dhabi investment fund, purchased 90% of the building for $800 million in 2008.\n","Real estate firm Tishman Speyer had owned the other 10%.\n","The buyer is RFR Holding, a New York real estate company.\n","Officials with Tishman and RFR did not immediately respond to a request for comments.\n","It's unclear when the deal will close.\n","The building sold fairly quickly after being publicly placed on the market only two months ago.\n","The sale was handled by CBRE Group.\n","The incentive to sell the building at such a huge loss was due to the soaring rent the owners pay to Cooper Union, a New York college, for the land under the building.\n","The rent is rising from $7.75 million last year to $32.5 million this year to $41 million in 2028.\n","Meantime, rents in the building itself are not rising nearly that fast.\n","While the building is an iconic landmark in the New York skyline, it is competing against newer office towers with large floor-to-ceiling windows and all the modern amenities.\n","Still the building is among the best known in the city, even to people who have never been to New York.\n","It is famous for its triangle-shaped, vaulted windows worked into the stylized crown, along with its distinctive eagle gargoyles near the top.\n","It has been featured prominently in many films, including Men in Black 3, Spider-Man, Armageddon, Two Weeks Notice and Independence Day.\n","The previous sale took place just before the 2008 financial meltdown led to a plunge in real estate prices.\n","Still there have been a number of high profile skyscrapers purchased for top dollar in recent years, including the Waldorf Astoria hotel, which Chinese firm Anbang Insurance purchased in 2016 for nearly $2 billion, and the Willis Tower in Chicago, which was formerly known as Sears Tower, once the world's tallest.\n","Blackstone Group (BX) bought it for $1.3 billion 2015.\n","The Chrysler Building was the headquarters of the American automaker until 1953, but it was named for and owned by Chrysler chief Walter Chrysler, not the company itself.\n","Walter Chrysler had set out to build the tallest building in the world, a competition at that time with another Manhattan skyscraper under construction at 40 Wall Street at the south end of Manhattan. He kept secret the plans for the spire that would grace the top of the building, building it inside the structure and out of view of the public until 40 Wall Street was complete.\n","Once the competitor could rise no higher, the spire of the Chrysler building was raised into view, giving it the title.\n","'''\n","\n","model = Summarizer()\n","result = model(body, min_length=60)\n","full = ''.join(result)\n","print(full)"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"562ba03c07194f618520cc022f5113df","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=434.0, style=ProgressStyle(description_…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"39465780c2674be0ac9b70760d35e203","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1344997306.0, style=ProgressStyle(descr…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"8957ac057b4c4ed4a9f53adecc91dd88","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","The Chrysler Building, the famous art deco New York skyscraper, will be sold for a small fraction of its previous sales price. The deal, first reported by The Real Deal, was for $150 million, according to a source familiar with the deal. The building sold fairly quickly after being publicly placed on the market only two months ago. The incentive to sell the building at such a huge loss was due to the soaring rent the owners pay to Cooper Union, a New York college, for the land under the building.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"7zOKE5HPJDJl","colab":{"base_uri":"https://localhost:8080/","height":162},"executionInfo":{"status":"error","timestamp":1603927396506,"user_tz":240,"elapsed":295,"user":{"displayName":"Shivika K Bisen","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgXYEJHMnxhEzLAhEnFo8Lhi33GVbNQAgVg14QwUQ=s64","userId":"00044211735934653548"}},"outputId":"3a19d066-4ef7-4dca-aae8-3194c1246553"},"source":["df=init_train.loc[:5]"],"execution_count":null,"outputs":[{"output_type":"error","ename":"NameError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-2-06f9f78a314b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minit_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;31mNameError\u001b[0m: name 'init_train' is not defined"]}]},{"cell_type":"code","metadata":{"id":"Ptayto4C-Bin","colab":{"base_uri":"https://localhost:8080/","height":225},"executionInfo":{"status":"ok","timestamp":1603925042224,"user_tz":240,"elapsed":254,"user":{"displayName":"Shivika K Bisen","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgXYEJHMnxhEzLAhEnFo8Lhi33GVbNQAgVg14QwUQ=s64","userId":"00044211735934653548"}},"outputId":"0f985216-8eba-4078-aec2-596946fc3d74"},"source":["df"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>ID</th>\n","      <th>Title</th>\n","      <th>Date</th>\n","      <th>Content</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>Microsoft Word - health-policy.doc</td>\n","      <td>2006-10-20T11:53:20Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1</td>\n","      <td>Health Promotion Glossary</td>\n","      <td>1998-01-06T11:10:08Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2</td>\n","      <td>The Use of Structured Observations in the Stud...</td>\n","      <td>2000-12-06T13:57:01Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>3</td>\n","      <td>RC_Health_message_Standard Operational_Procedu...</td>\n","      <td>2016-10-12T07:43:56Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>4</td>\n","      <td>Microsoft Word - JP7CIMCIProcessGuideFormat.doc</td>\n","      <td>2004-08-27T18:49:28Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>5</td>\n","      <td>null</td>\n","      <td>2012-12-04T08:08:25Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   ID  ...                                            Content\n","0   0  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","1   1  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","2   2  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","3   3  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","4   4  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","5   5  ...  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...\n","\n","[6 rows x 4 columns]"]},"metadata":{"tags":[]},"execution_count":12}]},{"cell_type":"code","metadata":{"id":"kSToclFQrdeC","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1603925305233,"user_tz":240,"elapsed":169395,"user":{"displayName":"Shivika K Bisen","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GgXYEJHMnxhEzLAhEnFo8Lhi33GVbNQAgVg14QwUQ=s64","userId":"00044211735934653548"}},"outputId":"08ed4768-a88a-4cf8-b331-1098c699e635"},"source":["from summarizer import Summarizer\n","import numpy as np\n","summary_list = []\n","model = Summarizer()\n","x = df['Content']\n","\n","for i in x:\n","  result = model(i, min_length=60, max_length=300)\n","  summary_list.append(result)\n","print(summary_list)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["['Activities that support health have always \\nbeen central to carrying out the humanitarian mandate of National Societies and their \\nInternational Federation. Scope  \\n \\nThis policy reaffirms the commitment of National Societies and their International \\nFederation to carrying out a broad scope of health activities for the most vulnerable, \\nthrough voluntary service. These include, but are not limited to, health promotion, \\nsocial support, primary health care, disease prevention, and provision of clean water \\nand sanitation facilities. In the absence of health, the full potential and \\ncapabilities of individuals as active, productive and responsible members of \\nsociety cannot be realized. • Health is not only an outcome of development but also a key contributor to \\nsocial cohesion and citizenship, as well as economic and social progress. This includes recognition of \\ncommunity capabilities in maintaining and developing their own health \\nthrough participatory approaches that increase health literacy and support \\nhealthy life styles. • Develop programmes that build social cohesion and strengthen the social \\nfabric and social support through volunteer action. • Identify, within the national and local context, the appropriate areas for the \\nRed Cross and Red Crescent interventions to meet the needs of the most \\nvulnerable. • Ensure that their health programmes consider and take into account the health \\npromotion, disease prevention, treatment and control protocols officially \\npromoted by the World Health Organization, as the world’s leading \\nintergovernmental health body. • Encourage their governments to implement the policies adopted by WHO, and to \\nparticipate in other major global health efforts and alliances.', 'Health Promotion Glossary\\n\\n\\nWorld Health Organization\\nGeneva\\n\\nHealth Promotion\\n○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○\\n\\nGlossary\\n\\nWHO/HPR/HEP/98.1\\nDistr.: This document is not issued to the general public and all rights are reserved by the World Health Organization (WHO). The document may not be reviewed, abstracted, quoted, reproduced, translated, in part or in whole, without the prior\\nwritten permission of WHO. Limited\\n\\nDesign: Marilyn Langfeld\\nPrinted in Switzerland on recycled paper, 4 000 copies. A pre-publication of the glossary was prepared as a resource\\ndocument for the Fourth International Conference on Health Promotion, New\\nPlayers for a New Era: Leading Health Promotion into the 21st Century, Jakarta,\\nIndonesia, 21-25 July 1997. The pre-publication was subsequently revised to\\naccount for the outcomes from that Conference, specifically the Jakarta\\nDeclaration on Leading Health Promotion into the 21st Century. Most notably, in October\\n1986 the First International Conference on Health Promotion was held in Ottawa, Canada,\\nproducing what is now widely known as the Ottawa Charter for Health Promotion. This was taken a step further at the Fourth International\\nConference on Health Promotion, New Players for a New Era: Leading Health Promotion into the\\n21st Century, which was held in Jakarta, Indonesia, in July 1997. Several WHO programmes and projects have been developed and implemented which have\\nsought to translate health promotion concepts and strategies into practical action. Recent developments in health systems around the world have given new prominence to health\\npromotion approaches. A number of terms that are\\ncentral to the Jakarta Declaration have therefore been included in this new version of the\\nhealth promotion glossary. Some terms have been\\nomitted, many have been modified in the light of experiences and evolution in concepts, and\\n19 new terms which are in current use have been included. Despite these obvious restrictions, the glossary has been assembled to enable as wide an\\naudience as possible to understand the basic ideas and concepts which are central to the\\ndevelopment of health promotion strategies and practical action. Don Nutbeam\\nWHO Collaborating Centre for Health Promotion\\nUniversity of Sydney, Australia\\nJanuary 1998\\n\\n\\n\\n○ ○ ○ ○ ○\\n\\n○ ○ ○ ○ ○\\n\\nvii\\n\\n○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○ ○\\n\\nNotes\\n\\nThe glossary consists of two sections. Earlier drafts of this revision to\\nthe glossary were circulated to all Regional Offices of WHO for comment and consultation. Many of the definitions are derived from the first version of the glossary published in 1986. This cross referencing is intended to improve\\nunderstanding of the inter-relationships between different terms and concepts. Within the context of health promotion, health has been considered less as an abstract\\nstate and more as a means to an end which can be expressed in functional terms as a\\nresource which permits people to lead an individually, socially and economically productive\\nlife. Health is a resource for everyday life, not the object of living. WHO, Geneva, 1986\\n\\nIn keeping with the concept of health as a fundamental human right, the Ottawa Charter\\nemphasises certain pre-requisites for health which include peace, adequate economic resources,\\nfood and shelter, and a stable eco-system and sustainable resource use. These links provide the key to an holistic\\nunderstanding of health which is central to the definition of health promotion. Health is regarded by WHO\\nas a fundamental human right, and correspondingly, all people should have access to basic\\nresources for health. See also social responsibility for health\\n\\nHealth promotion\\nHealth promotion is the process of enabling people to increase control over, and to improve\\ntheir health. The Ottawa Charter identifies three basic strategies for health promotion. The Health for All strategy is currently being re-\\ndeveloped to ensure its continued relevance into the next century. Such a distinction between the “old” and the “new” may not be necessary\\nin the future as the mainstream concept of public health develops and expands. Through health education with clients, and\\nadvocacy on behalf of their community, PHC personnel are well placed both to support\\nindividual needs and to influence the policies and programmes that affect the health of the\\ncommunity. The primary health care concept and themes are currently being reviewed by WHO. Disease prevention is sometimes used as a complementary term alongside health promotion. Reference: Report of the Inter-Agency Meeting on Advocacy Strategies for Health and\\nDevelopment: Development Communication in Action. Reference: new definition\\n\\nAlliance building will often involve some form of mediation between the different partners in\\nthe definition of goals and ethical ground rules, joint action areas, and agreement on the form\\nof cooperation which is reflected in the alliance. Determinants of health\\nThe range of personal, social, economic and environmental factors which determine the\\nhealth status of individuals or populations. Reference: new definition\\n\\nThe factors which influence health are multiple and interactive. In this sense health\\npromotion is directed at creating the conditions which offer a better chance of there being a\\nrelationship between the efforts of individuals and groups, and subsequent health outcomes\\nin the way described above. Social\\nepidemiology is the study of health and illness in populations which is informed by a\\nsocial, psychological, economic and public policy information, and uses that information\\nin the definition of public health problems and proposal of solutions. Equity in health means that people’s needs guide the\\ndistribution of opportunities for well-being. Inequalities in health status between individuals\\nand populations are inevitable consequences of genetic differences, of different social\\nand economic conditions, or a result of personal lifestyle choices. Health communication\\nHealth communication is a key strategy to inform the public about health concerns and\\nto maintain important health issues on the public agenda. WHO (AMRO/PAHO), Washington, 1996\\n\\nHealth communication is directed towards improving the health status of individuals and\\npopulations. It can take\\nmany forms from mass and multi media communications to traditional and culture-specific\\ncommunication such as story telling, puppet shows and songs. Reference: new definition\\n\\nHealth expectancy belongs to a new generation or type of health indicator which are\\ncurrently being developed. It can be used to reflect\\nthe relative advantage of one form of health intervention over another in producing\\nthe greatest health gain. Reference: new definition\\n\\nHealth goals are general statements of intent and aspiration, intended to reflect the values\\nof the community in general, and the health sector in particular, regarding a healthy society. WHO has\\nsupported the development, and promoted the use of health goals and targets at global\\nand regional, national and local levels. See also intermediate health outcomes, and health promotion outcomes. As with most policies, health policies arise from a systematic process of building support\\nfor public health action that draws upon available evidence, integrated with community\\npreferences, political realities and resource availability. Reference: based on Budapest Declaration on Health Promoting Hospitals. Report of a WHO Expert Committee on\\nComprehensive School Health Education and Promotion. ○ ○ ○ ○ ○\\n\\n○ ○ ○ ○ ○\\n\\n13\\n\\nReference: adapted from Glossary of Terms used in Health for All series N° 9. The healthy cities concept is evolving to encompass\\nother forms of settlement including healthy villages and municipalities. Reference: Adelaide Recommendations on Healthy Public Policy. Investment for health is a\\nstrategy for optimizing the health promoting impact of public policies. Infrastructure for health promotion\\nThose human and material resources, organizational and administrative structures,\\npolicies, regulations and incentives which facilitate an organized health promotion\\nresponse to public health issues and challenges. Investment for health\\nInvestment for health refers to resources which are explicitly dedicated to the\\nproduction of health and health gain. Investment for health\\nstrategies are based on knowledge about the determinants of health and seek to\\ngain political commitment to healthy public policies. Greater investment for health also implies reorientation of existing resource\\ndistribution within the health sector towards health promotion and disease prevention. It is a broad ranging concept,\\nincorporating in a complex way a person’s physical health, psychological state, level\\nof independence, social relationships, personal beliefs and relationship to salient\\nfeatures of the environment. The domains of\\nhealth and quality of life are complementary and overlapping. ○ ○ ○ ○ ○\\n\\n○ ○ ○ ○ ○\\n\\n18\\n\\nQuality of life reflects the perception of individuals that their needs are being satisfied\\nand that they are not being denied opportunities to achieve happiness and fulfilment,\\nregardless of physical health status, or social and economic conditions. Settings for health\\nThe place or social context in which people engage in daily activities in which\\nenvironmental, organizational and personal factors interact to affect health and\\nwellbeing. It refers to the processes between people which establish networks, norms, and social\\ntrust, and facilitate co-ordination and co- operation for mutual benefit. In such circumstances action to promote health might focus on support for re-\\nestablishing social networks. Reference: The Jakarta Declaration on Leading Health Promotion into the 21st Century. Reference: modified definition\\n\\nSocial support may include emotional support, information sharing and the provision\\nof material resources and services. Supportive environments for health\\nSupportive environments for health offer people protection from threats to health,\\nand enable people to expand their capabilities and develop self reliance in health. WHO, Geneva, 1997\\n\\nHuman beings are at the centre of sustainable development. WHO, 1948\\n\\nPrimary Health Care: Report of the International Conference on Primary Health Care. WHO, Geneva, 1978\\n\\nNew Approaches to Health Education in Primary Health Care: Report of WHO Expert Committee. WHO, Geneva, 1983\\n\\nGlossary of Terms used in Health for All series (N°9). WHO Regional Office for Europe, Copenhagen,\\n1991\\n\\nBadura, B. and Kickbusch, I. Health promotion research: Towards a new social epidemiology. WHO Regional Office for Europe, Copenhagen,1991\\n\\nLife skills education in schools (unpublished document 1991) WHO/MNH/PSF/93.7A. WHO, Geneva, 1993\\n\\nDhillon, HS. Berlin, Heidelberg, NewYork, London, Paris, Tokyo, HongKong, Barcelona, Budapest. WHO, Geneva,1996\\n\\nHealth and Environment in Sustainable Development. WHO, Geneva, 1997\\n\\nThe Jakarta Declaration on Leading Health Promotion into the 21st Century. For further information on relevant\\npublications refer to: Health Promotion Bibliography. WHO represents the culmination of efforts at international health cooperation that started\\nalmost 150 years ago. In 1851, the first International Sanitary Conference was initiated to\\ndiscuss measures against the importation of plague into Europe. The challenges WHO still faces are:\\n\\n• achieving health for all\\n• controlling old and new diseases\\n• achieving reproductive health for all\\n• building partnerships for health and\\n• promoting healthy lifestyles and environments.', 'The School\\'s wide-ranging research programmes\\nexamine ways of improving and promoting health and health services world-wide. iii\\n\\nTable of Contents\\n\\nGlossary iv\\n\\nPreface v\\n\\nIntroduction 1\\n\\nPart One: General Framework 3\\n1. The use of structured observations 7\\n\\nPart Two: Research Design 12\\n1. Generate list of potential \\'key\\' behaviours through\\n\\nliterature review and interviews with informants 19\\n3. Conduct data analysis and use study results for project\\n\\nplanning, implementation or evaluation 28\\n\\nBibliography 30\\n\\n\\n\\niv\\n\\nAppendices:\\n\\n1. A. Example of a spot check observation form\\nB. Example of a demonstration: asking and observing\\n\\n5. Example of a field guide for preliminary qualitative research\\n6. Flow chart to assist in selecting type of structured observation\\n8. Example of a precoded data sheet used for spot check observations\\n11. The final draft was reviewed by Astier Almedom, Robert Black, Ursula\\nBlumenthal, Elena Hurtado, Ellen Piwoz,  Anita Shankar, and  Jan Teun Visscher. We hope that you will find it useful for your work and encourage you to share\\nyour experience in due course. Health behaviour is what people do to maintain health\\nand/or overcome illness (Scrimshaw and Hutardo, 1987). These projects have also provided numerous examples of the importance of health\\nbehaviour studies for effective interventions. First we have to learn about people’s present behaviours and the\\nfactors that influence them. Health behaviour research is also useful for  project implementation. In addition, but not covered by these guidelines, health behaviour studies sometimes have as\\na main purpose exploring or investigating the links between behaviour and health. They can be broadly categorized\\ninto observation and interview methods, which can be applied both unstructured and\\nstructured. In participant observation, the observer shares the life and\\nactivities of the people for a few days, weeks, several months, or even years. Rating checks resemble spot checks but require the observer to make a\\njudgement about what is observed. Unstructured interviews\\nInterviews are unstructured when the persons being interviewed respond in their own words\\nor when we allow the interview to occur spontaneously, more like a conversation. They are guided by a list of questions that are asked\\nin the exact wording and order as they have been written down. The answers, however, are\\nstill open-ended, and the respondent is free to give his or her own words, thoughts and\\ninsights in answering the questions. Questionnaires usually contain a mix of open and closed questions. Secondary\\ndata, direct observation in the field, semi-structured interviews, analytical games, and the\\npreparation of diagrams all contribute to a progressively accurate analysis of the situation\\nunder investigation (McCracken et al., Then we must decide which features of these behaviours we want to measure. We may wish to observe whose hands are being\\nwashed, where, when, with what, and for how long. The second type of continuous monitoring is time-point observation. Continuous monitoring is more time consuming and complicated than the\\nother types of structured observations with respect to data collection, data analysis and\\nstatistical interpretation. Spot checks are a much quicker and easier method than continuous monitoring. Spot checks are not useful for direct\\nobservation of behaviours of limited frequency and duration, such as tooth brushing, since it\\n\\n9\\n\\n\\n\\nis unlikely that a person will be brushing his or her teeth at the precise moment a spot\\nobservation is carried out. If it is even difficult for one observer to make\\nconsistent judgements for similar observations, more difficult to maintain consistening\\namong several different observers. It will help when the\\nratings are specifically defined (see Part 3, step 8c). Advantages of structured observations\\nThere are several important advantages in using structured observations for health behaviour\\nstudies. Data obtained this way provides information about\\nactual behaviour. The disadvantages are especially related to continuous monitoring. How the research team is selected depends on local resources and the level of participation\\ndesired. However, this is not to say that representatives of all these groups should be part of the\\nresearch team itself; rather, a small active team should be selected and the others should\\nparticipate in decision-making processes where their involvement matters. Preliminary qualitative research\\nStructured observations should always be preceded by preliminary qualitative research. If a risky behaviour appears to be very rare amongst the group of study -\\nfor example if weaning food is seldom stored - it would not be an interesting behaviour to\\nstudy as it is not an important factor for the group. Preliminary qualitative research will\\nalso help to determine the most simple types of structured observations that will still\\nproduce useful results. Simple random sampling\\nIn a simple random sample every member of the population has an equal chance of being\\nselected. ( Clusters are more easily\\nsampled because it takes less effort to make a listing of all clusters than of all members of\\nthe population. If countries have to be selected, this is seldom done randomly for\\npractical or political reasons. Instead,\\noften a number of selection criteria are formulated in relation to the purpose of the study,\\nand based on these criteria, the required number of towns/villages with the right\\ncharacteristics is chosen. But when there are\\nvarious children under the age of five, again we have to make a choice, whether to include\\nthem all or not. b. How long should the observations take, and what is the best time of the day? Since observations are costly and  of long duration this may fatigue the\\nobserver. However, the preferences of the people\\nbeing observed and the availability of the observers should also be taken into account. When we know somebody is observing us, we tend to show our best side, or\\nbehave as we expect the observer would like it. Also, we may consider some behaviours too\\nsensitive to show and therefore wait until the observer has left. People may still act differently under observation, but this appears to be reduced through\\nuse of repeated observations, as the observed grow accustomed to the presence of the\\nobserver. It is important to actively involve the field\\nworkers in the development of the observation scheme. d. Detailed discussion on the role and tasks of the observers. g. Detailed discussion on how the observers should introduce themselves and establish\\nrapport in the community and the households. In addition, if the team is working together in one locale, it is\\nadvantageous to discuss the progress each night, as it will help to maintain motivation, and\\nadditional relevant information not captured by the observation forms may emerge in these\\ndiscussions. 18\\n\\n\\n\\nPart Three: Steps in the Use of Structured Observations\\n\\nThis part provides step-by-step information on the process of developing, pretesting,\\nimplementing and analysing structured observations in a behaviour study for a health\\nintervention project. The steps\\nresemble ordinary steps in any behavioural study, but with a purposeful bias towards\\nstructured observations. Decide if structured observations are needed and can be done considerin available\\nresources\\n1a. c. Decide if available resources are sufficient to do structured observations. Generate list of potential ’key’ behaviours through literature review and interviews with informants\\na. Review the literature and ’brainstorm’. Conduct qualitative research to identify which and how behaviours should be observed\\na. Develop a field guide. Refine list of key behaviours and determine the heterogeneity of the population\\na. Refine the list of behaviours to be observed in a structured format. c. Ascertain cultural, economic and religious heterogeneity of the study population. Identify who to observe, where and when\\na. Identify who (and what) to observe. b. Determine the mean number of observational episodes that can be conducted by one observer. Train observers and pretest instruments\\na. Involve observers in development of structured observations. Conduct data analysis and use study results for project planning, implementation or evaluation\\na. Do descriptive analysis. Perhaps weight can be given to behaviours that appear to be particularly\\nimportant, for example behaviours that were frequently cited. However, for many studies it will not be necessary to depend on\\nspecialist software packages. This is important to know as it may influence\\nthe sample methods and sample size and the type of structured observations to conduct\\nwithin the various groups. If\\nnot, we have to go through the steps again and see where we went wrong. Thus, if we are\\ninterested in child defecation practices we should focus on the child and its caretaker. Identify times that key behaviours occur\\n\\n22\\n\\n\\n\\nInsight into the duration, frequency and seasonality of the behaviours of interest is equally\\nimportant for deciding how to organize the structured observations. If the preliminary data\\nsuggest that a behaviour occurs infrequently or at odd times, this will influence the\\nobservation period required, or may require us to look for alternatives, such as spot checks\\nof physical clues or demonstrations. If key behaviours only occur during certain times of the\\nday, the structured observations should focus on those time periods. Use flow chart to select type(s) of structured observations\\nA more scientific way of deciding on the type(s) of structured observations that will best\\n’capture’ the behaviours of interest is by using the flow chart in Appendix 7. Select test sites\\nA reasonable number of sites should be chosen and repeated visits made to each site. An\\nobserver should not go to the same site on the same day of the week or time of the day. Estimate variation in key behaviours\\nThe use of a line graph may be a handy tool to get an indication of the variation in key\\nbehaviours, as shown in the sample figure in Appendix 8. Other\\naspects of the handwashing may also be important to record: Did the person wash his own\\nhands or were they washed by someone else? Both methods make use of a codebook that lists, defines and operationalizes all\\nkey behaviours together with codes for recording. An example of a pre-coded data sheet for structured observations is presented in  Appendix\\n10. Key behaviours are printed on the data collection sheet, so that indicating whether a\\nbehaviour has occurred involves little more than checking a box. Appendix 11 presents an example of an uncoded structured data sheet. Blank spaces are left\\nto be filled in by the observer with the appropriate behavioural codes as they occur. Determine the number of days for data collection\\nThe number of days available for data collection will depend largely on when the study\\nshould be completed and on project resources. Research that is meant to assist in project\\nplanning and implementation is likely to be needed quickly, whereas resources are usually\\nlimited and competing with other needs. Determine the number of observers required to conduct structured observations\\nWhen we know the number of days available for observation, and how many observations\\ncan be done per day, we can calculate the number of observers required based on the\\nselected sample size. Generally, fewer (two to five), more highly trained observers should\\nbe used for continuous monitoring observations. There are two basic steps in data cleaning: range checks and consistency checks. For example, if an item was supposed to be coded \"1=no, 2=yes, 8=not\\napplicable, 9=unknown\", the investigator should circle any occurrences of values other than\\n1, 2, 8 or 9 on the printout and check with the observation sheets to see what caused the\\nerror. Otherwise, the ’context’ (such\\nas related items), should be examined to determine what the correct code would be. Common types of scaling include: additive scales,\\nGuttman scale and factor analysis. One way to make the analysis an educational and productive experience is to call one or\\nmore workshop type of meetings and discuss the figures with the people who are expected\\nto work with the results of the study. For example: ’We found that around 90 percent of the\\nwomen and 62 percent of the men wash hands after defecation. Wider dissemination of the study results may be very useful for colleagues in similar\\nprojects or with similar interests. Maternal feeding behavior and child acceptance of food during diarrhea, convalescence, and health in the central\\nSierra of Peru\\'. The Hague, The\\nNetherlands, IRC International Water and Sanitation Centre, and London, UK, London School of Hygiene and Tropical Medicine. An educational intervention for altering water-sanitation behaviors to reduce childhood diarrhoea in\\nUrban Bangladesh I. Application of the case control method for development of an intervention\\'. Superordinate behavior in urban homes : a video analysis of request-compliance and food control behavior in two\\nblack and two white families living in New York City. ( Interventions for the control of diarrhoeal diseases among young children:  promotion of personal and domestic\\nhygiene\\'. Improving young child feeding during diarrhea :  a guide for investigators and program managers. Washington,\\nDC, USA, The Weaning Project, Manoff International. Understanding infant feeding practices : qualitative research methodologies used in the weaning project\\'. Boston, MA, USA, International Nutrition Foundation for Developing Countries, pp. Time allocation :  a tool for the study of cultural behavior\\'. Rapid Assessment Procedures (RAP) : to improve the household management of diarrhoea. Boston, MA,\\nUSA, Johns Hopkins University, International Nutrition Foundation for Developing Countries. Applied research for a dietary management of diarrhea program in Peru :  comparison of anthropological and\\nsurvey methods. Structured observations of hygiene behaviours in Burkina Faso\\'. In : Transactions of the Royal Society of Tropical Medicine and Hygiene,\\nno. An introduction to rapid rural appraisal for agricultural development. The role of massage in South Asia :  child health and development\\'. An educational intervention for altering water-sanitation behaviors to reduce childhood diarrhea in urban\\nBangladesh  II. Twenty-four-hour recall, knowledge-attitude-practice questionnaires, and direct observations of sanitary\\npractices :  a comparative study\\'. washing of eating/kitchen utensils\\n\\nE. Domestic and environmental hygiene\\n- household hygiene\\n\\n. safe disposal of animal faeces\\n\\nSource: Boot and Cairncross, 1993. The major objective of the study was to gather\\nindepth information on water, sanitation and hygiene practices which will enable the\\ndevelopment of an appropriate health education project. The faeces of sick and invalid people present\\nvery similar risks, as they are disposed in the same way. To\\nensure that the observations were collected in a systematic fashion, the observers were asked\\nto note observations for all entries on a highly structured data form. The rate of diarrhoea in children under six years of\\nage was 26 percent lower in the educational intervention areas than in the non-intervention\\nareas. 38\\n\\nAppendix 4:   A. Example of a spot check observation form\\n\\nFAMILY CODE:     . . . . . . . . . . . . . . . . . . Is the latrine clean (free of cleaning material and faeces)? _____ Is soap or ash for handwashing available _____\\n\\nSource: Brown et al, 1992. Se ora,  could you please show me how you usually _______\\nwash your hands? YES\\n\\n(YES) a) USES RUNNING WATER . . . . . . . . . . . . . . . a) _______\\nb) USES SOAP OR ASHES . . . . . . . . . . . . . . . c) _______\\n\\n________________________________________________________________________________\\n\\nInstructions from the codebook:\\n\\n37. To the extent possible, the extensionist should insist that the mother demonstrate her hand\\nwashing technique. For each item, there must\\nbe a code, as follows:\\n\\na) Uses running water\\n\\n0. NO Does not use running water (perhaps she places her hands in a container)\\n\\n1. YES Does use a clean rag to dry her hands or she air dries them. 40\\n\\nAppendix 5: Example of a field guide for preliminary qualitative research\\n\\nETHNOGRAPHIC FIELD GUIDE COLLECTION OF INFORMATION ON\\nFEEDING PRACTICES DURING DIARRHOEA\\n\\nINFORMATION TO BE COLLECTED\\n\\nThis guide covers the breadth of information that should result from interviews with key\\ninformants. The guide should be read through carefully several times before conducting the first interview\\nand may be helpful for the first several interviews. The sequence\\nthat is presented here need not be strictly followed. However, all the key elements should be\\ncovered in each topical area. Explain your programme association and that you are interested in\\nlearning about child feeding, health, and illness. GENERAL ILLNESS TAXONOMIES\\n\\nBegin by asking, \\'Can you give me the names of all the kinds of illness people have around\\nhere?\\' For this primarily taxonomic exercise, there is no need to ask about causes or\\ntreatments. With this list, repeat the exercise used for general\\nillnesses. Don\\'t do this for all illnesses, as it\\nwould require too much time. The objective is to be able to make some distinctions between\\ndiarrhoea and other illnesses. Ask if any one type of diarrhoea is more serious than another. Here probe for cognitive\\nbeliefs about a developmental sequence for a diarrhoea episode. Get good descriptions of the\\nsymptoms and physical characteristics of each stage. Refer to the causes that have been given for each diarrhoea type. Find out when during an\\nepisode a particular treatment should be done, and why. WOMEN\\'S WORK/CHILD CARE\\n\\nFind out what kind of work women do during the day, and how this changes throughout the\\nyear. FOOD PRODUCTION/AVAILABILITY/CONSUMPTION\\n\\nAsk what foods are currently being grown. Within a household, do all family members eat these foods -  how is\\nthe food that is grown distributed? Which foods are obtained not from food production or the market, but from\\nalternative sources (e.g. food aid, exchange/barter). For example, an increase in the number of stools may result in the mother (or other\\ncaretaker) making a change in feeding patterns, or the child may be taken to the doctor. Often, the perceived cause may lead to clues about intervention barriers. It may not be seen as serious, but\\nrather a positive benchmark of growth. One way of finding out what people do is to ask about the last diarrhoea\\nepisode. In order to contrast differences in feeding\\npatterns during normal or healthy times compared to when the child is having diarrhoea,\\nnormal infant and child feeding patterns must be understood: When should a child be weaned? Foods that are considered\\n\\n\\n\\n44\\n\\nuseful and harmful during diarrhoea should be listed, as well as  the reasons for these beliefs. Fieldwork should be\\ncarried out for about two weeks in each site. Without it, there is little chance of conducting free-flowing interviews over an\\nextended period. If it is possible to live directly in the village or where the preliminary ethnography is to be\\ndone, this is optimal. Decisions about who to interview should be based upon one important principle:\\ncapture the variation. On the first day of ethnography\\nin this village, some quick demographic surveillance should be done to \\'map\\' or stratify the\\nvillage into important divisions. Lengthy, repeated interviews should be\\ndone with key informant, such as midwives, village health guides or mothers who have\\nseveral young children. In many cultural settings, group discussions (focus group interviews) provide valuable\\ninformation. When conducting interviews, it is important to cover the same topics for each interview. 46\\n\\nOne of the most important tasks is to look for the variation. At 6:41, Bodhi K. returns with kindling and began preparing potato taarkhari (a vegetable\\nstew). Dhami gives one sixth of her\\nroti to her granddaughter Khuma K.. She then serves Suk Lal a small roti. Kul B. serves\\nhimself a banana, as does Bodhi K.. Dhami then begins preparing the rice. The fourth question, \\'Is the study sample greater than 100 households?\\' 51\\n\\nAppendix 9: General suggestions regarding the appropriate construction of data\\ncollection forms\\n\\nData collection forms should be designed to minimize the decision making of the field worker\\nand to ease the task of data entry. 2) Whenever possible, allow for the recording of simultaneous codes. By being consistent, you will help field\\nworkers make fewer recording errors and also make data cleaning go more smoothly. Village /Ward:  ______________ DATE A: ________ TIME A: ________\\n\\n2. Neighbourhood:  ______________ DATE B: ________ TIME B: ________\\n\\n3. Observer:   ______________ Observation No.: ___________________ Households larger than three\\nmembers will require additional forms. If you are able to determine the activity of the household member through direct\\nobservation, write your initials in the first row of boxes (Observer/Informant) under the\\nappropriate column (A, B, C or D depending on observation number). Record the ID number of the\\nindividual who gave you the information in the appropriate Observer/Informant row-column. 54\\n\\nAppendix 11: Example of an uncoded, but structured data sheet used for\\ncontinuous monitoring observations\\n\\nForm 15: Direct Observation of Nepali Meals Date: _______________   Page:  _______\\n\\n1. The key behaviour itself should be recorded under the\\nACTIVITY CODE column with codes drawn from the codebook. Record conversations of interest, especially if food-related. GENERAL CLEANLINESS RATINGS:\\n\\nDefinitions:\\n\\n1 = Clean, no visible stains or dirt on object/area observed, no dried food or other organic\\nmaterial. 2 = Moderate, visible stains or dirt on object/area observed, possibly some dried food or\\nother organic material. observational episodes required per household)\\n\\nObservers = _________________________________________________________\\nRequired\\n\\n  (D. No. Pretesting, training and supervision\\n\\n\\tPart III Steps in the Use of Structured Observations\\n\\t1.', '', \"xiii \\nWhy Was It Written? ........................................................................................... xiii \\nWho Is the Audience?.......................................................................................... 35 \\nWhy Is It Done? ......................................................................................... Currently a Training Advisor for The Manoff Group, she provides \\ntechnical assistance in training, environmental health, and other child survival areas \\nin Africa, Asia and Central America. While the logic of the process is the same regardless of the health issue, \\nthe technical and behavioral issues differ, so it may help program designers and \\nmanagers to have guidance on applying this approach to particular areas of interest. These small-scale experiences, \\nimplemented by private voluntary organizations (PVOs) in collaboration with \\nministries of health, provide some of the examples of materials and instruments \\ndescribed in this guide. This guide is intended for health planners and managers in governmental, non-\\ngovernmental and international organizations working at the district, provincial, \\nregional or national levels in developing countries, as well as for organizations that \\nsupport programs at those levels. xv\\n\\nGetting \\nOrganized \\n\\nMonitoring \\nand \\nEvaluation \\n\\nPlanning \\nand Imple-\\nmentation \\n \\n\\nStrategy  \\nFormulation \\n\\nFormative \\nResearch \\n\\nSituational \\nAssessment \\n\\nHow Is the Guide Organized? After an introduction on diarrheal disease and hygiene promotion (and their broader \\nchild health context) and a synopsis of BCP, the guide follows the chronological steps \\nin a methodology designed to plan and implement program activities. It can provide hygiene promotion training and support materials for \\nprivate and public health personnel and community-based volunteers. In this listing, Key Family Practice 6 addresses the hygiene behaviors that most \\ndirectly affect the prevalence of childhood diarrhea. Key Practices 1, 2, and 9 also \\ncontribute to reducing deaths due to diarrhea/dehydration (but are not a focus of this \\nmanual). Although planners should look for \\nopportunities to address multiple behaviors, it is wise to guard against overloading \\nmothers, community volunteers, and others with too much information and too many \\nsuggested tasks at the same time. 5\\n\\n \\nKey Family Practices \\n\\nFor physical growth and mental development \\n\\n1. Take appropriate actions to prevent and manage child injuries and accidents. Prevent child abuse and neglect, and take appropriate action when it has occurred. Ensure that men actively participate in providing childcare, and are involved in the \\nreproductive health of the family. Recognize when sick children need treatment outside the home and seek care from \\nappropriate providers. In the past, some programs have simply introduced new services or technologies to \\naddress a health problem. However, even adding an educational component does not ensure that the \\nstrategy to address the health problem is correct, nor that the education about a \\ntechnological intervention will ensure its use. The essence of the \\nstrategy is providing support for those feasible behaviors. 9\\n\\nThe following table lists the most common determinants of health behaviors. ( Behavior Change Myths and Realities \\nMyths Realities \\n\\nBehavior change takes a long time. Most behaviors have to be reinforced and sustained \\nover time, at least until they become social norms. Knowledge alone is rarely enough to change behavior; \\nmotivation and enabling factors must also be addressed. People may be much more likely to wash their hands \\nwhen it is perceived as socially desirable or for many \\nother reasons. Well-trained and properly supported community \\nvolunteers or animators are often the most effective \\nchannels for change communications. People must know how to do a behavior correctly – as well as when, and where to do \\nit. For example, virtually all people who smoke or engage in unsafe sex know that these \\nbehaviors are potentially harmful, but clearly their behavior is not consistent with their knowledge. For example: \\n\\nIn a setting where a program seeks to reduce the incidence of diarrheal disease, research may \\nreveal that mothers are willing to chlorinate water but have a hard time figuring out how much \\nchlorine to use for their particular water-storage container. Steps to develop program \\nrecommendations \\n\\nHow to do each step When to do \\neach step \\n\\n1. Break down each ideal behavior into essential \\nactions (sub-behaviors). Learn about current practices and which sub-\\nbehaviors are least practiced and why. Subsequent chapters provide substantial detail \\non each step. Select the essential hygiene practices that your program intends to promote. Other essential family practices (e.g., related to breastfeeding, \\ncomplementary feeding and measles immunization) that contribute to preventing \\ndiarrhea will not be covered in detail in this guide. • Always cover water-storage \\ncontainers or use narrow neck \\ncontainers. • Always use a different (clean) \\nvessel or dipper to transfer \\nwater for drinking (or use a \\ncontainer with a tap). • Clean the water storage \\ncontainer twice a week with \\nsoap. Always safely dispose \\nof feces, especially of \\nchildren under 36 \\nmonths of age. • Have young children always \\nuse a potty, washable diapers \\nor disposable diapers. • Always wash the potty or \\nwashable diapers with soap \\nand ensure that the \\nwastewater from washing ends \\nup in a sanitation facility. • Always use correct \\nhandwashing technique: rub at \\nleast three times, especially \\nfingers; use a cleansing agent; \\nuse dripping or running water if \\npossible; air dry or wipe on \\nclean cloth. example of Tippy Tap in Chapter \\n8, Technologies or Products). • In order of preference, people \\nshould use manufactured soap, \\nhomemade soap, ashes or clean \\nsand, or appropriate local plants \\nas cleansing agents. It may be \\nnecessary to start with \\nnegotiating proper washing after \\ndefecation and then add other \\ncritical moments to wash. • Never consume animal \\nproducts that have been \\nimproperly stored or \\ninsufficiently prepared for \\nconsumption. They do this after defecating and using the latrine and before eating. Parents make sure \\nthat their children wash their hands or help them do so. • Review existing information on current practices from previous research and \\nprograms \\n\\n• Gather new information on current practices from participant groups in the \\nprogram area \\n\\nIdentifying and focusing on specific gaps allows program planners to address the \\nneed for specific improvements. Mothers who \\ndo not even rinse their hands after cleaning a child would have to be addressed \\ndifferently. For each gap discovered, identify and test POSSIBLE SOLUTIONS. After all, most people practice their \\nbehaviors for reasons that make sense to them. Again, feasible solutions \\nare those where most people are both willing and able to take actions in their actual \\nsituations, if the program can effectively motivate and facilitate these actions. The behaviors and essential actions that will make a difference to the health \\nproblem and that people are both willing and able to do become the \\nPROGRAM RECOMMENDATIONS. Program planners \\nuse information about barriers and motivations or other supports for each feasible \\nbehavior to select sets of activities that will directly affect the feasible behaviors. When possible, communities have direct input into \\ncharacters and icons used and the visual context of the materials. • Baseline data are collected after formative research and a focus on the specific behaviors and \\nmajor barriers are identified. • There is follow-up at key points during the implementation process to assess and adjust the \\nimplementation as needed. • In-depth, focused monitoring occurs several times during implementation and findings are used \\n\\nto modify program activities. 19\\n\\nGetting \\nOrganized \\n\\nMonitoring \\nand \\nEvaluation \\n\\nPlanning \\nand Imple-\\nmentation \\n \\n\\nStrategy  \\nFormulation \\n\\nFormative \\nResearch \\n\\nSituational \\nAssessment \\n\\n3. Partners bring a variety of skills and resources, are able to \\nreach different groups in a community and offer a greater likelihood of sustainability. Meet with the Project Team and Partners \\nBring the project team and partners together to introduce plans for the new activity. 22 \\n\\n• Inquire about partners’ interests in participating in research, strategy formulation \\nand implementation. • Describe the organization of the management team and the responsibilities of \\neach member. Consider the following ideas to nurture good relationships with partners: \\n\\n• Update partners periodically. Use a wide \\nrange of sources to get the information to make decisions on next steps. This review \\ncan not only provide key information for planning, but also reduce the amount of new \\nresearch needed by helping focus it on unanswered questions. Activities related to the review are summarized in the Task Box. Hygiene practices, handwashing practices, disposal of feces, water \\ncollection and handling, water use and treatment. Surveys; qualitative and quantitative studies; national and regional data; \\nlocal experts; program documents. Review published and unpublished documents that provide information on \\nhygiene improvement related to diarrheal disease in the program areas. Note what is known about practices, problems, motivations, and constraints \\nin the appropriate columns. Include summaries of existing information, gaps identified in the \\nworksheets, guidelines on what issues the consultative research needs to \\naddress. For hygiene improvement, young children and their caretakers are the chief \\nbeneficiaries. Diarrheal-disease prevention usually targets low-income families. If not, is your program willing to create and \\nsupport such a group? Demographic and Health Surveys \\nare good sources of up-to-date demographic information and usually include data on \\nhygiene-related practices. National environmental health surveys may have been \\nconducted in some countries, or hygiene data may have been collected as part of other \\nlarge surveys. This is especially important if \\nmore than one person conducts the review. Review the worksheets and the report \\noutline included in later sections of this chapter for ideas on how to organize review \\nnotes. Conduct Key Informant Interviews \\n\\nInterview knowledgeable individuals to supplement and explain the information that \\nis gathered during the literature review. Key informants also can report on “common \\nknowledge” – i.e., issues and practices that may not be written down. Keep \\ndetailed notes of the interviews or tape record them (with permission) and write notes \\nlater. Analyze and Write up the Review \\n\\nSummarize by Theme or Question \\n\\nThe review is most useful if it is well organized and brief. • What are the most important causes of diarrhea in young children? • To what extent are communities themselves managing their water systems? • What have been the strategies, experiences and effectiveness of previous programs \\n(governmental and NGO) to improve hygiene behaviors? • What individuals, services and media might influence hygiene practices? • What partners could be included in program design, implementation and monitoring – including \\n\\ngovernmental and non-governmental organizations as well as community partners (groups and \\nleaders)? • What are current beliefs about causes of diarrhea, concepts of cleanliness, water, feces, etc., • What population groups seem to differ greatly in hygiene practices, how and why? In the Peru pilot hygiene project, \\n\\nspeaking Spanish or the indigenous language differentiates groups in the Cusco area). • Who is responsible for obtaining, handling and using water? • Do government workers or community health volunteers provide counseling? Worksheet: Situational Assessment – Practices \\n\\nEssential Hygiene Practice: \\n____________________________________________________  \\n\\nEXISTING INFORMATION ON PRACTICES ANALYSIS \\n\\nSub-Behaviors \\n\\n \\n\\nCurrent Practices \\n\\n \\n\\nIs This Sub-Behavior \\nPracticed? Formative Research \\n\\n \\n\\nChapters 5 and 6 provide guidance on planning and managing formative research. The purpose of formative research is to provide a solid basis for effective program \\nactions. Any formative research – even “quick and dirty” research – is usually \\nbetter than none. In some cases, one \\nmay skip the “exploratory” research phase and move directly from reviewing existing \\ninformation into behavioral trials (TIPs). Checking research is optional, and if carried \\nout, should not take more than a few days. There are advantages to having health staff conduct fieldwork. And second, their presence can introduce bias into the \\nresults because respondents may be more likely to give “correct” rather than honest \\nanswers, and if not well coached in advance, health workers may react positively or \\nnegatively to the answers given. While men and women are \\n\\npotential team candidates, women may be more at ease when talking with women about these \\nissues.) These considerations will help to determine how extensive \\nthe plan will be. Programs staff should draft a budget for the planned research and \\nmake adjustments in plans if necessary (see sample budget in Chapter 6). − “What relationship, if any, do people (i.e., mothers, fathers, other caregivers \\nof young children) see between feces and disease?” And you have just said that child feces can be dirty and can contain \\ndisease. If you help a child clean himself after he has defecated and your hands \\nlook clean, do you believe they are in fact clean?” If more insights are needed on \\nthese issues, carry out a few focus group discussions to clarify. As the research proceeds, the team may find that it needs to change the information in \\nthe planning table. Based on a review of existing information, the team should determine the types of \\npeople of interest (participant groups). These concepts may change based on information gained from the initial information-gathering. For research on preventing diarrheal disease in a particular context, the primary \\ngroups might be mothers and older female children who have child-care and cooking \\nresponsibilities (this will vary by program situation). Secondary groups might include \\nfathers, grandmothers, community volunteers and community leaders. Some of the \\nparticipant groups will need to be sub-divided into sub-categories, such as mothers of \\nchildren of certain ages, those from families with or without a latrine, or persons from \\ndifferent ethnic or religious groups (if such groups may have different hygiene \\npractices). Participants should be sampled from various communities to avoid findings that are unduly \\ninfluenced by a few communities that may have unusual characteristics. Although qualitative methods are emphasized in \\nformative research, opportunities to quantify something that may prove meaningful \\nshould not be missed. The interviewer uses an interview guide, \\nwhile still allowing, even encouraging, the respondent to go beyond the programmed \\ntopics and order of questions. However, planners must be sensitive to respondents’ needs and levels of comfort. 48 \\n\\nObservations \\n\\nEven if an interviewer develops a good rapport with a respondent, the reliability of \\ninterview responses always must be assessed. Only observations (or follow-up \\nquestioning) may reveal this. Examine where people say they wash their hands and \\nthe containers, water, and cleansing material they say they use. 49\\n\\nHighlights from Observations for a School Health Program in Malawi \\n\\nObservations in homes and communities: \\n\\n1. Families usually gather drinking water from the closest fresh-tasting source, whether a river, lake \\nor well. All of the family members use a single plastic cup to retrieve water and drink. No toilets have materials for students to use to clean themselves after defecating. Early in formative research, FGDs are most useful for exploring social norms, broad \\nconcepts, and the range of behaviors in a community, rather than focusing on \\nindividual practices. The team may find it helpful to use the “Task Box for In-depth Interviews and Group \\nDiscussions” in Chapter 6 to support their planning and implementation of focus \\ngroup discussions. Actual hygiene behavior (handwashing, disposal of \\nfeces and handling of water) especially if you believe \\nreported details may not be accurate \\n\\nConduct observations in conjunction \\nwith interviews. Most likely paths of transmission Conduct in-depth observations and \\ninterviews in families with high diarrhea \\nprevalence. It usually includes in-\\ndepth interviews and observations and, sometimes, FGDs. Task Box for In-Depth Interviews & Group Discussions \\nPreparation Tasks \\n1. Tape \\nrecord verbal interchange and \\ntake notes about non-verbal \\ndata if possible. To avoid influencing responses, do \\nnot talk specifically about hygiene. There are normally a few background \\nquestions that facilitate analysis and interpretation; for example, the ages of \\nchildren in a household, the mother’s level of education, and the family’s socio-\\neconomic circumstances (as indicated by occupation, land or animals owned) and \\nethnic group. Interviewers should \\nask for elaboration and explanation of any new topics or relevant issues that \\nsurface during the conversations. This is precisely how qualitative research differs \\nfrom survey or quantitative research. The key to successful qualitative \\nassessments lies in training interviewers to recognize relevant issues and to \\nencourage participants to discuss them openly. Once the research guides have been developed, the team should test each guide with \\nseveral people from the participant group. should \\nto be planned, along with logistical details such as transportation and \\naccommodations. Prior to field visits, the team should contact community leaders in \\nwriting or in person, as appropriate. They may \\nneed to create new questions to pursue new motivations or barriers or even new \\ninfluential participant groups. Analyze Results  \\n\\nInterview results include summaries, tabulations and any particularly insightful \\nverbatim answers. 58 \\n\\nBarriers to Improved Hygiene Practices in Nicaragua \\n\\nFormative research in Nicaragua identified a series of barriers to improving hygiene practices. Each page is coded with the selection and other criteria, such as the age of the child, \\narea of residence, and other selection criteria. Initial \\nresearch questions guide the direction of the analysis. After sorting the information, they will create \\nsummary tabulations for important pieces of information. For additional details about sorting and summarizing the data gathered, see Chapter 4 \\nof Designing by Dialogue (Dickin, Griffiths, and Piwoz) on the CD-ROM. The moderator’s guide can be written as a list of specific questions. To make the flow of the focus group discussion more \\nnatural, the sequence of the topics in the guide should move from the general to the \\nspecific. The draft report is then incorporated into the final research report. Detailed editing and printing are not required at this point. Existing latrines were all in use and were shared with neighbors. Despite the poverty of most households, residents made widespread \\nuse of disposable diapers. There was no garbage collection service, and some households were in \\nthe habit of burning their trash. The interviewer then returns to the home to assess \\nthe outcome of the trial. ( Although the quality of counseling and \\nnegotiation in TIPs may be higher than can be expected in the actual program, the \\ntrial results normally give planners a good (although not precise) idea of the \\nproportion of people who will adopt the new behaviors and of their potential health \\nimpact. Draft a counseling guide on \\n\\nbehavior-change \\nrecommendations. Determine the sampling frame and procedures for each \\nhousehold visit. Identify hygiene-practice problems and plan recommendations to \\nsuggest in each household. Preliminary analysis: what recommendations are mothers \\nwilling/not willing to try and why? Summarize hygiene problems, practices recommended, \\npractices accepted, practices tried, practices carried out through \\nthe trial period, any intentions to continue. At the next stage (strategy \\nformulation), the research methods, findings and program implications should be \\nshared to kick off a workshop designed to draft or review and modify the program \\nstrategy (see next chapter). 68 \\n \\n\\nSample In-Depth Interview Discussion Guide (on Key \\nHygiene Behaviors for Preventing Diarrhea): Mothers \\nof Children Under 5 Years \\n\\n \\n\\n[To save space, this guide is not formatted with appropriate spaces to record all of the \\ninformation.] If this person \\nis not eligible, thank her and end the interview.] • We will return in a few weeks to tell your community what everyone said in these \\n\\ndiscussions. Your name will not be used, and no one will be told about what any \\none person says. [If she declines to interview, ask \\nwhy, try briefly to convince but do not pressure. Types of diarrhea Cause How to prevent \\n   \\n   \\n   \\n   \\n \\n11. Yes ___  No ___ \\n \\n\\n• [If no, ask] what different types are there and what should each be used for? If not mentioned, ask \\n\\nif people around here use boiling, chlorinating, or filtering.] Where does the water come from that your family drinks at home? Would you pay to have more water available in or very near your home? Note type of container, if covered, type of cup or ladle \\n\\nto retrieve water] \\n• How do you retrieve the water? [Note if mother uses water and from where, uses soap or a \\nsubstitute, rubs hands together at least 3 times, dries hands and how. Also look for a \\nregular place for hand washing, presence of a washing basin or Tippy Tap, water \\ndrainage, cleanliness of material used for drying.] Location   \\nWater from where  \\nCleansing agent  \\nNumber of times rub hands \\ntogether \\n\\n \\n\\nHow dry  \\nWater drainage  \\n \\n32. Sources of Information Briefly Describe Most \\nOften \\n\\nSometimes Most \\nUseful \\n\\nFamily or friends     \\nCHW     \\nTraditional healer     \\nFacility-based health \\nworker \\n\\n    \\n\\nRadio/TV     \\nMeetings/clubs/community \\nevents \\n\\n    \\n\\nDrama, songs     \\nPrint material     \\nOther     \\n\\n \\n40. Are there any other ways that you think would be good for giving people such \\n\\ninformation? Observe]   \\n \\n\\n• [If yes, ask]  What do you think attracts flies? Do some particular parts of the body get \\ndirtier than others? Do you think all water is the same, or are \\nthere different types of water? With an infant’s feces, a 2 year old’s, a 5 year old’s \\n\\nLatrines \\n[assume \\nthese \\nmothers do \\nnot have \\nlatrines; use \\ndifferent \\nquestions for \\na group of \\nmothers that \\ndo] \\n\\nSome [a few] families around here always \\nseem to put feces in a latrine. Causes of \\nchild illness \\n\\nWhat are the main illnesses that kids \\naround here get? Prevention/ \\nfatalism \\n\\nCan families like yours prevent these \\nillnesses in children? 77 \\n \\n\\nWorksheet:  TIPs Analysis  \\n\\n \\nRecommended \\n\\nPractices \\nNo. The next \\nstep is to use the key insights from the situational analysis and formative research as \\nthe basis for designing a program strategy. You can use as few as two or \\nthree components or as many as six or seven – whatever is needed based on your \\nanalysis. Current behaviors are learned from the situational analysis and formative \\nresearch. Barriers include both internal (attitudes, knowledge) and external (environmental) \\nconstraints to moving from the current practices to the feasible practices. TIPs are aimed at providing \\ninformation on how to overcome these barriers. Saving time or money, enabling children to play more or do better in \\nschool, increasing one’s social status, and other perceived benefits may be as or \\neven more important than reducing diarrhea in children. The types \\nof activities most likely to affect hygiene behavior change are communication, \\n\\n\\n\\n \\n\\n84 \\n \\n\\ntraining, community mobilization, facilitating new technologies or products, and \\npolicy change. Existing supportive activities should be \\nlisted under motivations and supports. Nevertheless, such actions should \\nbe noted so that the entire team can understand factors that might limit behavior \\nchange despite planned efforts. Do people need practical information about what to do or \\nhow, motivation and encouragement to do it, new skills, help in remembering what \\nneeds to be done or how? For \\nexample, interpersonal communication, ideally including home visits, is highly \\ndesirable. Print materials, especially aids to \\nsupport counseling by health workers and volunteers, and reminder materials for \\npeople at home, are often important. Such efforts may be grouped under communications \\nactivities or may be shown in a separate strategy column. Whenever possible, try to coordinate with existing in-service and pre-\\nservice training programs. Programs also may decide to facilitate the education of a concerned group of \\ncommunity members in identifying problems and taking appropriate courses of \\naction. Although community revolving \\nfunds in general have been difficult to establish and sustain, such funds can represent \\na viable strategy, assuming careful planning, monitoring and ongoing support (see \\nbox). For Peru, this element is crucial to the process, as \\nvirtually none of the families was using the plastic water tanks, children's potties, chlorine droppers or \\ncovered wastepaper baskets. Communication activities \\nhave the potential for doing many things to support essential hygiene behaviors, \\nfeasible sub-behaviors, and even policy change. The content of BCC is not what planners’ explanations of the reasons for \\ndifferent behaviors, but rather the information that formative research, including trials \\nhas shown to be essential to support behavior change. To complete the communication strategy, planners must identify the communication \\ntasks, specific audiences (participant groups), communication channels, key messages \\nand information, and materials. Although simply transmitting information \\n(“messages”) may be needed, at its core, behavior-change is a problem-solving \\nprocess for both individuals and groups. Individual counseling should not only suggest small improvements in behaviors but \\nalso, to the extent necessary, offers people a choice of improved behaviors and how to \\naccomplish them. The cards should have a \\nlogical organization to facilitate discussions between the counselor and the families \\nabout information that is relevant to them. The full set of counseling materials from three pilot projects are found on the \\nCD-ROM that accompanies this guide. ( However, where mothers and fathers have limited literacy, materials \\nwith too many words are not likely to be understood by many. Examples include \\ntraditional storytellers, singers or musicians. Also, seek organizational resources, including \\nsocial, professional and economic associations, such as rural water cooperatives, to support \\nyour communications efforts. • Size of audience: It may prove difficult to provide individual counseling if the audience is large \\nand budgets are limited. The “wall life” of posters placed on outside walls is often very limited. Radio spots may use a dialogue format that features a \\ncharacter specifically developed and promoted as “a voice of wisdom” on good \\nhygiene practices. Prepare work plans and finalize budgets for each strategy component. Work Plans for Each Strategy Component \\nThe behavior change strategy focuses on creating a logical flow of program activities \\nfor achieving behavioral and health objectives. The next step is to develop a work \\nplan with implementation steps for each component. • When each activity will start, how long it will last, and when it will end. For \\nexample, if radio programs are to promote community workers as credible sources of hygiene \\ninformation, then before radio materials are broadcast, these workers need to be trained and \\nequipped and become active in their communities. Necessary time, resources, and guidance for \\nproper supervision must be included in program plans. For example, \\nwhat types of people should be represented in any photos or drawings? In addition, even as the artists or scriptwriters are in the process of creating the \\nmaterials, they should observe at least some of the interviews, so they can hear \\ndirectly from the target audiences what they identify with, what they like or don’t \\nlike. • The changes suggested by mothers were quite different from those that seemed important to \\nthe project staff. This pretest indicated the need for good training \\nand follow-up in the program. It is essential to budget for the time required to fully analyze the pretest and make \\nnecessary adjustments. This includes communication materials \\nthat people are being trained to use.) Then think about \\nthe specific tasks that they will need to perform after the training. Each objective should be \\nlinked directly or indirectly to the knowledge, skills or attitudes that the instructional \\nanalysis has identified as necessary for carrying out the specific tasks to be \\nperformed. • While a personal cleansing material is not essential for safe feces disposal, lack of \\nsuch material may constitute an important barrier to latrine use, so programs may \\nwant to facilitate its availability. It can be tipped to let out a small amount of water at a time for \\nhandwashing. In these cases, the proper use of the technology or product is critical, requiring \\neducation/motivation on the use of technologies. In the case of major technologies \\nsuch as latrines, families may be offered a choice of more than one design that \\nsatisfies both engineering criteria as well as the user preferences learned through \\nformative research. In formative \\nresearch, the team should learn if the chlorine available to families is always full \\nstrength or is sometimes watered down. They include: \\n\\n• Collaborate with private companies to expand promotion, distribution and sale of \\nuseful products. • Have community health workers, volunteers, committees, health facilities or other \\nlocal groups distribute products for free or at a significantly subsidized price. This \\napproach may require a significant program expenditure. Because the \\nmanagement and sustainability of such funds are challenges, this should be \\nconsidered only where the program has the stability and resources to offer long-\\nterm support (see the discussion in Chapter 7). Examples are soap substitutes, including \\nlocal plants which aid cleansing, and solar purification of water. A good guide is \\nAdvocacy for Immunization: How to Generate and Maintain Support for Vaccination \\nPrograms. Community (or Collective) Actions \\nIndividuals usually live in families and communities. Therefore other people may \\ninfluence their behaviors, even intimate ones. It \\nmay require an entire community, or those most affected by a problem, to get together \\nto identify a problem, decide how to solve it and then implement the solution. At the other end of the scale, a program may tell community members what actions \\nthey need to take and how. Other families used water that was \\nnot safe to avoid paying for a connection. After formative research, a strategy was \\ndeveloped to hold meetings with community leaders to explain reasons for \\nestablishing legal connections and paying for water. The method used, \\nwhile not participatory at the individual or family level, seemed to be effective in \\nmobilizing the unconnected communities to levy assessments for connections and for \\nfamilies to pay for water. Negotiate/make arrangements with   radio \\nstations (including broadcast schedule and payment) \\n9. Arrange logistics (including locations for \\npractice) \\n\\nFebruary 2004 \\n\\n \\n\\n• 2 trainers \\n• 1 admin staff \\n\\nMOH None • Training \\nlocales \\n\\n• Per diem & \\ntransport \\n\\n• Training \\nmaterials & \\nsupplies \\n\\nInterpersonal counseling \\nskills / use of support \\nmaterials \\n\\n \\n\\n1. Because evaluation is an important tool in generating \\nconfidence in the program, it is usually best done by persons not working in the \\nproject, in order to ensure the credibility of the findings. Clearly, monitoring and evaluation take time and costs money. This may seem obvious, but too often program managers do not take full \\nadvantage of monitoring and evaluation data to improve their program. Instead, it is often more practical to \\nmeasure service coverage, use of facilities and hygiene behaviors, specifically, \\nbehaviors that have a proven impact on health trends over time. To answer this question, monitoring should be \\ncarried out regularly throughout the program’s implementation. Are health \\nworkers or volunteers making home visits, using materials and using them correctly? As discussed under Community Mobilization in Chapter 8, the collection and clear \\npresentation of key information to each community – including information gained \\nwhile monitoring the program – can be a powerful stimulus to encouraging \\ncommunity analysis, planning and action. Health staff and, if feasible, volunteers should prepare and send routine \\nmonthly reports, which also become an important part of the routine monitoring \\nprocess. • Where radio is used, someone needs to monitor that radio spots are actually \\nbroadcast as scheduled. Periodic Monitoring \\n\\nIn addition to routine monitoring activities through the supervision and information \\nsystems, periodic reviews of project activities and their results should also occur. Although potentially very important for obtaining feedback and making important \\nadjustments to program activities, materials and messages, these monitoring activities \\nshould require a week or less of fieldwork and involve only a small portion of the \\ncommunities covered by the larger program. An increase from 15% to 31% was recorded for \\nreported handwashing of the youngest child before eating. • diarrhea prevalence (wherever possible, measured in the same season). • This allows participant groups to have more input into \\nthe issues, concepts and expressions used in the \\nsurvey instrument. It should give direction for modifying the existing program or designing a \\nmore effective follow-on program. Teams external to the program often carry out cluster sample surveys. The collection of quality data about hygiene improvement is an \\nimportant step toward better use of information, for example, to develop hygiene \\nimprovement program strategies and work plans, design community participation and \\nmobilization approaches, and prepare monitoring and evaluation plans. Monitoring the prevalence of practices can be a challenging task if the practices are \\nnot easily observable. It is helpful to think about each step in the behavior – and \\nwhere there is currently a problem – and find a proxy indicator that is, in fact, \\nobservable. Develop the overall design for the assessment including geographic focus, \\nmethods of data collection, sampling approach, personnel needs, budget and \\nassessment plan, indicating deadlines and responsibilities. Develop an assessment instrument based on model questions related to each \\nindicator. Clarify limitations of the selected indicators, questions and data collection \\nmethods. Translate the assessment instrument and then re-translate into the original \\nlanguage to correct translation errors. Designing by Dialogue: A Program Planners’ \\nGuide to Consultative Research for Improving Young Child Feeding. Available in Spanish and Portuguese at \\nwww.careinternational.org.uk/resource_centre. Activity Report 113, Arlington, Va.: \\nEnvironmental Health Project. USAID/Dominican Republic Reconstruction Activity, \\nEHP Activity Report.\", 'volunTary blood \\ndonaTion \\nTowards safe and healthy living \\n\\n©\\nIF\\n\\nR\\nC\\n\\n\\n\\nISSUE\\n\\nHOW WE HELP\\n\\nIMPACT\\n\\n92 million donations world wide\\n\\nThe IFRC plays an important \\nrole in promoting safe and \\nsustainable blood programmes. The long experience of National Red \\nCross and Red Crescent Societies in supplying blood and blood products \\nmeant that our voice was heard when we advocated for recognition of \\nVNRBD on the global health agenda. In addition, the Global Framework \\nfor Action: Towards 100 per cent Voluntary Blood Donation, jointly developed \\nwith the World Health Organization, provides guidance and support to \\ncountries seeking to establish effective VNRBD programmes. National Society engagement in blood \\nprogrammes in some cases means responsibility for blood collection and \\nsupply. Club 25 is one of many successful initiatives that National Societies support. The viability of a country’s blood service, however, rests with its government \\nand its commitment to safer blood starts with enforcing a national blood policy \\nbased on the principle of 100 per cent voluntary blood donation. IF\\nR\\n\\nC\\n\\n©\\nA\\n\\nn\\nd\\n\\nre\\nw\\n\\n M\\nc\\nC\\n\\no\\nll/\\n\\nIF\\nR\\n\\nC\\n\\n©\\nM\\n\\na\\nrk\\n\\no\\n K\\n\\no\\nki\\n\\nc\\n/I\\n\\nC\\nR\\n\\nC\\n\\n\\n\\nCase study  \\nYouth clubs save lives\\nInternational Club 25 is a global network of young people committed to saving \\nlives by regularly donating blood. But as young people search for new connections with their peers, the \\n‘international’ dimension has now supplanted the ‘club’ aspect. Impartiality It makes no discrimination as to nation-\\nality, race, religious beliefs, class or political opinions. It endeavours to relieve the suffering of individuals, \\nbeing guided solely by their needs, and to give priority \\nto the most urgent cases of distress. Unity There can be only one Red Cross or Red Cres-\\ncent Society in any one country.']\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"v8R-rePrhWIl"},"source":["df['summary'] = summary_list"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"iARSskzEq726"},"source":["df=init_train.loc[5:10]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"coESDzaMq1xA"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"nSnBvntGgpHb"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XLvJ3xxLgpEq"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"b7zNKrJPgpCc"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1KeAXAzAgpAL"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"CogPkO8Cgo9x"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"4Scp7o9ygoyW"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"H5Ab-45pgoth"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XHMANy7PKTqF"},"source":["init_train.to_csv('baseline1.csv', index=None)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"j9H_EvR2ClJF","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1603572659674,"user_tz":240,"elapsed":285,"user":{"displayName":"Yashaswini Joshi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjuSma0c0dqrFcStVWZPlSUGaMr2qrZLTquOCtu=s64","userId":"16993955789369918569"}},"outputId":"442fd4d7-659a-4608-d81e-a9570f058108"},"source":["from google.colab import  drive\n","\n","drive.mount('/drive')\n"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Drive already mounted at /drive; to attempt to forcibly remount, call drive.mount(\"/drive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Olx93gGzaFbI"},"source":["df = pd.DataFrame(init_train, columns= ['ID', 'clean_text'])\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"WM-Q0KGJbdBK","colab":{"base_uri":"https://localhost:8080/","height":417},"executionInfo":{"status":"ok","timestamp":1603572601921,"user_tz":240,"elapsed":682,"user":{"displayName":"Yashaswini Joshi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjuSma0c0dqrFcStVWZPlSUGaMr2qrZLTquOCtu=s64","userId":"16993955789369918569"}},"outputId":"2c9d3255-43d9-44ad-b5d3-c89cd1319738"},"source":["df"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>ID</th>\n","      <th>clean_text</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>Microsoft Word health policy doc Health polic...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1</td>\n","      <td>Health Promotion Glossary World Health Organi...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2</td>\n","      <td>The Use of Structured Observations in the Stu...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>3</td>\n","      <td>RC Health message Standard Operational Proced...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>4</td>\n","      <td>Microsoft Word JP7CIMCIProcessGuideFormat doc...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>260</th>\n","      <td>07</td>\n","      <td>Early Recovery</td>\n","    </tr>\n","    <tr>\n","      <th>261</th>\n","      <td>08</td>\n","      <td>Education</td>\n","    </tr>\n","    <tr>\n","      <th>262</th>\n","      <td>09</td>\n","      <td>Emergency Telecom s</td>\n","    </tr>\n","    <tr>\n","      <th>263</th>\n","      <td>10</td>\n","      <td>Food Security</td>\n","    </tr>\n","    <tr>\n","      <th>264</th>\n","      <td>11</td>\n","      <td>Humanitarian Emergency Relief</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>265 rows × 2 columns</p>\n","</div>"],"text/plain":["     ID                                         clean_text\n","0     0   Microsoft Word health policy doc Health polic...\n","1     1   Health Promotion Glossary World Health Organi...\n","2     2   The Use of Structured Observations in the Stu...\n","3     3   RC Health message Standard Operational Proced...\n","4     4   Microsoft Word JP7CIMCIProcessGuideFormat doc...\n","..   ..                                                ...\n","260  07                                     Early Recovery\n","261  08                                          Education\n","262  09                                Emergency Telecom s\n","263  10                                      Food Security\n","264  11                      Humanitarian Emergency Relief\n","\n","[265 rows x 2 columns]"]},"metadata":{"tags":[]},"execution_count":14}]},{"cell_type":"code","metadata":{"id":"EUEt0qKhKTqI","outputId":"66a03745-2812-4ff1-ff11-8d31faa0bc63"},"source":["init_train[218:]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>ID</th>\n","      <th>Title</th>\n","      <th>Date</th>\n","      <th>Content</th>\n","      <th>clean_text</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>218</th>\n","      <td>218</td>\n","      <td>null</td>\n","      <td>2018-06-10T11:03:52Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>1 2017 CONSOLIDATED EMERGENCY REPORT UNICEF U...</td>\n","    </tr>\n","    <tr>\n","      <th>219</th>\n","      <td>219</td>\n","      <td>null</td>\n","      <td>2019-03-23T20:22:20Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>1 UNICEF Regional Office for West Central Afr...</td>\n","    </tr>\n","    <tr>\n","      <th>220</th>\n","      <td>220</td>\n","      <td>A Gender Review in Education, Turkey 2003</td>\n","      <td>2005-01-08T20:36:29Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>A Gender Review in Education Turkey 2003 A GE...</td>\n","    </tr>\n","    <tr>\n","      <th>221</th>\n","      <td>221</td>\n","      <td></td>\n","      <td>2019-04-19T07:12:53Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>8 EDUCATION AND CHILD DEVELOPMENT 108 Educati...</td>\n","    </tr>\n","    <tr>\n","      <th>222</th>\n","      <td>222</td>\n","      <td>null</td>\n","      <td>2018-09-03T20:01:21Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Raising Learning Outcomes the opportunities a...</td>\n","    </tr>\n","    <tr>\n","      <th>223</th>\n","      <td>223</td>\n","      <td>null</td>\n","      <td>2013-04-09T11:00:44Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>WASH in Schools Empowers Girls Education Proc...</td>\n","    </tr>\n","    <tr>\n","      <th>224</th>\n","      <td>224</td>\n","      <td>null</td>\n","      <td>2014-03-12T16:13:03Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>WASH in Schools Empowers Girls Education Proc...</td>\n","    </tr>\n","    <tr>\n","      <th>225</th>\n","      <td>225</td>\n","      <td>null</td>\n","      <td>2017-12-11T04:25:17Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>THE STATE OF THE WORLD S CHILDREN 2017 Childr...</td>\n","    </tr>\n","    <tr>\n","      <th>226</th>\n","      <td>226</td>\n","      <td>null</td>\n","      <td>2020-06-08T19:45:16Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Making urban food environments fit for childr...</td>\n","    </tr>\n","    <tr>\n","      <th>227</th>\n","      <td>227</td>\n","      <td>MHM study</td>\n","      <td>2013-09-20T14:48:56Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>MHM study i Menstrual Hygiene in Schools in 2...</td>\n","    </tr>\n","    <tr>\n","      <th>228</th>\n","      <td>228</td>\n","      <td>Layout 1</td>\n","      <td>2013-06-19T19:46:51Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Layout 1 Breaking the Silence on Violence aga...</td>\n","    </tr>\n","    <tr>\n","      <th>229</th>\n","      <td>229</td>\n","      <td>null</td>\n","      <td>2017-07-12T04:17:15Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>EMERGING GLOBAL CHALLENGES Protecting Uganda ...</td>\n","    </tr>\n","    <tr>\n","      <th>230</th>\n","      <td>230</td>\n","      <td>null</td>\n","      <td>2018-05-22T08:34:01Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>THE REPUBLIC OF UGANDATHE REPUBLIC OF UGANDA ...</td>\n","    </tr>\n","    <tr>\n","      <th>231</th>\n","      <td>231</td>\n","      <td>UNICEF Uganda End-of-Year Humanitarian Situati...</td>\n","      <td>2020-06-08T06:34:57Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>UNICEF Uganda End of Year Humanitarian Situat...</td>\n","    </tr>\n","    <tr>\n","      <th>232</th>\n","      <td>232</td>\n","      <td>UNICEF Uganda End-of-Year Humanitarian Situati...</td>\n","      <td>2020-06-02T12:10:12Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>UNICEF Uganda End of Year Humanitarian Situat...</td>\n","    </tr>\n","    <tr>\n","      <th>233</th>\n","      <td>233</td>\n","      <td>Microsoft Word - UNICEF Turkey Humanitarian Si...</td>\n","      <td>2017-02-22T22:37:17Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Microsoft Word UNICEF Turkey Humanitarian Sit...</td>\n","    </tr>\n","    <tr>\n","      <th>234</th>\n","      <td>234</td>\n","      <td>null</td>\n","      <td>2015-05-01T03:54:46Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>1 Situation Overview and Humanitarian Needs O...</td>\n","    </tr>\n","    <tr>\n","      <th>235</th>\n","      <td>235</td>\n","      <td>null</td>\n","      <td>2013-08-05T20:54:52Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>UNICEF s Work in Water Sanitation and Hygiene...</td>\n","    </tr>\n","    <tr>\n","      <th>236</th>\n","      <td>236</td>\n","      <td>Health/Nutrition/WASH InterCluster Matrix</td>\n","      <td>2017-10-30T06:40:52Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Health Nutrition WASH InterCluster Matrix INT...</td>\n","    </tr>\n","    <tr>\n","      <th>237</th>\n","      <td>237</td>\n","      <td>null</td>\n","      <td>2019-11-19T07:41:51Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Strengthening IFRC Responses to Internal Disp...</td>\n","    </tr>\n","    <tr>\n","      <th>238</th>\n","      <td>238</td>\n","      <td>Shelter Training Summary.doc</td>\n","      <td>null</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Shelter Training Summary doc 1 FEDERATION SHE...</td>\n","    </tr>\n","    <tr>\n","      <th>239</th>\n","      <td>239</td>\n","      <td>null</td>\n","      <td>2019-11-21T03:00:01Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>FOURTH PROGRESS REPORT ON THE IMPLEMENTATION ...</td>\n","    </tr>\n","    <tr>\n","      <th>240</th>\n","      <td>240</td>\n","      <td>null</td>\n","      <td>2017-11-21T14:46:30Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>International Federation of Red Cross and Red...</td>\n","    </tr>\n","    <tr>\n","      <th>241</th>\n","      <td>241</td>\n","      <td>null</td>\n","      <td>2018-07-17T06:33:10Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>International Federation of Red Cross and Red...</td>\n","    </tr>\n","    <tr>\n","      <th>242</th>\n","      <td>242</td>\n","      <td>TITlE OF\\nTHE DOCUMENTS</td>\n","      <td>2020-03-17T08:31:18Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>TITlE OF THE DOCUMENTS Interim Guidance SCALI...</td>\n","    </tr>\n","    <tr>\n","      <th>243</th>\n","      <td>243</td>\n","      <td>null</td>\n","      <td>2019-12-20T10:33:05Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>International Federation of Red Cross and Red...</td>\n","    </tr>\n","    <tr>\n","      <th>244</th>\n","      <td>244</td>\n","      <td>Background and Rationale for School Sanitation...</td>\n","      <td>2006-11-09T17:23:45Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Background and Rationale for School Sanitatio...</td>\n","    </tr>\n","    <tr>\n","      <th>245</th>\n","      <td>245</td>\n","      <td>null</td>\n","      <td>2017-09-26T08:50:13Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Process Evaluation Asia Pacific Multi Nationa...</td>\n","    </tr>\n","    <tr>\n","      <th>246</th>\n","      <td>246</td>\n","      <td>PowerPoint Presentation</td>\n","      <td>2018-12-13T09:17:21Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>PowerPoint Presentation IFRC INNOVATION AND F...</td>\n","    </tr>\n","    <tr>\n","      <th>247</th>\n","      <td>247</td>\n","      <td>null</td>\n","      <td>2018-11-27T09:22:32Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Leaving Millions Behind World Disasters Repor...</td>\n","    </tr>\n","    <tr>\n","      <th>248</th>\n","      <td>248</td>\n","      <td>Climate change, water and food security - ODI ...</td>\n","      <td>2009-03-11T09:04:07Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Climate change water and food security ODI Ba...</td>\n","    </tr>\n","    <tr>\n","      <th>249</th>\n","      <td>249</td>\n","      <td>IFRS - Group, Report (new structure)</td>\n","      <td>2018-04-18T07:02:36Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>IFRS Group Report new structure KPMG SA Genev...</td>\n","    </tr>\n","    <tr>\n","      <th>250</th>\n","      <td>250</td>\n","      <td>Food Security and Nutrition Indicator Basics</td>\n","      <td>2017-11-29T13:20:33Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Food Security and Nutrition Indicator Basics ...</td>\n","    </tr>\n","    <tr>\n","      <th>251</th>\n","      <td>251</td>\n","      <td>Background document about the Clusters and WAS...</td>\n","      <td>2008-03-28T08:12:19Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Background document about the Clusters and WA...</td>\n","    </tr>\n","    <tr>\n","      <th>252</th>\n","      <td>252</td>\n","      <td>null</td>\n","      <td>2019-06-03T07:27:58Z</td>\n","      <td>\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n","      <td>Digital Water Industry leaders chart the tran...</td>\n","    </tr>\n","    <tr>\n","      <th>253</th>\n","      <td>00</td>\n","      <td>topic</td>\n","      <td>2020</td>\n","      <td>health</td>\n","      <td>health</td>\n","    </tr>\n","    <tr>\n","      <th>254</th>\n","      <td>01</td>\n","      <td>topic</td>\n","      <td>2020</td>\n","      <td>logistics</td>\n","      <td>logistics</td>\n","    </tr>\n","    <tr>\n","      <th>255</th>\n","      <td>02</td>\n","      <td>topic</td>\n","      <td>2020</td>\n","      <td>nutrition</td>\n","      <td>nutrition</td>\n","    </tr>\n","    <tr>\n","      <th>256</th>\n","      <td>03</td>\n","      <td>topic</td>\n","      <td>2020</td>\n","      <td>protection</td>\n","      <td>protection</td>\n","    </tr>\n","    <tr>\n","      <th>257</th>\n","      <td>04</td>\n","      <td>topic</td>\n","      <td>2020</td>\n","      <td>shelter</td>\n","      <td>shelter</td>\n","    </tr>\n","    <tr>\n","      <th>258</th>\n","      <td>05</td>\n","      <td>topic</td>\n","      <td>2020</td>\n","      <td>Water, Sanitation and Hygiene</td>\n","      <td>Water Sanitation and Hygiene</td>\n","    </tr>\n","    <tr>\n","      <th>259</th>\n","      <td>06</td>\n","      <td>topic</td>\n","      <td>2020</td>\n","      <td>Camp Coordination &amp; Mgmt</td>\n","      <td>Camp Coordination Mgmt</td>\n","    </tr>\n","    <tr>\n","      <th>260</th>\n","      <td>07</td>\n","      <td>topic</td>\n","      <td>2020</td>\n","      <td>Early Recovery</td>\n","      <td>Early Recovery</td>\n","    </tr>\n","    <tr>\n","      <th>261</th>\n","      <td>08</td>\n","      <td>topic</td>\n","      <td>2020</td>\n","      <td>Education</td>\n","      <td>Education</td>\n","    </tr>\n","    <tr>\n","      <th>262</th>\n","      <td>09</td>\n","      <td>topic</td>\n","      <td>2020</td>\n","      <td>Emergency Telecom’s</td>\n","      <td>Emergency Telecom s</td>\n","    </tr>\n","    <tr>\n","      <th>263</th>\n","      <td>10</td>\n","      <td>topic</td>\n","      <td>2020</td>\n","      <td>Food Security</td>\n","      <td>Food Security</td>\n","    </tr>\n","    <tr>\n","      <th>264</th>\n","      <td>11</td>\n","      <td>topic</td>\n","      <td>2020</td>\n","      <td>Humanitarian &amp; Emergency Relief</td>\n","      <td>Humanitarian Emergency Relief</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["      ID                                              Title  \\\n","218  218                                               null   \n","219  219                                               null   \n","220  220          A Gender Review in Education, Turkey 2003   \n","221  221                                                      \n","222  222                                               null   \n","223  223                                               null   \n","224  224                                               null   \n","225  225                                               null   \n","226  226                                               null   \n","227  227                                          MHM study   \n","228  228                                           Layout 1   \n","229  229                                               null   \n","230  230                                               null   \n","231  231  UNICEF Uganda End-of-Year Humanitarian Situati...   \n","232  232  UNICEF Uganda End-of-Year Humanitarian Situati...   \n","233  233  Microsoft Word - UNICEF Turkey Humanitarian Si...   \n","234  234                                               null   \n","235  235                                               null   \n","236  236          Health/Nutrition/WASH InterCluster Matrix   \n","237  237                                               null   \n","238  238                       Shelter Training Summary.doc   \n","239  239                                               null   \n","240  240                                               null   \n","241  241                                               null   \n","242  242                            TITlE OF\\nTHE DOCUMENTS   \n","243  243                                               null   \n","244  244  Background and Rationale for School Sanitation...   \n","245  245                                               null   \n","246  246                            PowerPoint Presentation   \n","247  247                                               null   \n","248  248  Climate change, water and food security - ODI ...   \n","249  249               IFRS - Group, Report (new structure)   \n","250  250       Food Security and Nutrition Indicator Basics   \n","251  251  Background document about the Clusters and WAS...   \n","252  252                                               null   \n","253   00                                              topic   \n","254   01                                              topic   \n","255   02                                              topic   \n","256   03                                              topic   \n","257   04                                              topic   \n","258   05                                              topic   \n","259   06                                              topic   \n","260   07                                              topic   \n","261   08                                              topic   \n","262   09                                              topic   \n","263   10                                              topic   \n","264   11                                              topic   \n","\n","                     Date                                            Content  \\\n","218  2018-06-10T11:03:52Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","219  2019-03-23T20:22:20Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","220  2005-01-08T20:36:29Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","221  2019-04-19T07:12:53Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","222  2018-09-03T20:01:21Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","223  2013-04-09T11:00:44Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","224  2014-03-12T16:13:03Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","225  2017-12-11T04:25:17Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","226  2020-06-08T19:45:16Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","227  2013-09-20T14:48:56Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","228  2013-06-19T19:46:51Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","229  2017-07-12T04:17:15Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","230  2018-05-22T08:34:01Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","231  2020-06-08T06:34:57Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","232  2020-06-02T12:10:12Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","233  2017-02-22T22:37:17Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","234  2015-05-01T03:54:46Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","235  2013-08-05T20:54:52Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","236  2017-10-30T06:40:52Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","237  2019-11-19T07:41:51Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","238                  null  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","239  2019-11-21T03:00:01Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","240  2017-11-21T14:46:30Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","241  2018-07-17T06:33:10Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","242  2020-03-17T08:31:18Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","243  2019-12-20T10:33:05Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","244  2006-11-09T17:23:45Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","245  2017-09-26T08:50:13Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","246  2018-12-13T09:17:21Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","247  2018-11-27T09:22:32Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","248  2009-03-11T09:04:07Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","249  2018-04-18T07:02:36Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","250  2017-11-29T13:20:33Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","251  2008-03-28T08:12:19Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","252  2019-06-03T07:27:58Z  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n","253                  2020                                             health   \n","254                  2020                                          logistics   \n","255                  2020                                          nutrition   \n","256                  2020                                         protection   \n","257                  2020                                            shelter   \n","258                  2020                      Water, Sanitation and Hygiene   \n","259                  2020                          Camp Coordination & Mgmt    \n","260                  2020                                     Early Recovery   \n","261                  2020                                          Education   \n","262                  2020                                Emergency Telecom’s   \n","263                  2020                                      Food Security   \n","264                  2020                    Humanitarian & Emergency Relief   \n","\n","                                            clean_text  \n","218   1 2017 CONSOLIDATED EMERGENCY REPORT UNICEF U...  \n","219   1 UNICEF Regional Office for West Central Afr...  \n","220   A Gender Review in Education Turkey 2003 A GE...  \n","221   8 EDUCATION AND CHILD DEVELOPMENT 108 Educati...  \n","222   Raising Learning Outcomes the opportunities a...  \n","223   WASH in Schools Empowers Girls Education Proc...  \n","224   WASH in Schools Empowers Girls Education Proc...  \n","225   THE STATE OF THE WORLD S CHILDREN 2017 Childr...  \n","226   Making urban food environments fit for childr...  \n","227   MHM study i Menstrual Hygiene in Schools in 2...  \n","228   Layout 1 Breaking the Silence on Violence aga...  \n","229   EMERGING GLOBAL CHALLENGES Protecting Uganda ...  \n","230   THE REPUBLIC OF UGANDATHE REPUBLIC OF UGANDA ...  \n","231   UNICEF Uganda End of Year Humanitarian Situat...  \n","232   UNICEF Uganda End of Year Humanitarian Situat...  \n","233   Microsoft Word UNICEF Turkey Humanitarian Sit...  \n","234   1 Situation Overview and Humanitarian Needs O...  \n","235   UNICEF s Work in Water Sanitation and Hygiene...  \n","236   Health Nutrition WASH InterCluster Matrix INT...  \n","237   Strengthening IFRC Responses to Internal Disp...  \n","238   Shelter Training Summary doc 1 FEDERATION SHE...  \n","239   FOURTH PROGRESS REPORT ON THE IMPLEMENTATION ...  \n","240   International Federation of Red Cross and Red...  \n","241   International Federation of Red Cross and Red...  \n","242   TITlE OF THE DOCUMENTS Interim Guidance SCALI...  \n","243   International Federation of Red Cross and Red...  \n","244   Background and Rationale for School Sanitatio...  \n","245   Process Evaluation Asia Pacific Multi Nationa...  \n","246   PowerPoint Presentation IFRC INNOVATION AND F...  \n","247   Leaving Millions Behind World Disasters Repor...  \n","248   Climate change water and food security ODI Ba...  \n","249   IFRS Group Report new structure KPMG SA Genev...  \n","250   Food Security and Nutrition Indicator Basics ...  \n","251   Background document about the Clusters and WA...  \n","252   Digital Water Industry leaders chart the tran...  \n","253                                             health  \n","254                                          logistics  \n","255                                          nutrition  \n","256                                         protection  \n","257                                            shelter  \n","258                       Water Sanitation and Hygiene  \n","259                            Camp Coordination Mgmt   \n","260                                     Early Recovery  \n","261                                          Education  \n","262                                Emergency Telecom s  \n","263                                      Food Security  \n","264                      Humanitarian Emergency Relief  "]},"metadata":{"tags":[]},"execution_count":6}]},{"cell_type":"code","metadata":{"id":"oooaLl2yKTqK"},"source":["X_train=init_train\n","from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n","\n","count_vectorizer = CountVectorizer(binary=True,ngram_range=(1,3))\n","tfidf_transformer = TfidfVectorizer(use_idf=True,ngram_range=(1,3)) \n","dataTrain = count_vectorizer.fit_transform(X_train['clean_text'].astype('unicode'))\n","tfidf_train_data = tfidf_transformer.fit_transform(X_train['clean_text'].astype('unicode'))\n","\n","# testData = count_vectorizer.transform(X_valid['clean_text'])\n","# tfidf_test_data = tfidf_transformer.transform(X_valid['clean_text'])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"6Yetj0AyKTqM","outputId":"5fb42d27-82fc-4363-f6e4-9418693022f2"},"source":["pd.DataFrame(tfidf_train_data.toarray(),columns=tfidf_transformer.get_feature_names())"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>00</th>\n","      <th>00 00</th>\n","      <th>00 00 00</th>\n","      <th>00 00 10</th>\n","      <th>00 00 12</th>\n","      <th>00 00 20</th>\n","      <th>00 00 200000</th>\n","      <th>00 00 50</th>\n","      <th>00 00 as</th>\n","      <th>00 00 capital</th>\n","      <th>...</th>\n","      <th>zz 47</th>\n","      <th>zz 47 organization</th>\n","      <th>zz philippines</th>\n","      <th>zz philippines 43</th>\n","      <th>zzi</th>\n","      <th>zzi the</th>\n","      <th>zzi the investment</th>\n","      <th>zziwa</th>\n","      <th>zziwa 2014</th>\n","      <th>zziwa 2014 land</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0.00000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0.00000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0.00081</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0.00000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0.00212</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>260</th>\n","      <td>0.00000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>261</th>\n","      <td>0.00000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>262</th>\n","      <td>0.00000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>263</th>\n","      <td>0.00000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>264</th>\n","      <td>0.00000</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>265 rows × 3702495 columns</p>\n","</div>"],"text/plain":["          00  00 00  00 00 00  00 00 10  00 00 12  00 00 20  00 00 200000  \\\n","0    0.00000    0.0       0.0       0.0       0.0       0.0           0.0   \n","1    0.00000    0.0       0.0       0.0       0.0       0.0           0.0   \n","2    0.00081    0.0       0.0       0.0       0.0       0.0           0.0   \n","3    0.00000    0.0       0.0       0.0       0.0       0.0           0.0   \n","4    0.00212    0.0       0.0       0.0       0.0       0.0           0.0   \n","..       ...    ...       ...       ...       ...       ...           ...   \n","260  0.00000    0.0       0.0       0.0       0.0       0.0           0.0   \n","261  0.00000    0.0       0.0       0.0       0.0       0.0           0.0   \n","262  0.00000    0.0       0.0       0.0       0.0       0.0           0.0   \n","263  0.00000    0.0       0.0       0.0       0.0       0.0           0.0   \n","264  0.00000    0.0       0.0       0.0       0.0       0.0           0.0   \n","\n","     00 00 50  00 00 as  00 00 capital  ...  zz 47  zz 47 organization  \\\n","0         0.0       0.0            0.0  ...    0.0                 0.0   \n","1         0.0       0.0            0.0  ...    0.0                 0.0   \n","2         0.0       0.0            0.0  ...    0.0                 0.0   \n","3         0.0       0.0            0.0  ...    0.0                 0.0   \n","4         0.0       0.0            0.0  ...    0.0                 0.0   \n","..        ...       ...            ...  ...    ...                 ...   \n","260       0.0       0.0            0.0  ...    0.0                 0.0   \n","261       0.0       0.0            0.0  ...    0.0                 0.0   \n","262       0.0       0.0            0.0  ...    0.0                 0.0   \n","263       0.0       0.0            0.0  ...    0.0                 0.0   \n","264       0.0       0.0            0.0  ...    0.0                 0.0   \n","\n","     zz philippines  zz philippines 43  zzi  zzi the  zzi the investment  \\\n","0               0.0                0.0  0.0      0.0                 0.0   \n","1               0.0                0.0  0.0      0.0                 0.0   \n","2               0.0                0.0  0.0      0.0                 0.0   \n","3               0.0                0.0  0.0      0.0                 0.0   \n","4               0.0                0.0  0.0      0.0                 0.0   \n","..              ...                ...  ...      ...                 ...   \n","260             0.0                0.0  0.0      0.0                 0.0   \n","261             0.0                0.0  0.0      0.0                 0.0   \n","262             0.0                0.0  0.0      0.0                 0.0   \n","263             0.0                0.0  0.0      0.0                 0.0   \n","264             0.0                0.0  0.0      0.0                 0.0   \n","\n","     zziwa  zziwa 2014  zziwa 2014 land  \n","0      0.0         0.0              0.0  \n","1      0.0         0.0              0.0  \n","2      0.0         0.0              0.0  \n","3      0.0         0.0              0.0  \n","4      0.0         0.0              0.0  \n","..     ...         ...              ...  \n","260    0.0         0.0              0.0  \n","261    0.0         0.0              0.0  \n","262    0.0         0.0              0.0  \n","263    0.0         0.0              0.0  \n","264    0.0         0.0              0.0  \n","\n","[265 rows x 3702495 columns]"]},"metadata":{"tags":[]},"execution_count":8}]},{"cell_type":"code","metadata":{"id":"03isuBZwKTqO","outputId":"ea84ea06-d23c-4ae0-863c-6389ac73ecbd"},"source":["from sklearn.metrics.pairwise import cosine_similarity\n","print('health',cosine_similarity(tfidf_train_data[1], tfidf_train_data[253]))\n","print('logistics',cosine_similarity(tfidf_train_data[1], tfidf_train_data[254]))\n","print('nutrition',cosine_similarity(tfidf_train_data[1], tfidf_train_data[255]))\n","print('protection',cosine_similarity(tfidf_train_data[1], tfidf_train_data[256]))\n","print('shelter',cosine_similarity(tfidf_train_data[1], tfidf_train_data[257]))\n","print('Water, Sanitation and Hygiene',cosine_similarity(tfidf_train_data[1], tfidf_train_data[258]))\n","print('Camp Coordination & Mgmt ',cosine_similarity(tfidf_train_data[1], tfidf_train_data[259]))\n","print('Early Recovery',cosine_similarity(tfidf_train_data[1], tfidf_train_data[260]))\n","print('Education',cosine_similarity(tfidf_train_data[1], tfidf_train_data[261]))\n","print('Emergency Telecom’s',cosine_similarity(tfidf_train_data[1], tfidf_train_data[262]))\n","print('Food Security',cosine_similarity(tfidf_train_data[1], tfidf_train_data[263]))\n","print('Humanitarian & Emergency Relief',cosine_similarity(tfidf_train_data[1], tfidf_train_data[264]))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["health [[0.40530126]]\n","logistics [[0.]]\n","nutrition [[0.00100521]]\n","protection [[0.00148882]]\n","shelter [[0.00101022]]\n","Water, Sanitation and Hygiene [[0.06555704]]\n","Camp Coordination & Mgmt  [[0.]]\n","Early Recovery [[0.00097696]]\n","Education [[0.023505]]\n","Emergency Telecom’s [[0.]]\n","Food Security [[0.00112937]]\n","Humanitarian & Emergency Relief [[0.]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"7TjFP04UKTqQ","outputId":"96e6631b-a596-431f-9f21-fde44819e10c"},"source":["tfidf_train_data.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(265, 3702495)"]},"metadata":{"tags":[]},"execution_count":12}]},{"cell_type":"code","metadata":{"id":"LYNhNWUmKTqT","outputId":"16258843-ccd1-4445-ba15-8cfd9e2a393c"},"source":["print(tfidf_train_data)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["  (0, 2520185)\t0.016203315945286463\n","  (0, 1573635)\t0.014296589787173668\n","  (0, 3207313)\t0.01508795264364636\n","  (0, 2748735)\t0.016203315945286463\n","  (0, 2520591)\t0.016203315945286463\n","  (0, 3231160)\t0.01508795264364636\n","  (0, 87385)\t0.016203315945286463\n","  (0, 2214402)\t0.014296589787173668\n","  (0, 2971102)\t0.016203315945286463\n","  (0, 996038)\t0.016203315945286463\n","  (0, 2283793)\t0.005671395877715878\n","  (0, 1340800)\t0.005889088747894446\n","  (0, 1800134)\t0.005889088747894446\n","  (0, 2295164)\t0.00576259177761011\n","  (0, 585584)\t0.011776034909507724\n","  (0, 1477313)\t0.011776034909507724\n","  (0, 3204438)\t0.010159137025780666\n","  (0, 2294785)\t0.00948484819394147\n","  (0, 2905392)\t0.010159137025780666\n","  (0, 53804)\t0.016203315945286463\n","  (0, 3173048)\t0.016203315945286463\n","  (0, 756505)\t0.016203315945286463\n","  (0, 313106)\t0.008177041101685802\n","  (0, 3545486)\t0.01127450032742077\n","  (0, 2521022)\t0.01508795264364636\n","  :\t:\n","  (258, 2840910)\t0.28183155512707536\n","  (258, 3557972)\t0.25230926396672093\n","  (258, 398101)\t0.21012810205020038\n","  (259, 764354)\t0.5195829148262169\n","  (259, 965216)\t0.5195829148262169\n","  (259, 764350)\t0.3972990146229582\n","  (259, 2080592)\t0.48381716679579967\n","  (259, 764235)\t0.2225564832274632\n","  (259, 964404)\t0.13641936485748682\n","  (260, 1176479)\t0.779785614333597\n","  (260, 2703352)\t0.4627263441752194\n","  (260, 1175473)\t0.42168557727839223\n","  (261, 1189730)\t1.0\n","  (262, 1218124)\t0.8455349417541524\n","  (262, 3138095)\t0.4888370018431951\n","  (262, 1215283)\t0.21473017464194033\n","  (263, 1388059)\t0.7102101315167996\n","  (263, 1385218)\t0.48947786607921\n","  (263, 2878768)\t0.5059772600714709\n","  (264, 1624207)\t0.7092526623440809\n","  (264, 1624193)\t0.4838703210855272\n","  (264, 1217314)\t0.3831783035780146\n","  (264, 1215283)\t0.18012022984460854\n","  (264, 2738998)\t0.2162556059562356\n","  (264, 1622709)\t0.19181964625846798\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"pdQmpCiFKTqW","outputId":"f5c06992-5478-43e8-d89c-dfb2bbb11391"},"source":["import numpy as np\n","from sklearn.metrics.pairwise import cosine_similarity\n","health=[]\n","other=[]\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[253])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.19:\n","        health.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_health = np.array(health)\n","\n","print(topic_health)\n","    "],"execution_count":null,"outputs":[{"output_type":"stream","text":["[[[0.21823508]\n","  [0.40530126]\n","  [0.0473445 ]\n","  [0.0526906 ]\n","  [0.05430843]\n","  [0.02792875]\n","  [0.02696357]\n","  [0.        ]\n","  [0.00957978]\n","  [0.00042376]\n","  [0.0150329 ]\n","  [0.        ]\n","  [0.02261672]\n","  [0.00209608]\n","  [0.00623346]\n","  [0.02761352]\n","  [0.        ]\n","  [0.00553169]\n","  [0.06170613]\n","  [0.02451048]\n","  [0.00348082]\n","  [0.06443576]\n","  [0.02865574]\n","  [0.06992007]\n","  [0.03056862]\n","  [0.06992007]\n","  [0.03710314]\n","  [0.05000697]\n","  [0.00312244]\n","  [0.00312244]\n","  [0.0018137 ]\n","  [0.00431548]\n","  [0.00186804]\n","  [0.        ]\n","  [0.00331223]\n","  [0.00249369]\n","  [0.00206615]\n","  [0.00333901]\n","  [0.00327926]\n","  [0.00336069]\n","  [0.00317315]\n","  [0.00316132]\n","  [0.00330456]\n","  [0.00486922]\n","  [0.00323738]\n","  [0.00322514]\n","  [0.04804482]\n","  [0.        ]\n","  [0.01269365]\n","  [0.02416163]\n","  [0.02320412]\n","  [0.        ]\n","  [0.01992383]\n","  [0.02892946]\n","  [0.02512294]\n","  [0.02563169]\n","  [0.02488109]\n","  [0.01772467]\n","  [0.02250207]\n","  [0.02563482]\n","  [0.00751986]\n","  [0.00930765]\n","  [0.00961149]\n","  [0.01676962]\n","  [0.0020463 ]\n","  [0.06745565]\n","  [0.01675084]\n","  [0.01401744]\n","  [0.00519812]\n","  [0.00243875]\n","  [0.00802411]\n","  [0.00399377]\n","  [0.01447789]\n","  [0.01233137]\n","  [0.01231608]\n","  [0.01474027]\n","  [0.06109221]\n","  [0.00519597]\n","  [0.01105768]\n","  [0.02996546]\n","  [0.00174712]\n","  [0.13941505]\n","  [0.        ]\n","  [0.        ]\n","  [0.01729756]\n","  [0.03011968]\n","  [0.        ]\n","  [0.00554569]\n","  [0.00640176]\n","  [0.02092077]\n","  [0.02778762]\n","  [0.00659391]\n","  [0.02354341]\n","  [0.0119024 ]\n","  [0.02780967]\n","  [0.02719695]\n","  [0.04458866]\n","  [0.06478361]\n","  [0.0773559 ]\n","  [0.03786772]\n","  [0.0071103 ]\n","  [0.02595132]\n","  [0.00565468]\n","  [0.01512177]\n","  [0.04644463]\n","  [0.02645926]\n","  [0.00796963]\n","  [0.02089329]\n","  [0.01742632]\n","  [0.0133447 ]\n","  [0.00273006]\n","  [0.01319977]\n","  [0.        ]\n","  [0.08502468]\n","  [0.00838151]\n","  [0.01168584]\n","  [0.        ]\n","  [0.01105689]\n","  [0.00406119]\n","  [0.01756183]\n","  [0.00083019]\n","  [0.00128636]\n","  [0.00710726]\n","  [0.        ]\n","  [0.01468952]\n","  [0.02360527]\n","  [0.00545538]\n","  [0.00903206]\n","  [0.        ]\n","  [0.036793  ]\n","  [0.01024639]\n","  [0.02207567]\n","  [0.01533473]\n","  [0.00338895]\n","  [0.01811337]\n","  [0.01433556]\n","  [0.0023484 ]\n","  [0.0098977 ]\n","  [0.00094712]\n","  [0.        ]\n","  [0.        ]\n","  [0.00393026]\n","  [0.02251626]\n","  [0.00046644]\n","  [0.        ]\n","  [0.00057096]\n","  [0.00522431]\n","  [0.00724307]\n","  [0.00168312]\n","  [0.00815074]\n","  [0.01446889]\n","  [0.03328482]\n","  [0.00309232]\n","  [0.03835674]\n","  [0.00426269]\n","  [0.00220706]\n","  [0.00092665]\n","  [0.02386485]\n","  [0.03096351]\n","  [0.01736524]\n","  [0.00671375]\n","  [0.        ]\n","  [0.005439  ]\n","  [0.00252629]\n","  [0.00981298]\n","  [0.00394672]\n","  [0.00243872]\n","  [0.        ]\n","  [0.00539494]\n","  [0.        ]\n","  [0.00042216]\n","  [0.00666448]\n","  [0.        ]\n","  [0.0009697 ]\n","  [0.01968862]\n","  [0.00061606]\n","  [0.03229554]\n","  [0.04248788]\n","  [0.11580216]\n","  [0.00183291]\n","  [0.00938541]\n","  [0.00155737]\n","  [0.02762331]\n","  [0.00616602]\n","  [0.06425905]\n","  [0.05202456]\n","  [0.00969042]\n","  [0.02032093]\n","  [0.03290629]\n","  [0.01356468]\n","  [0.09232085]\n","  [0.01034595]\n","  [0.04367963]\n","  [0.0116741 ]\n","  [0.0258684 ]\n","  [0.03331412]\n","  [0.00948365]\n","  [0.01596836]\n","  [0.02193003]\n","  [0.02343319]\n","  [0.01230189]\n","  [0.00869276]\n","  [0.02982467]\n","  [0.01556879]\n","  [0.00460953]\n","  [0.02044134]\n","  [0.03246393]\n","  [0.03384801]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.01136124]\n","  [0.04569164]\n","  [0.00752391]\n","  [0.04668908]\n","  [0.00973706]\n","  [0.00485298]\n","  [0.02357003]\n","  [0.0312935 ]\n","  [0.02810546]\n","  [0.01012093]\n","  [0.00248187]\n","  [0.00163543]\n","  [0.01256405]\n","  [0.02703523]\n","  [0.01104509]\n","  [0.01794259]\n","  [0.01077214]\n","  [0.01413037]\n","  [0.01515656]\n","  [0.03942752]\n","  [0.05620345]\n","  [0.06658783]\n","  [0.00413798]\n","  [0.03779829]\n","  [0.02251003]\n","  [0.1211319 ]\n","  [0.0051976 ]\n","  [0.        ]\n","  [0.00073201]\n","  [0.02107251]\n","  [0.01801517]\n","  [0.05723362]\n","  [0.        ]\n","  [0.07487197]\n","  [0.02927349]\n","  [0.        ]\n","  [0.01422165]\n","  [0.00546053]\n","  [0.00113361]\n","  [0.01603329]\n","  [0.00892514]]\n","\n"," [[0.21823508]\n","  [0.40530126]\n","  [0.0473445 ]\n","  [0.0526906 ]\n","  [0.05430843]\n","  [0.02792875]\n","  [0.02696357]\n","  [0.        ]\n","  [0.00957978]\n","  [0.00042376]\n","  [0.0150329 ]\n","  [0.        ]\n","  [0.02261672]\n","  [0.00209608]\n","  [0.00623346]\n","  [0.02761352]\n","  [0.        ]\n","  [0.00553169]\n","  [0.06170613]\n","  [0.02451048]\n","  [0.00348082]\n","  [0.06443576]\n","  [0.02865574]\n","  [0.06992007]\n","  [0.03056862]\n","  [0.06992007]\n","  [0.03710314]\n","  [0.05000697]\n","  [0.00312244]\n","  [0.00312244]\n","  [0.0018137 ]\n","  [0.00431548]\n","  [0.00186804]\n","  [0.        ]\n","  [0.00331223]\n","  [0.00249369]\n","  [0.00206615]\n","  [0.00333901]\n","  [0.00327926]\n","  [0.00336069]\n","  [0.00317315]\n","  [0.00316132]\n","  [0.00330456]\n","  [0.00486922]\n","  [0.00323738]\n","  [0.00322514]\n","  [0.04804482]\n","  [0.        ]\n","  [0.01269365]\n","  [0.02416163]\n","  [0.02320412]\n","  [0.        ]\n","  [0.01992383]\n","  [0.02892946]\n","  [0.02512294]\n","  [0.02563169]\n","  [0.02488109]\n","  [0.01772467]\n","  [0.02250207]\n","  [0.02563482]\n","  [0.00751986]\n","  [0.00930765]\n","  [0.00961149]\n","  [0.01676962]\n","  [0.0020463 ]\n","  [0.06745565]\n","  [0.01675084]\n","  [0.01401744]\n","  [0.00519812]\n","  [0.00243875]\n","  [0.00802411]\n","  [0.00399377]\n","  [0.01447789]\n","  [0.01233137]\n","  [0.01231608]\n","  [0.01474027]\n","  [0.06109221]\n","  [0.00519597]\n","  [0.01105768]\n","  [0.02996546]\n","  [0.00174712]\n","  [0.13941505]\n","  [0.        ]\n","  [0.        ]\n","  [0.01729756]\n","  [0.03011968]\n","  [0.        ]\n","  [0.00554569]\n","  [0.00640176]\n","  [0.02092077]\n","  [0.02778762]\n","  [0.00659391]\n","  [0.02354341]\n","  [0.0119024 ]\n","  [0.02780967]\n","  [0.02719695]\n","  [0.04458866]\n","  [0.06478361]\n","  [0.0773559 ]\n","  [0.03786772]\n","  [0.0071103 ]\n","  [0.02595132]\n","  [0.00565468]\n","  [0.01512177]\n","  [0.04644463]\n","  [0.02645926]\n","  [0.00796963]\n","  [0.02089329]\n","  [0.01742632]\n","  [0.0133447 ]\n","  [0.00273006]\n","  [0.01319977]\n","  [0.        ]\n","  [0.08502468]\n","  [0.00838151]\n","  [0.01168584]\n","  [0.        ]\n","  [0.01105689]\n","  [0.00406119]\n","  [0.01756183]\n","  [0.00083019]\n","  [0.00128636]\n","  [0.00710726]\n","  [0.        ]\n","  [0.01468952]\n","  [0.02360527]\n","  [0.00545538]\n","  [0.00903206]\n","  [0.        ]\n","  [0.036793  ]\n","  [0.01024639]\n","  [0.02207567]\n","  [0.01533473]\n","  [0.00338895]\n","  [0.01811337]\n","  [0.01433556]\n","  [0.0023484 ]\n","  [0.0098977 ]\n","  [0.00094712]\n","  [0.        ]\n","  [0.        ]\n","  [0.00393026]\n","  [0.02251626]\n","  [0.00046644]\n","  [0.        ]\n","  [0.00057096]\n","  [0.00522431]\n","  [0.00724307]\n","  [0.00168312]\n","  [0.00815074]\n","  [0.01446889]\n","  [0.03328482]\n","  [0.00309232]\n","  [0.03835674]\n","  [0.00426269]\n","  [0.00220706]\n","  [0.00092665]\n","  [0.02386485]\n","  [0.03096351]\n","  [0.01736524]\n","  [0.00671375]\n","  [0.        ]\n","  [0.005439  ]\n","  [0.00252629]\n","  [0.00981298]\n","  [0.00394672]\n","  [0.00243872]\n","  [0.        ]\n","  [0.00539494]\n","  [0.        ]\n","  [0.00042216]\n","  [0.00666448]\n","  [0.        ]\n","  [0.0009697 ]\n","  [0.01968862]\n","  [0.00061606]\n","  [0.03229554]\n","  [0.04248788]\n","  [0.11580216]\n","  [0.00183291]\n","  [0.00938541]\n","  [0.00155737]\n","  [0.02762331]\n","  [0.00616602]\n","  [0.06425905]\n","  [0.05202456]\n","  [0.00969042]\n","  [0.02032093]\n","  [0.03290629]\n","  [0.01356468]\n","  [0.09232085]\n","  [0.01034595]\n","  [0.04367963]\n","  [0.0116741 ]\n","  [0.0258684 ]\n","  [0.03331412]\n","  [0.00948365]\n","  [0.01596836]\n","  [0.02193003]\n","  [0.02343319]\n","  [0.01230189]\n","  [0.00869276]\n","  [0.02982467]\n","  [0.01556879]\n","  [0.00460953]\n","  [0.02044134]\n","  [0.03246393]\n","  [0.03384801]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.01136124]\n","  [0.04569164]\n","  [0.00752391]\n","  [0.04668908]\n","  [0.00973706]\n","  [0.00485298]\n","  [0.02357003]\n","  [0.0312935 ]\n","  [0.02810546]\n","  [0.01012093]\n","  [0.00248187]\n","  [0.00163543]\n","  [0.01256405]\n","  [0.02703523]\n","  [0.01104509]\n","  [0.01794259]\n","  [0.01077214]\n","  [0.01413037]\n","  [0.01515656]\n","  [0.03942752]\n","  [0.05620345]\n","  [0.06658783]\n","  [0.00413798]\n","  [0.03779829]\n","  [0.02251003]\n","  [0.1211319 ]\n","  [0.0051976 ]\n","  [0.        ]\n","  [0.00073201]\n","  [0.02107251]\n","  [0.01801517]\n","  [0.05723362]\n","  [0.        ]\n","  [0.07487197]\n","  [0.02927349]\n","  [0.        ]\n","  [0.01422165]\n","  [0.00546053]\n","  [0.00113361]\n","  [0.01603329]\n","  [0.00892514]]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"-RITQG7oKTqZ","outputId":"65c19f91-1867-4c15-b0c5-403a9112a066"},"source":["logistics=[]\n","other=[]\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[254])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.07:\n","        logistics.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_logistic = np.array(logistics)\n","\n","print(topic_logistic)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[[[0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00195896]\n","  [0.        ]\n","  [0.00251992]\n","  [0.07786404]\n","  [0.00328274]\n","  [0.00261381]\n","  [0.04415477]\n","  [0.00790285]\n","  [0.00516678]\n","  [0.00430963]\n","  [0.00106802]\n","  [0.00492266]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00032672]\n","  [0.        ]\n","  [0.00032672]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00500567]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.0006599 ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00178038]\n","  [0.        ]\n","  [0.00042383]\n","  [0.        ]\n","  [0.02858085]\n","  [0.        ]\n","  [0.00821137]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00456373]\n","  [0.        ]\n","  [0.00299137]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00097888]\n","  [0.01164736]\n","  [0.04086327]\n","  [0.0470135 ]\n","  [0.03292536]\n","  [0.0171943 ]\n","  [0.01668381]\n","  [0.00056227]\n","  [0.03811221]\n","  [0.        ]\n","  [0.        ]\n","  [0.01909846]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.01791466]\n","  [0.00914577]\n","  [0.00510284]\n","  [0.        ]\n","  [0.00021732]\n","  [0.00060911]\n","  [0.00152054]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00739665]\n","  [0.00528962]\n","  [0.        ]\n","  [0.00034104]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00020669]\n","  [0.        ]\n","  [0.        ]\n","  [0.00048506]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.0124919 ]\n","  [0.00881224]\n","  [0.00039419]\n","  [0.00084172]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00050422]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.0007563 ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00996198]\n","  [0.00122885]\n","  [0.0008844 ]\n","  [0.00309885]\n","  [0.        ]\n","  [0.00265897]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00031591]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00231144]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.01972922]\n","  [0.01122194]\n","  [0.        ]\n","  [0.01665322]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00822721]\n","  [0.00150505]\n","  [0.        ]\n","  [0.00043576]\n","  [0.00287012]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00143805]\n","  [0.        ]\n","  [0.00233075]\n","  [0.        ]\n","  [0.00611683]]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Z7urosxlKTqb","outputId":"3782cdb8-92e9-4aa3-af56-9fe2067212db"},"source":["nutrition=[]\n","other=[]\n","\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[255])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.19:\n","        nutrition.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_nutrition = np.array(nutrition)\n","\n","print(topic_nutrition)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[[[0.        ]\n","  [0.00100521]\n","  [0.00245683]\n","  ...\n","  [0.        ]\n","  [0.10275314]\n","  [0.00501744]]\n","\n"," [[0.        ]\n","  [0.00100521]\n","  [0.00245683]\n","  ...\n","  [0.        ]\n","  [0.10275314]\n","  [0.00501744]]\n","\n"," [[0.        ]\n","  [0.00100521]\n","  [0.00245683]\n","  ...\n","  [0.        ]\n","  [0.10275314]\n","  [0.00501744]]\n","\n"," ...\n","\n"," [[0.        ]\n","  [0.00100521]\n","  [0.00245683]\n","  ...\n","  [0.        ]\n","  [0.10275314]\n","  [0.00501744]]\n","\n"," [[0.        ]\n","  [0.00100521]\n","  [0.00245683]\n","  ...\n","  [0.        ]\n","  [0.10275314]\n","  [0.00501744]]\n","\n"," [[0.        ]\n","  [0.00100521]\n","  [0.00245683]\n","  ...\n","  [0.        ]\n","  [0.10275314]\n","  [0.00501744]]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"aKyx0LgCKTqd","outputId":"46f1e481-de0e-46e8-b7c5-494ba302856f"},"source":["protection=[]\n","other=[]\n","\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[256])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.19:\n","        protection.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_protection = np.array(protection)\n","\n","print(topic_protection)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[[[0.        ]\n","  [0.00148882]\n","  [0.0009097 ]\n","  [0.        ]\n","  [0.00029749]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.01595279]\n","  [0.01058505]\n","  [0.00268218]\n","  [0.00160019]\n","  [0.00313855]\n","  [0.        ]\n","  [0.        ]\n","  [0.01235975]\n","  [0.        ]\n","  [0.        ]\n","  [0.00048167]\n","  [0.        ]\n","  [0.        ]\n","  [0.00394493]\n","  [0.        ]\n","  [0.00277856]\n","  [0.00181802]\n","  [0.00277856]\n","  [0.0014711 ]\n","  [0.        ]\n","  [0.08709435]\n","  [0.08709435]\n","  [0.01359124]\n","  [0.10394587]\n","  [0.05366072]\n","  [0.00521018]\n","  [0.00827357]\n","  [0.08409071]\n","  [0.07655485]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00608137]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00176151]\n","  [0.00120706]\n","  [0.00393564]\n","  [0.        ]\n","  [0.00103682]\n","  [0.00301094]\n","  [0.        ]\n","  [0.        ]\n","  [0.00372901]\n","  [0.        ]\n","  [0.00153293]\n","  [0.        ]\n","  [0.        ]\n","  [0.0038081 ]\n","  [0.0007695 ]\n","  [0.03099756]\n","  [0.00255571]\n","  [0.03017363]\n","  [0.03096284]\n","  [0.02677537]\n","  [0.0251571 ]\n","  [0.00243669]\n","  [0.03758118]\n","  [0.        ]\n","  [0.04549678]\n","  [0.05133725]\n","  [0.10547706]\n","  [0.00046024]\n","  [0.00293464]\n","  [0.00463533]\n","  [0.01657249]\n","  [0.01663339]\n","  [0.00654616]\n","  [0.00550476]\n","  [0.        ]\n","  [0.        ]\n","  [0.01800305]\n","  [0.03097932]\n","  [0.        ]\n","  [0.00440762]\n","  [0.07310109]\n","  [0.02177403]\n","  [0.00433814]\n","  [0.01235312]\n","  [0.00518901]\n","  [0.00208116]\n","  [0.00707517]\n","  [0.        ]\n","  [0.        ]\n","  [0.00181822]\n","  [0.        ]\n","  [0.        ]\n","  [0.0078557 ]\n","  [0.01389073]\n","  [0.00176559]\n","  [0.        ]\n","  [0.        ]\n","  [0.01016803]\n","  [0.        ]\n","  [0.00372779]\n","  [0.00870579]\n","  [0.00277779]\n","  [0.0043396 ]\n","  [0.01055088]\n","  [0.00224422]\n","  [0.00666006]\n","  [0.00477219]\n","  [0.00029786]\n","  [0.        ]\n","  [0.00106226]\n","  [0.00710107]\n","  [0.07260817]\n","  [0.        ]\n","  [0.00064263]\n","  [0.00040348]\n","  [0.00103581]\n","  [0.00262091]\n","  [0.0049136 ]\n","  [0.        ]\n","  [0.00595361]\n","  [0.00216409]\n","  [0.00338993]\n","  [0.00037639]\n","  [0.        ]\n","  [0.00309382]\n","  [0.        ]\n","  [0.00159314]\n","  [0.        ]\n","  [0.00125701]\n","  [0.00247233]\n","  [0.0023658 ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00335226]\n","  [0.00025565]\n","  [0.00131076]\n","  [0.        ]\n","  [0.00124792]\n","  [0.        ]\n","  [0.00603078]\n","  [0.00052553]\n","  [0.00290852]\n","  [0.01255768]\n","  [0.00403299]\n","  [0.02703494]\n","  [0.00149704]\n","  [0.00266193]\n","  [0.00229708]\n","  [0.00115733]\n","  [0.01598283]\n","  [0.        ]\n","  [0.00103277]\n","  [0.        ]\n","  [0.        ]\n","  [0.00097043]\n","  [0.        ]\n","  [0.00061279]\n","  [0.00457714]\n","  [0.        ]\n","  [0.        ]\n","  [0.00336899]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00242221]\n","  [0.00670635]\n","  [0.00153885]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.0022892 ]\n","  [0.00837274]\n","  [0.        ]\n","  [0.        ]\n","  [0.00825108]\n","  [0.00888328]\n","  [0.04134816]\n","  [0.        ]\n","  [0.        ]\n","  [0.032771  ]\n","  [0.00564717]\n","  [0.00223168]\n","  [0.        ]\n","  [0.        ]\n","  [0.00243005]\n","  [0.00648069]\n","  [0.        ]\n","  [0.01184454]\n","  [0.17201326]\n","  [0.15108313]\n","  [0.11507138]\n","  [0.22950498]\n","  [0.        ]\n","  [0.00524249]\n","  [0.01539358]\n","  [0.00863555]\n","  [0.00892514]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00567581]\n","  [0.00908698]\n","  [0.00530083]\n","  [0.00676835]\n","  [0.00076006]\n","  [0.00151527]\n","  [0.        ]\n","  [0.0192317 ]\n","  [0.03135788]\n","  [0.00337079]\n","  [0.00206647]\n","  [0.00272341]\n","  [0.0026153 ]\n","  [0.00562757]\n","  [0.00714367]\n","  [0.        ]\n","  [0.01938928]\n","  [0.01047851]\n","  [0.0033803 ]\n","  [0.01014698]\n","  [0.01369656]\n","  [0.0218136 ]\n","  [0.05168102]\n","  [0.02697593]\n","  [0.00527132]\n","  [0.        ]\n","  [0.02515459]\n","  [0.        ]\n","  [0.00914243]\n","  [0.06015623]\n","  [0.0383822 ]\n","  [0.01743451]\n","  [0.01767581]\n","  [0.        ]\n","  [0.01566896]\n","  [0.        ]\n","  [0.01193841]\n","  [0.00170497]\n","  [0.00035395]\n","  [0.        ]\n","  [0.00371566]]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"o90S7hQNKTqj","outputId":"4ef9445d-0204-48ce-8c4c-ddd16c9e8ca3"},"source":["shelter=[]\n","other=[]\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[257])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.19:\n","        shelter.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_shelter = np.array(shelter)\n","\n","print(topic_shelter)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[[[0.        ]\n","  [0.00101022]\n","  [0.00061727]\n","  ...\n","  [0.00192137]\n","  [0.        ]\n","  [0.01008488]]\n","\n"," [[0.        ]\n","  [0.00101022]\n","  [0.00061727]\n","  ...\n","  [0.00192137]\n","  [0.        ]\n","  [0.01008488]]\n","\n"," [[0.        ]\n","  [0.00101022]\n","  [0.00061727]\n","  ...\n","  [0.00192137]\n","  [0.        ]\n","  [0.01008488]]\n","\n"," ...\n","\n"," [[0.        ]\n","  [0.00101022]\n","  [0.00061727]\n","  ...\n","  [0.00192137]\n","  [0.        ]\n","  [0.01008488]]\n","\n"," [[0.        ]\n","  [0.00101022]\n","  [0.00061727]\n","  ...\n","  [0.00192137]\n","  [0.        ]\n","  [0.01008488]]\n","\n"," [[0.        ]\n","  [0.00101022]\n","  [0.00061727]\n","  ...\n","  [0.00192137]\n","  [0.        ]\n","  [0.01008488]]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"W6Lo_91EKTqn","outputId":"2246011d-d1f8-40d5-f884-2c50d2c42e4f"},"source":["WASH=[]\n","other=[]\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[258])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.19:\n","        shelter.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_shelter = np.array(shelter)\n","\n","print(topic_shelter)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[[[0.        ]\n","  [0.00101022]\n","  [0.00061727]\n","  ...\n","  [0.00192137]\n","  [0.        ]\n","  [0.01008488]]\n","\n"," [[0.        ]\n","  [0.00101022]\n","  [0.00061727]\n","  ...\n","  [0.00192137]\n","  [0.        ]\n","  [0.01008488]]\n","\n"," [[0.        ]\n","  [0.00101022]\n","  [0.00061727]\n","  ...\n","  [0.00192137]\n","  [0.        ]\n","  [0.01008488]]\n","\n"," ...\n","\n"," [[0.07273377]\n","  [0.06555704]\n","  [0.08066469]\n","  ...\n","  [0.03702365]\n","  [0.09240919]\n","  [0.09870254]]\n","\n"," [[0.07273377]\n","  [0.06555704]\n","  [0.08066469]\n","  ...\n","  [0.03702365]\n","  [0.09240919]\n","  [0.09870254]]\n","\n"," [[0.07273377]\n","  [0.06555704]\n","  [0.08066469]\n","  ...\n","  [0.03702365]\n","  [0.09240919]\n","  [0.09870254]]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"LAuwMhTMKTqp"},"source":["WASH=[]\n","other=[]\n","\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[259])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.19:\n","        WASH.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_WASH = np.array(WASH)\n","\n","print(topic_WASH)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"jPoiejumKTqq","outputId":"92fa1b33-72e1-4a54-d865-115def9da221"},"source":["Camp_Coordination_Mgmt=[]\n","\n","other=[]\n","\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[259])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.04:\n","        Camp_Coordination_Mgmt.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_Camp_Coordination_Mgmt = np.array(Camp_Coordination_Mgmt)\n","\n","print(topic_Camp_Coordination_Mgmt)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[[[5.80364828e-04]\n","  [0.00000000e+00]\n","  [6.68269789e-05]\n","  [0.00000000e+00]\n","  [1.74830840e-04]\n","  [1.36397255e-03]\n","  [0.00000000e+00]\n","  [5.24461831e-04]\n","  [0.00000000e+00]\n","  [3.88790989e-04]\n","  [5.91100996e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [2.14754176e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [2.24878231e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.41847727e-03]\n","  [8.01315388e-04]\n","  [1.41847727e-03]\n","  [1.18874416e-03]\n","  [2.44694870e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [4.52498030e-04]\n","  [6.72891600e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.28665534e-02]\n","  [3.61037302e-03]\n","  [1.23334190e-02]\n","  [1.16451885e-03]\n","  [1.16017738e-03]\n","  [1.27337985e-02]\n","  [4.46739819e-04]\n","  [1.18808857e-03]\n","  [1.18359782e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [3.19215795e-03]\n","  [5.25661019e-05]\n","  [0.00000000e+00]\n","  [3.04661222e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.54485583e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [2.94467798e-04]\n","  [9.04444520e-05]\n","  [1.72130184e-03]\n","  [4.96001087e-02]\n","  [2.36318395e-03]\n","  [1.71937392e-03]\n","  [3.35313308e-03]\n","  [5.49423662e-03]\n","  [6.99407212e-03]\n","  [2.32748678e-03]\n","  [2.36676149e-03]\n","  [2.79901746e-03]\n","  [7.54250815e-04]\n","  [9.68548275e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [9.53436833e-04]\n","  [3.24645671e-03]\n","  [1.52736918e-03]\n","  [3.20589023e-04]\n","  [2.81995933e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [3.25107532e-04]\n","  [0.00000000e+00]\n","  [1.85020099e-04]\n","  [5.03440741e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.61015312e-03]\n","  [1.01649708e-03]\n","  [4.80488431e-04]\n","  [5.25850208e-03]\n","  [5.11749215e-03]\n","  [6.14844252e-03]\n","  [5.65091455e-03]\n","  [3.89533781e-03]\n","  [4.18437208e-03]\n","  [5.01811106e-05]\n","  [2.72111531e-03]\n","  [1.03760798e-03]\n","  [5.54955672e-04]\n","  [8.52238516e-04]\n","  [2.48789878e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.91859163e-03]\n","  [1.49431968e-03]\n","  [3.64329917e-04]\n","  [9.68840058e-05]\n","  [4.07303851e-04]\n","  [3.52675071e-03]\n","  [1.73837798e-03]\n","  [1.16471023e-04]\n","  [5.85312397e-04]\n","  [4.15378355e-04]\n","  [2.08659008e-03]\n","  [5.29958490e-04]\n","  [2.53895140e-04]\n","  [2.83248941e-04]\n","  [1.36343124e-03]\n","  [3.04363498e-05]\n","  [1.02485822e-03]\n","  [1.44382091e-03]\n","  [5.00519355e-04]\n","  [4.60373324e-05]\n","  [8.46228436e-04]\n","  [1.14196035e-03]\n","  [2.76495081e-04]\n","  [8.10157494e-04]\n","  [0.00000000e+00]\n","  [6.21857727e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [6.15601859e-05]\n","  [0.00000000e+00]\n","  [2.25930127e-03]\n","  [3.71620981e-04]\n","  [0.00000000e+00]\n","  [1.69944940e-04]\n","  [3.75603108e-05]\n","  [1.49782865e-04]\n","  [0.00000000e+00]\n","  [1.04768926e-04]\n","  [1.19829815e-04]\n","  [2.21511898e-04]\n","  [0.00000000e+00]\n","  [4.37431304e-05]\n","  [8.99992475e-05]\n","  [9.11584821e-05]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [3.91092391e-04]\n","  [8.77471145e-04]\n","  [8.50181515e-05]\n","  [4.44255942e-04]\n","  [1.95919539e-04]\n","  [1.51735471e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.54521001e-04]\n","  [2.39620994e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [2.29018578e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [9.78649393e-04]\n","  [0.00000000e+00]\n","  [1.01740204e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.42885522e-04]\n","  [0.00000000e+00]\n","  [2.42450880e-04]\n","  [8.39857269e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [2.19341223e-04]\n","  [1.02609229e-03]\n","  [2.20822078e-03]\n","  [6.01113420e-04]\n","  [2.37304422e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.12017412e-04]\n","  [5.34886979e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [6.77636013e-03]\n","  [6.51498134e-05]\n","  [1.50416763e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.69165501e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.69389835e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [5.36075307e-04]\n","  [3.20415836e-04]\n","  [2.70944596e-04]\n","  [2.80080842e-04]\n","  [3.47031654e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [2.79468451e-03]\n","  [2.97902322e-03]\n","  [1.11428751e-03]\n","  [0.00000000e+00]\n","  [5.83258471e-04]\n","  [6.07391724e-04]\n","  [5.90576870e-05]\n","  [8.31005557e-04]\n","  [0.00000000e+00]\n","  [5.81364812e-05]\n","  [1.35044783e-04]\n","  [0.00000000e+00]\n","  [1.25469096e-03]\n","  [3.52154142e-03]\n","  [2.80426243e-03]\n","  [2.65755556e-03]\n","  [5.76963449e-03]\n","  [5.42126881e-03]\n","  [8.71653681e-04]\n","  [5.49368884e-03]\n","  [7.34252836e-04]\n","  [6.71606798e-04]\n","  [6.74191182e-04]\n","  [1.90503985e-03]\n","  [1.03995688e-02]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [3.83682567e-04]\n","  [0.00000000e+00]\n","  [1.42458307e-03]\n","  [5.00991475e-04]\n","  [2.08012095e-04]\n","  [0.00000000e+00]\n","  [8.81270318e-03]]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"024cE0ldKTqs","outputId":"c31abe1c-da3a-439c-c418-4b3182c02aa8"},"source":["Early_Recovery=[]\n","other=[]\n","\n","\n","from sklearn.metrics.pairwise import cosine_similarity\n","\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[260])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.1:\n","        Early_Recovery.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_Early_Recovery = np.array(Early_Recovery)\n","\n","print(topic_Early_Recovery)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[[[0.        ]\n","  [0.00097696]\n","  [0.00059695]\n","  [0.        ]\n","  [0.00122442]\n","  [0.        ]\n","  [0.        ]\n","  [0.00078081]\n","  [0.        ]\n","  [0.00204129]\n","  [0.00282575]\n","  [0.00084292]\n","  [0.00330656]\n","  [0.        ]\n","  [0.00193463]\n","  [0.00518816]\n","  [0.        ]\n","  [0.        ]\n","  [0.00303604]\n","  [0.        ]\n","  [0.        ]\n","  [0.00494533]\n","  [0.        ]\n","  [0.00238147]\n","  [0.        ]\n","  [0.00238147]\n","  [0.00231814]\n","  [0.00285167]\n","  [0.00056867]\n","  [0.00056867]\n","  [0.        ]\n","  [0.0089161 ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.0008203 ]\n","  [0.00256928]\n","  [0.        ]\n","  [0.00647226]\n","  [0.        ]\n","  [0.00626284]\n","  [0.00623949]\n","  [0.        ]\n","  [0.00480518]\n","  [0.0063896 ]\n","  [0.00636545]\n","  [0.        ]\n","  [0.        ]\n","  [0.00603217]\n","  [0.0010561 ]\n","  [0.00235102]\n","  [0.        ]\n","  [0.00045358]\n","  [0.01096796]\n","  [0.        ]\n","  [0.        ]\n","  [0.00168682]\n","  [0.0022164 ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00151027]\n","  [0.00021572]\n","  [0.00206108]\n","  [0.01252669]\n","  [0.00980124]\n","  [0.00205877]\n","  [0.01323756]\n","  [0.00790143]\n","  [0.0069508 ]\n","  [0.0024158 ]\n","  [0.00286232]\n","  [0.00758017]\n","  [0.1019008 ]\n","  [0.01215509]\n","  [0.06399821]\n","  [0.00721977]\n","  [0.01422516]\n","  [0.02787982]\n","  [0.00741629]\n","  [0.00536826]\n","  [0.00517354]\n","  [0.        ]\n","  [0.        ]\n","  [0.0085188 ]\n","  [0.00329929]\n","  [0.        ]\n","  [0.00320437]\n","  [0.00120335]\n","  [0.        ]\n","  [0.        ]\n","  [0.00252748]\n","  [0.01259433]\n","  [0.04447356]\n","  [0.00090157]\n","  [0.00057152]\n","  [0.00062467]\n","  [0.00566548]\n","  [0.00469756]\n","  [0.00859589]\n","  [0.00158737]\n","  [0.01441817]\n","  [0.00511608]\n","  [0.        ]\n","  [0.0025376 ]\n","  [0.00758025]\n","  [0.        ]\n","  [0.005891  ]\n","  [0.00802533]\n","  [0.00121519]\n","  [0.0011694 ]\n","  [0.00283513]\n","  [0.        ]\n","  [0.00080932]\n","  [0.00078279]\n","  [0.        ]\n","  [0.00629568]\n","  [0.00148898]\n","  [0.01460095]\n","  [0.00269905]\n","  [0.00306418]\n","  [0.00262206]\n","  [0.00746599]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00112744]\n","  [0.        ]\n","  [0.00196723]\n","  [0.00036292]\n","  [0.        ]\n","  [0.00082735]\n","  [0.        ]\n","  [0.0007254 ]\n","  [0.        ]\n","  [0.05895723]\n","  [0.00184312]\n","  [0.00134651]\n","  [0.        ]\n","  [0.00390293]\n","  [0.01546181]\n","  [0.0020131 ]\n","  [0.00107274]\n","  [0.        ]\n","  [0.00115714]\n","  [0.00529392]\n","  [0.00032978]\n","  [0.00285742]\n","  [0.00977786]\n","  [0.00284666]\n","  [0.00130455]\n","  [0.00168956]\n","  [0.00065491]\n","  [0.00420663]\n","  [0.00048401]\n","  [0.00177204]\n","  [0.        ]\n","  [0.00210733]\n","  [0.0004518 ]\n","  [0.01103702]\n","  [0.01223252]\n","  [0.00144691]\n","  [0.00636643]\n","  [0.01769914]\n","  [0.00361503]\n","  [0.00240665]\n","  [0.00458856]\n","  [0.00177466]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00097794]\n","  [0.0010785 ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00200978]\n","  [0.00793884]\n","  [0.0013632 ]\n","  [0.        ]\n","  [0.0002652 ]\n","  [0.00142363]\n","  [0.00646017]\n","  [0.00264729]\n","  [0.00032655]\n","  [0.00437187]\n","  [0.        ]\n","  [0.00312479]\n","  [0.00141318]\n","  [0.00411471]\n","  [0.00106307]\n","  [0.00464489]\n","  [0.00022752]\n","  [0.        ]\n","  [0.        ]\n","  [0.01167847]\n","  [0.00100954]\n","  [0.00092593]\n","  [0.        ]\n","  [0.00931903]\n","  [0.00108321]\n","  [0.00058765]\n","  [0.00147136]\n","  [0.        ]\n","  [0.        ]\n","  [0.00083021]\n","  [0.        ]\n","  [0.        ]\n","  [0.00975458]\n","  [0.00031802]\n","  [0.01090957]\n","  [0.00113882]\n","  [0.00498755]\n","  [0.01538161]\n","  [0.00266451]\n","  [0.00685762]\n","  [0.00812128]\n","  [0.00516114]\n","  [0.00316405]\n","  [0.00074463]\n","  [0.00195294]\n","  [0.0003517 ]\n","  [0.0017462 ]\n","  [0.00108926]\n","  [0.00051932]\n","  [0.00098147]\n","  [0.00251391]\n","  [0.00261116]\n","  [0.00389776]\n","  [0.00298211]\n","  [0.00113044]\n","  [0.00532874]\n","  [0.00785953]\n","  [0.        ]\n","  [0.00790064]\n","  [0.01492763]\n","  [0.00979593]\n","  [0.00538819]\n","  [0.00654663]\n","  [0.0039692 ]\n","  [0.        ]\n","  [0.002922  ]\n","  [0.00894285]\n","  [0.        ]\n","  [0.00798659]\n","  [0.00298348]\n","  [0.00267032]\n","  [0.        ]\n","  [0.02153342]]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"DekkJnCYKTqu","outputId":"af482112-31d1-43bf-fac8-1d6aff674240"},"source":["Education=[]\n","other=[]\n","\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[261])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.19:\n","        Education.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_Education = np.array(Education)\n","\n","print(topic_Education)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[[[0.        ]\n","  [0.023505  ]\n","  [0.00478736]\n","  [0.        ]\n","  [0.00426974]\n","  [0.01001268]\n","  [0.        ]\n","  [0.        ]\n","  [0.03434425]\n","  [0.0005064 ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00062076]\n","  [0.00400564]\n","  [0.        ]\n","  [0.        ]\n","  [0.00276527]\n","  [0.        ]\n","  [0.00415967]\n","  [0.01698582]\n","  [0.00171222]\n","  [0.00797583]\n","  [0.        ]\n","  [0.00797583]\n","  [0.01196455]\n","  [0.00637436]\n","  [0.0012438 ]\n","  [0.0012438 ]\n","  [0.        ]\n","  [0.00221019]\n","  [0.00669707]\n","  [0.00997051]\n","  [0.        ]\n","  [0.01788012]\n","  [0.00823034]\n","  [0.0039902 ]\n","  [0.0039188 ]\n","  [0.00401611]\n","  [0.003792  ]\n","  [0.00377786]\n","  [0.00394903]\n","  [0.        ]\n","  [0.00386875]\n","  [0.00385413]\n","  [0.00956914]\n","  [0.        ]\n","  [0.        ]\n","  [0.00346485]\n","  [0.01335126]\n","  [0.        ]\n","  [0.01488093]\n","  [0.        ]\n","  [0.        ]\n","  [0.00510509]\n","  [0.02854421]\n","  [0.00453888]\n","  [0.07138222]\n","  [0.        ]\n","  [0.01123303]\n","  [0.00402725]\n","  [0.00353415]\n","  [0.00240482]\n","  [0.        ]\n","  [0.00538098]\n","  [0.00240212]\n","  [0.00849877]\n","  [0.00077649]\n","  [0.        ]\n","  [0.00239725]\n","  [0.        ]\n","  [0.00558112]\n","  [0.        ]\n","  [0.01051288]\n","  [0.00044038]\n","  [0.00561591]\n","  [0.00266113]\n","  [0.00264284]\n","  [0.00795767]\n","  [0.        ]\n","  [0.00690351]\n","  [0.        ]\n","  [0.        ]\n","  [0.07234863]\n","  [0.06351847]\n","  [0.        ]\n","  [0.00481982]\n","  [0.0010929 ]\n","  [0.00416681]\n","  [0.00415087]\n","  [0.00196997]\n","  [0.        ]\n","  [0.005405  ]\n","  [0.00246172]\n","  [0.        ]\n","  [0.00136627]\n","  [0.00173973]\n","  [0.        ]\n","  [0.0043098 ]\n","  [0.00653615]\n","  [0.00221518]\n","  [0.00168937]\n","  [0.        ]\n","  [0.00277513]\n","  [0.01945815]\n","  [0.        ]\n","  [0.00356686]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00063096]\n","  [0.        ]\n","  [0.00531045]\n","  [0.        ]\n","  [0.01567487]\n","  [0.        ]\n","  [0.00203281]\n","  [0.00194129]\n","  [0.00361842]\n","  [0.        ]\n","  [0.        ]\n","  [0.00540487]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.0014991 ]\n","  [0.        ]\n","  [0.00840929]\n","  [0.04105575]\n","  [0.0158286 ]\n","  [0.00028193]\n","  [0.03239908]\n","  [0.00030487]\n","  [0.        ]\n","  [0.00160366]\n","  [0.0047312 ]\n","  [0.00056592]\n","  [0.        ]\n","  [0.        ]\n","  [0.00126011]\n","  [0.0017123 ]\n","  [0.00027871]\n","  [0.        ]\n","  [0.00034116]\n","  [0.0015608 ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00214074]\n","  [0.00263756]\n","  [0.00029684]\n","  [0.        ]\n","  [0.00286483]\n","  [0.        ]\n","  [0.        ]\n","  [0.00166106]\n","  [0.        ]\n","  [0.00510375]\n","  [0.01284642]\n","  [0.        ]\n","  [0.        ]\n","  [0.00092853]\n","  [0.00100633]\n","  [0.00058634]\n","  [0.00572709]\n","  [0.00291433]\n","  [0.00277826]\n","  [0.0064471 ]\n","  [0.00298299]\n","  [0.001009  ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00115882]\n","  [0.        ]\n","  [0.00073621]\n","  [0.00643233]\n","  [0.        ]\n","  [0.        ]\n","  [0.00073012]\n","  [0.        ]\n","  [0.01209716]\n","  [0.        ]\n","  [0.00236846]\n","  [0.0480678 ]\n","  [0.08053931]\n","  [0.01737046]\n","  [0.04999655]\n","  [0.03135629]\n","  [0.00720451]\n","  [0.0067619 ]\n","  [0.00463638]\n","  [0.00359988]\n","  [0.        ]\n","  [0.00164142]\n","  [0.09654225]\n","  [0.00755547]\n","  [0.02146794]\n","  [0.02265633]\n","  [0.00869799]\n","  [0.02205161]\n","  [0.00148401]\n","  [0.03220914]\n","  [0.02364399]\n","  [0.00036723]\n","  [0.02502374]\n","  [0.01939762]\n","  [0.0018386 ]\n","  [0.        ]\n","  [0.01208606]\n","  [0.04604625]\n","  [0.0597387 ]\n","  [0.00695576]\n","  [0.03481234]\n","  [0.032879  ]\n","  [0.27393179]\n","  [0.07249291]\n","  [0.04096988]\n","  [0.04155172]\n","  [0.03940846]\n","  [0.18867846]\n","  [0.27483963]\n","  [0.08208399]\n","  [0.03440792]\n","  [0.03846167]\n","  [0.01402411]\n","  [0.00952971]\n","  [0.0371045 ]\n","  [0.00949846]\n","  [0.04237027]\n","  [0.02655681]\n","  [0.01801977]\n","  [0.02869888]\n","  [0.08406495]\n","  [0.01613212]\n","  [0.00672502]\n","  [0.        ]\n","  [0.00077641]\n","  [0.00478187]\n","  [0.        ]\n","  [0.01438983]\n","  [0.00278605]\n","  [0.00166819]\n","  [0.00563758]\n","  [0.0596493 ]\n","  [0.00249876]\n","  [0.00367285]\n","  [0.00250749]\n","  [0.        ]\n","  [0.        ]\n","  [0.00766408]\n","  [0.00711051]]\n","\n"," [[0.        ]\n","  [0.023505  ]\n","  [0.00478736]\n","  [0.        ]\n","  [0.00426974]\n","  [0.01001268]\n","  [0.        ]\n","  [0.        ]\n","  [0.03434425]\n","  [0.0005064 ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00062076]\n","  [0.00400564]\n","  [0.        ]\n","  [0.        ]\n","  [0.00276527]\n","  [0.        ]\n","  [0.00415967]\n","  [0.01698582]\n","  [0.00171222]\n","  [0.00797583]\n","  [0.        ]\n","  [0.00797583]\n","  [0.01196455]\n","  [0.00637436]\n","  [0.0012438 ]\n","  [0.0012438 ]\n","  [0.        ]\n","  [0.00221019]\n","  [0.00669707]\n","  [0.00997051]\n","  [0.        ]\n","  [0.01788012]\n","  [0.00823034]\n","  [0.0039902 ]\n","  [0.0039188 ]\n","  [0.00401611]\n","  [0.003792  ]\n","  [0.00377786]\n","  [0.00394903]\n","  [0.        ]\n","  [0.00386875]\n","  [0.00385413]\n","  [0.00956914]\n","  [0.        ]\n","  [0.        ]\n","  [0.00346485]\n","  [0.01335126]\n","  [0.        ]\n","  [0.01488093]\n","  [0.        ]\n","  [0.        ]\n","  [0.00510509]\n","  [0.02854421]\n","  [0.00453888]\n","  [0.07138222]\n","  [0.        ]\n","  [0.01123303]\n","  [0.00402725]\n","  [0.00353415]\n","  [0.00240482]\n","  [0.        ]\n","  [0.00538098]\n","  [0.00240212]\n","  [0.00849877]\n","  [0.00077649]\n","  [0.        ]\n","  [0.00239725]\n","  [0.        ]\n","  [0.00558112]\n","  [0.        ]\n","  [0.01051288]\n","  [0.00044038]\n","  [0.00561591]\n","  [0.00266113]\n","  [0.00264284]\n","  [0.00795767]\n","  [0.        ]\n","  [0.00690351]\n","  [0.        ]\n","  [0.        ]\n","  [0.07234863]\n","  [0.06351847]\n","  [0.        ]\n","  [0.00481982]\n","  [0.0010929 ]\n","  [0.00416681]\n","  [0.00415087]\n","  [0.00196997]\n","  [0.        ]\n","  [0.005405  ]\n","  [0.00246172]\n","  [0.        ]\n","  [0.00136627]\n","  [0.00173973]\n","  [0.        ]\n","  [0.0043098 ]\n","  [0.00653615]\n","  [0.00221518]\n","  [0.00168937]\n","  [0.        ]\n","  [0.00277513]\n","  [0.01945815]\n","  [0.        ]\n","  [0.00356686]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00063096]\n","  [0.        ]\n","  [0.00531045]\n","  [0.        ]\n","  [0.01567487]\n","  [0.        ]\n","  [0.00203281]\n","  [0.00194129]\n","  [0.00361842]\n","  [0.        ]\n","  [0.        ]\n","  [0.00540487]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.        ]\n","  [0.0014991 ]\n","  [0.        ]\n","  [0.00840929]\n","  [0.04105575]\n","  [0.0158286 ]\n","  [0.00028193]\n","  [0.03239908]\n","  [0.00030487]\n","  [0.        ]\n","  [0.00160366]\n","  [0.0047312 ]\n","  [0.00056592]\n","  [0.        ]\n","  [0.        ]\n","  [0.00126011]\n","  [0.0017123 ]\n","  [0.00027871]\n","  [0.        ]\n","  [0.00034116]\n","  [0.0015608 ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00214074]\n","  [0.00263756]\n","  [0.00029684]\n","  [0.        ]\n","  [0.00286483]\n","  [0.        ]\n","  [0.        ]\n","  [0.00166106]\n","  [0.        ]\n","  [0.00510375]\n","  [0.01284642]\n","  [0.        ]\n","  [0.        ]\n","  [0.00092853]\n","  [0.00100633]\n","  [0.00058634]\n","  [0.00572709]\n","  [0.00291433]\n","  [0.00277826]\n","  [0.0064471 ]\n","  [0.00298299]\n","  [0.001009  ]\n","  [0.        ]\n","  [0.        ]\n","  [0.00115882]\n","  [0.        ]\n","  [0.00073621]\n","  [0.00643233]\n","  [0.        ]\n","  [0.        ]\n","  [0.00073012]\n","  [0.        ]\n","  [0.01209716]\n","  [0.        ]\n","  [0.00236846]\n","  [0.0480678 ]\n","  [0.08053931]\n","  [0.01737046]\n","  [0.04999655]\n","  [0.03135629]\n","  [0.00720451]\n","  [0.0067619 ]\n","  [0.00463638]\n","  [0.00359988]\n","  [0.        ]\n","  [0.00164142]\n","  [0.09654225]\n","  [0.00755547]\n","  [0.02146794]\n","  [0.02265633]\n","  [0.00869799]\n","  [0.02205161]\n","  [0.00148401]\n","  [0.03220914]\n","  [0.02364399]\n","  [0.00036723]\n","  [0.02502374]\n","  [0.01939762]\n","  [0.0018386 ]\n","  [0.        ]\n","  [0.01208606]\n","  [0.04604625]\n","  [0.0597387 ]\n","  [0.00695576]\n","  [0.03481234]\n","  [0.032879  ]\n","  [0.27393179]\n","  [0.07249291]\n","  [0.04096988]\n","  [0.04155172]\n","  [0.03940846]\n","  [0.18867846]\n","  [0.27483963]\n","  [0.08208399]\n","  [0.03440792]\n","  [0.03846167]\n","  [0.01402411]\n","  [0.00952971]\n","  [0.0371045 ]\n","  [0.00949846]\n","  [0.04237027]\n","  [0.02655681]\n","  [0.01801977]\n","  [0.02869888]\n","  [0.08406495]\n","  [0.01613212]\n","  [0.00672502]\n","  [0.        ]\n","  [0.00077641]\n","  [0.00478187]\n","  [0.        ]\n","  [0.01438983]\n","  [0.00278605]\n","  [0.00166819]\n","  [0.00563758]\n","  [0.0596493 ]\n","  [0.00249876]\n","  [0.00367285]\n","  [0.00250749]\n","  [0.        ]\n","  [0.        ]\n","  [0.00766408]\n","  [0.00711051]]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"sVM_XcZvKTqv","outputId":"19f70e14-a295-4cf6-98b2-c26dac75696c"},"source":["Emergency_Telecom=[]\n","other=[]\n","\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[262])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.02:\n","        Emergency_Telecom.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_Emergency_Telecom = np.array(Emergency_Telecom)\n","\n","print(topic_Emergency_Telecom)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[[[0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [6.65450051e-05]\n","  [0.00000000e+00]\n","  [6.84806276e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.30225574e-03]\n","  [2.69985448e-03]\n","  [0.00000000e+00]\n","  [9.82874793e-03]\n","  [0.00000000e+00]\n","  [1.88657528e-03]\n","  [6.68882628e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [3.23230734e-04]\n","  [0.00000000e+00]\n","  [2.91733011e-03]\n","  [2.73553019e-03]\n","  [0.00000000e+00]\n","  [3.10763686e-03]\n","  [4.06667662e-04]\n","  [3.10763686e-03]\n","  [8.22665197e-04]\n","  [3.53920632e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.84288354e-02]\n","  [5.21878342e-04]\n","  [0.00000000e+00]\n","  [9.25344283e-04]\n","  [3.48333204e-04]\n","  [1.92408097e-04]\n","  [2.79847607e-03]\n","  [4.58066192e-03]\n","  [2.81664527e-03]\n","  [4.43245099e-03]\n","  [4.41592625e-03]\n","  [2.76960187e-03]\n","  [8.16193855e-03]\n","  [4.52216324e-03]\n","  [4.50507035e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [3.94027881e-04]\n","  [2.70003428e-04]\n","  [0.00000000e+00]\n","  [1.63092723e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [4.48327336e-04]\n","  [6.88508564e-05]\n","  [2.06138472e-03]\n","  [1.14336009e-03]\n","  [3.04813697e-03]\n","  [2.05907590e-03]\n","  [1.01645331e-02]\n","  [1.63373648e-03]\n","  [5.53112012e-03]\n","  [1.68128384e-03]\n","  [2.57480175e-03]\n","  [3.78376999e-03]\n","  [1.03351244e-02]\n","  [1.47461512e-03]\n","  [1.48248697e-02]\n","  [1.31288157e-03]\n","  [3.73270614e-03]\n","  [3.70704869e-03]\n","  [5.11592436e-03]\n","  [0.00000000e+00]\n","  [1.27916154e-02]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [2.41622843e-03]\n","  [1.48492903e-03]\n","  [0.00000000e+00]\n","  [2.25354537e-03]\n","  [3.57694737e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [3.72634120e-03]\n","  [4.32656967e-03]\n","  [4.66741796e-03]\n","  [4.19863479e-03]\n","  [1.79607381e-02]\n","  [1.13541098e-02]\n","  [1.05196603e-02]\n","  [1.51606582e-02]\n","  [8.65605525e-03]\n","  [1.02110104e-03]\n","  [1.20053613e-02]\n","  [2.83666655e-03]\n","  [9.44834740e-03]\n","  [1.04986884e-02]\n","  [1.70584099e-02]\n","  [0.00000000e+00]\n","  [1.66771469e-03]\n","  [8.85203383e-03]\n","  [4.97084741e-03]\n","  [1.66407731e-03]\n","  [2.21258904e-03]\n","  [2.95295366e-05]\n","  [3.88994995e-03]\n","  [3.49047766e-03]\n","  [6.66264764e-05]\n","  [3.56455340e-03]\n","  [2.37614434e-04]\n","  [3.63066766e-03]\n","  [2.25576398e-03]\n","  [1.54622179e-04]\n","  [1.43748873e-04]\n","  [1.08303891e-03]\n","  [9.26787085e-05]\n","  [1.11390036e-02]\n","  [4.39643579e-03]\n","  [6.09632727e-03]\n","  [7.00918563e-04]\n","  [9.68158733e-04]\n","  [2.30293136e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [3.29545884e-05]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.32299502e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.34391906e-03]\n","  [9.72154988e-04]\n","  [3.25777814e-05]\n","  [0.00000000e+00]\n","  [3.98776825e-05]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.17554078e-04]\n","  [2.29774201e-04]\n","  [6.85119444e-05]\n","  [1.52667706e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [2.05530793e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [2.67947658e-04]\n","  [0.00000000e+00]\n","  [4.34143883e-04]\n","  [0.00000000e+00]\n","  [1.37073601e-04]\n","  [4.72544090e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.17941044e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [5.00041091e-04]\n","  [1.72110556e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.70687706e-04]\n","  [3.74574727e-04]\n","  [2.17543262e-04]\n","  [7.34969746e-04]\n","  [6.15219262e-05]\n","  [1.02779695e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [7.21027341e-04]\n","  [4.21066020e-04]\n","  [5.82397080e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [3.83729743e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [4.03172284e-03]\n","  [1.98381232e-04]\n","  [3.43680448e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [4.53071395e-04]\n","  [3.00480296e-04]\n","  [0.00000000e+00]\n","  [3.02317182e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.34558127e-03]\n","  [0.00000000e+00]\n","  [1.54480524e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [2.26688598e-04]\n","  [1.69473275e-03]\n","  [1.99539624e-04]\n","  [1.42933443e-02]\n","  [1.28770874e-02]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [4.70792967e-04]\n","  [3.07128928e-03]\n","  [1.79830702e-04]\n","  [6.26233029e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [7.56128539e-05]\n","  [8.01087271e-04]\n","  [3.06374181e-03]\n","  [2.43970893e-03]\n","  [1.73405544e-03]\n","  [6.03416387e-03]\n","  [2.04381886e-02]\n","  [1.32709306e-03]\n","  [1.63357359e-03]\n","  [1.34148027e-02]\n","  [2.65855618e-03]\n","  [2.52303054e-03]\n","  [2.42765143e-03]\n","  [3.89987213e-04]\n","  [1.31794928e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [4.89094311e-03]\n","  [0.00000000e+00]\n","  [3.16698494e-04]\n","  [1.79170175e-03]\n","  [4.15572300e-03]]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"mEBubyjkKTqy","outputId":"1a77cc0f-9ae4-4a9a-e5cd-fde2fd55762c"},"source":["Food_Security=[]\n","other=[]\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[263])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.18:\n","        Food_Security.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_Food_Security = np.array(Food_Security)\n","\n","print(topic_Food_Security)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[[[4.26920855e-03]\n","  [1.12937022e-03]\n","  [1.65616931e-02]\n","  [0.00000000e+00]\n","  [3.76113108e-03]\n","  [1.88492236e-03]\n","  [1.54821307e-03]\n","  [1.92899089e-03]\n","  [4.20905065e-03]\n","  [2.55563086e-03]\n","  [0.00000000e+00]\n","  [8.64711837e-04]\n","  [1.58720550e-03]\n","  [1.41465030e-03]\n","  [7.01164084e-04]\n","  [9.73470333e-03]\n","  [0.00000000e+00]\n","  [4.89138454e-02]\n","  [2.28820079e-02]\n","  [0.00000000e+00]\n","  [2.75284942e-02]\n","  [4.87952795e-02]\n","  [9.04956443e-03]\n","  [3.43980425e-02]\n","  [7.67479027e-03]\n","  [3.43980425e-02]\n","  [2.47212679e-02]\n","  [8.15651691e-02]\n","  [7.02448805e-04]\n","  [7.02448805e-04]\n","  [0.00000000e+00]\n","  [2.85914779e-03]\n","  [1.88303314e-02]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [2.85860974e-02]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [9.51887959e-04]\n","  [3.65269773e-02]\n","  [5.61015564e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [3.93900518e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.32241315e-02]\n","  [2.45374629e-03]\n","  [4.13452717e-03]\n","  [2.69851596e-03]\n","  [0.00000000e+00]\n","  [3.11022481e-04]\n","  [1.75492464e-03]\n","  [4.73403102e-03]\n","  [0.00000000e+00]\n","  [8.11536141e-02]\n","  [4.72872875e-03]\n","  [6.71592827e-03]\n","  [2.13637799e-03]\n","  [1.64592248e-04]\n","  [0.00000000e+00]\n","  [6.71630042e-03]\n","  [5.59270619e-03]\n","  [0.00000000e+00]\n","  [1.11127214e-03]\n","  [1.89391863e-03]\n","  [5.93633696e-03]\n","  [1.86745728e-02]\n","  [0.00000000e+00]\n","  [7.34784670e-03]\n","  [0.00000000e+00]\n","  [2.41814059e-02]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [2.31480279e-03]\n","  [0.00000000e+00]\n","  [5.75350751e-03]\n","  [5.77627035e-04]\n","  [2.20227951e-03]\n","  [4.38770440e-03]\n","  [3.85828611e-03]\n","  [6.12300469e-03]\n","  [3.47621330e-02]\n","  [7.41481953e-03]\n","  [1.00008207e-02]\n","  [3.70957624e-03]\n","  [1.41098954e-02]\n","  [4.00675595e-03]\n","  [9.50179702e-03]\n","  [1.42918097e-03]\n","  [9.52680533e-03]\n","  [3.72672719e-03]\n","  [9.55099697e-03]\n","  [4.40020414e-03]\n","  [4.52247896e-02]\n","  [4.11442748e-02]\n","  [1.87204129e-01]\n","  [1.10086140e-01]\n","  [8.52488560e-03]\n","  [1.00372511e-01]\n","  [4.89238790e-02]\n","  [7.33856913e-03]\n","  [6.24580915e-03]\n","  [4.96390873e-02]\n","  [5.97927853e-03]\n","  [0.00000000e+00]\n","  [1.50784199e-02]\n","  [4.88214430e-03]\n","  [1.52677316e-02]\n","  [1.10305693e-03]\n","  [1.70370917e-03]\n","  [6.60813640e-03]\n","  [1.62321803e-03]\n","  [3.51035948e-02]\n","  [2.65521457e-03]\n","  [0.00000000e+00]\n","  [1.46458339e-03]\n","  [6.94157521e-03]\n","  [9.00531236e-03]\n","  [1.22035152e-03]\n","  [0.00000000e+00]\n","  [6.70534919e-04]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [3.01814063e-03]\n","  [1.94388524e-03]\n","  [9.82867069e-03]\n","  [3.71790236e-02]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [9.71484712e-03]\n","  [6.64152841e-04]\n","  [9.45833310e-02]\n","  [0.00000000e+00]\n","  [9.10118805e-02]\n","  [7.23807909e-03]\n","  [8.33336576e-02]\n","  [8.15519336e-04]\n","  [7.00041810e-03]\n","  [6.19568433e-04]\n","  [5.40965881e-03]\n","  [4.17404314e-03]\n","  [0.00000000e+00]\n","  [1.34616959e-03]\n","  [1.03534748e-01]\n","  [2.33223315e-02]\n","  [3.83348389e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [4.28196836e-03]\n","  [0.00000000e+00]\n","  [2.09759945e-03]\n","  [1.13666843e-03]\n","  [3.74000347e-03]\n","  [1.60249467e-03]\n","  [0.00000000e+00]\n","  [3.03744360e-03]\n","  [6.93175032e-03]\n","  [0.00000000e+00]\n","  [8.54761798e-04]\n","  [9.01477331e-02]\n","  [6.88973295e-03]\n","  [6.12951067e-02]\n","  [3.07268967e-02]\n","  [2.95446952e-02]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [1.64938147e-03]\n","  [0.00000000e+00]\n","  [4.03572929e-03]\n","  [0.00000000e+00]\n","  [1.09025910e-02]\n","  [1.58095700e-03]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [7.54989079e-04]\n","  [5.66414454e-03]\n","  [1.36582093e-02]\n","  [4.40847439e-02]\n","  [0.00000000e+00]\n","  [1.52211338e-02]\n","  [3.07226184e-02]\n","  [7.46982929e-03]\n","  [5.26034696e-04]\n","  [0.00000000e+00]\n","  [1.34713882e-02]\n","  [7.44673536e-03]\n","  [1.60747380e-02]\n","  [5.70803211e-03]\n","  [1.17651346e-03]\n","  [1.68885490e-02]\n","  [2.10152568e-03]\n","  [6.02238849e-04]\n","  [1.27902641e-02]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [0.00000000e+00]\n","  [3.27212129e-03]\n","  [7.67214875e-04]\n","  [1.88621916e-03]\n","  [1.91526634e-03]\n","  [2.73815835e-04]\n","  [0.00000000e+00]\n","  [2.78513483e-03]\n","  [1.93992541e-02]\n","  [2.24625088e-02]\n","  [2.67383265e-03]\n","  [5.58340501e-04]\n","  [1.83959608e-03]\n","  [3.30646878e-04]\n","  [4.20497482e-04]\n","  [9.68463734e-04]\n","  [1.26661161e-01]\n","  [1.21437905e-03]\n","  [3.15680054e-03]\n","  [1.05656122e-02]\n","  [2.31552983e-02]\n","  [2.59744541e-03]\n","  [6.98915988e-03]\n","  [0.00000000e+00]\n","  [1.70525806e-03]\n","  [1.24267257e-02]\n","  [1.50014895e-03]\n","  [2.13616500e-03]\n","  [2.70061032e-03]\n","  [0.00000000e+00]\n","  [1.90135955e-03]\n","  [8.05732304e-03]\n","  [5.29010894e-03]\n","  [1.24364257e-02]\n","  [1.20312806e-03]\n","  [0.00000000e+00]\n","  [4.01548965e-03]\n","  [1.42918739e-02]\n","  [1.05807048e-01]\n","  [1.12964541e-03]\n","  [4.42924590e-02]\n","  [0.00000000e+00]]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"ArzgPpwZKTq1","outputId":"54383593-7791-4d35-f0b7-3a5904e5c0d0"},"source":["Humanitarian_Emergency_Relief=[]\n","other=[]\n","\n","\n","from sklearn.metrics.pairwise import cosine_similarity\n","\n","c_score=cosine_similarity(tfidf_train_data[0:252], tfidf_train_data[264])\n","for x in np.nditer(c_score):\n","#     print(x)\n","    if x > 0.03:\n","        Humanitarian_Emergency_Relief.append(c_score)\n","    else:\n","        other.append(c_score)\n","    \n","topic_Humanitarian_Emergency_Relief = np.array(Humanitarian_Emergency_Relief)\n","\n","print(topic_Humanitarian_Emergency_Relief)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[[[0.004203  ]\n","  [0.        ]\n","  [0.        ]\n","  ...\n","  [0.00203111]\n","  [0.00150292]\n","  [0.0123978 ]]\n","\n"," [[0.004203  ]\n","  [0.        ]\n","  [0.        ]\n","  ...\n","  [0.00203111]\n","  [0.00150292]\n","  [0.0123978 ]]\n","\n"," [[0.004203  ]\n","  [0.        ]\n","  [0.        ]\n","  ...\n","  [0.00203111]\n","  [0.00150292]\n","  [0.0123978 ]]\n","\n"," [[0.004203  ]\n","  [0.        ]\n","  [0.        ]\n","  ...\n","  [0.00203111]\n","  [0.00150292]\n","  [0.0123978 ]]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"OA_CVWAQKTq3","outputId":"3c27879a-8d52-47a4-90fa-2b09d199f110"},"source":["from sklearn.metrics.pairwise import cosine_similarity\n","print('health',cosine_similarity(tfidf_train_data[0], tfidf_train_data[253]))\n","print('logistics',cosine_similarity(tfidf_train_data[0], tfidf_train_data[254]))\n","print('nutrition',cosine_similarity(tfidf_train_data[0], tfidf_train_data[255]))\n","print('protection',cosine_similarity(tfidf_train_data[0], tfidf_train_data[256]))\n","print('shelter',cosine_similarity(tfidf_train_data[0], tfidf_train_data[257]))\n","print('Water, Sanitation and Hygiene',cosine_similarity(tfidf_train_data[0], tfidf_train_data[258]))\n","print('Camp Coordination & Mgmt ',cosine_similarity(tfidf_train_data[0], tfidf_train_data[259]))\n","print('Early Recovery',cosine_similarity(tfidf_train_data[0], tfidf_train_data[260]))\n","print('Education',cosine_similarity(tfidf_train_data[0], tfidf_train_data[261]))\n","print('Emergency Telecom’s',cosine_similarity(tfidf_train_data[0], tfidf_train_data[262]))\n","print('Food Security',cosine_similarity(tfidf_train_data[0], tfidf_train_data[263]))\n","print('Humanitarian & Emergency Relief',cosine_similarity(tfidf_train_data[0], tfidf_train_data[264]))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["health [[0.21823508]]\n","logistics [[0.]]\n","nutrition [[0.]]\n","protection [[0.]]\n","shelter [[0.]]\n","Water, Sanitation and Hygiene [[0.07273377]]\n","Camp Coordination & Mgmt  [[0.00058036]]\n","Early Recovery [[0.]]\n","Education [[0.]]\n","Emergency Telecom’s [[0.]]\n","Food Security [[0.00426921]]\n","Humanitarian & Emergency Relief [[0.004203]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"BK48Lw7WKTq5","outputId":"1168b5e0-3ad6-4342-dda6-a691b7b064b8"},"source":["from sklearn.metrics.pairwise import cosine_similarity\n","print('health',cosine_similarity(tfidf_train_data[18], tfidf_train_data[253]))\n","print('logistics',cosine_similarity(tfidf_train_data[18], tfidf_train_data[254]))\n","print('nutrition',cosine_similarity(tfidf_train_data[18], tfidf_train_data[255]))\n","print('protection',cosine_similarity(tfidf_train_data[18], tfidf_train_data[256]))\n","print('shelter',cosine_similarity(tfidf_train_data[18], tfidf_train_data[257]))\n","print('Water, Sanitation and Hygiene',cosine_similarity(tfidf_train_data[18], tfidf_train_data[258]))\n","print('Camp Coordination & Mgmt ',cosine_similarity(tfidf_train_data[18], tfidf_train_data[259]))\n","print('Early Recovery',cosine_similarity(tfidf_train_data[18], tfidf_train_data[260]))\n","print('Education',cosine_similarity(tfidf_train_data[18], tfidf_train_data[261]))\n","print('Emergency Telecom’s',cosine_similarity(tfidf_train_data[18], tfidf_train_data[262]))\n","print('Food Security',cosine_similarity(tfidf_train_data[18], tfidf_train_data[263]))\n","print('Humanitarian & Emergency Relief',cosine_similarity(tfidf_train_data[18], tfidf_train_data[264]))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["health [[0.06170613]]\n","logistics [[0.]]\n","nutrition [[0.11512521]]\n","protection [[0.00048167]]\n","shelter [[0.]]\n","Water, Sanitation and Hygiene [[0.09195602]]\n","Camp Coordination & Mgmt  [[0.]]\n","Early Recovery [[0.00303604]]\n","Education [[0.00276527]]\n","Emergency Telecom’s [[0.00032323]]\n","Food Security [[0.02288201]]\n","Humanitarian & Emergency Relief [[0.00081141]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"J_D8nMDAKTq6","outputId":"c47cd8c8-0df0-4cb7-9c22-1448e06add7e"},"source":["from sklearn.metrics.pairwise import cosine_similarity\n","print('health',cosine_similarity(tfidf_train_data[264], tfidf_train_data[253]))\n","print('logistics',cosine_similarity(tfidf_train_data[264], tfidf_train_data[254]))\n","print('nutrition',cosine_similarity(tfidf_train_data[264], tfidf_train_data[255]))\n","print('protection',cosine_similarity(tfidf_train_data[264], tfidf_train_data[256]))\n","print('shelter',cosine_similarity(tfidf_train_data[264], tfidf_train_data[257]))\n","print('Water, Sanitation and Hygiene',cosine_similarity(tfidf_train_data[264], tfidf_train_data[258]))\n","print('Camp Coordination & Mgmt ',cosine_similarity(tfidf_train_data[264], tfidf_train_data[259]))\n","print('Early Recovery',cosine_similarity(tfidf_train_data[264], tfidf_train_data[260]))\n","print('Education',cosine_similarity(tfidf_train_data[264], tfidf_train_data[261]))\n","print('Emergency Telecom’s',cosine_similarity(tfidf_train_data[264], tfidf_train_data[262]))\n","print('Food Security',cosine_similarity(tfidf_train_data[264], tfidf_train_data[263]))\n","print('Humanitarian & Emergency Relief',cosine_similarity(tfidf_train_data[264], tfidf_train_data[264]))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["health [[0.]]\n","logistics [[0.]]\n","nutrition [[0.]]\n","protection [[0.]]\n","shelter [[0.]]\n","Water, Sanitation and Hygiene [[0.]]\n","Camp Coordination & Mgmt  [[0.]]\n","Early Recovery [[0.]]\n","Education [[0.]]\n","Emergency Telecom’s [[0.03867725]]\n","Food Security [[0.]]\n","Humanitarian & Emergency Relief [[1.]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"7Zh4Xpt2KTq8"},"source":[""],"execution_count":null,"outputs":[]}]}