updated resources and requirements & fixed errors

- fixed scraping errors (xpath)
- added todo tasks
- completed done tasks
- 0.8.5 version preparation
- efficiency/errors checks need

.travis.yml

@@ -9,11 +9,11 @@ dist: xenial
 
 install:
   - pip install pandas==0.24.2
-  - pip install requests==2.21.0
+  - pip install requests==2.22.0
   - pip install lxml==4.3.3
   - pip install unidecode==1.0.23
-  - pip install investpy==0.8.4.3
-  - pip install pytest==4.1.1
+  - pip install investpy==0.8.4.5
+  - pip install pytest==4.5
 
 script:
   - pytest

README.md

@@ -16,7 +16,7 @@ To conclude this section, I am in the need to specify that this is not the final
 
 In order to get this package working you will need to install [**investpy**](https://pypi.org/project/investpy/) from PyPi via Terminal typing:
 
-``pip install investpy==0.8.4.4``
+``pip install investpy==0.8.4.5``
 
 All the dependencies are already listed on the setup file of the package, but to sum them up, you will need the following requirements:
 
@@ -51,6 +51,7 @@ If needed you can open an [issue](https://github.com/alvarob96/investpy/issues)
 * Updated docstrings as reStructuredText (via PyCharm)
 * Modified JSON output to fit current standard for historical data
 * Added function to retrieve information from listed ETFs (id, name, symbol and tag)
+* 
 
 ## Additional Information
 

investpy/Data.py

@@ -75,12 +75,13 @@ def fund_to_dict(self):
         }
 
     def fund_as_json(self):
-        return {self.date.strftime('%d/%m/%Y'): {
-            'Open': self.open,
-            'High': self.high,
-            'Low': self.low,
-            'Close': self.close,
-        }}
+        return {
+            'date': self.date.strftime('%d/%m/%Y'),
+            'open': self.open,
+            'high': self.high,
+            'low': self.low,
+            'close': self.close,
+        }
 
     def etf_to_dict(self):
         return {

investpy/__init__.py

@@ -6,13 +6,13 @@
 __author__ = "Alvaro Bartolome <[email protected]>"
 
 import datetime
+import json
 from random import randint
-import unidecode
 
 import pandas as pd
 import pkg_resources
 import requests
-import json
+import unidecode
 from lxml.html import fromstring
 
 from investpy import user_agent as ua, equities as ts, funds as fs, etfs as es
@@ -22,6 +22,8 @@
 # TODO: add country/market param and mapping of ‘resources/available_markets’ in order to allow users retrieve
 #  historical data from different markets.
 
+# DONE: available_languages replaced by available_markets
+
 # TODO: create thread pools to increase scraping efficiency and improve ‘investpy’ performance => CHECK BOOK DOC
 
 # TODO: generate sphinx documentation for version 1.0
@@ -51,6 +53,12 @@
 
 # TODO: fix dosctrings and unify structure with Google docstrings or similar
 
+# WARNING: RE-GENERATE MARKET FILES BEFORE EVERY RELEASE
+
+# TODO: add 'clase de activo', 'isin' and 'emisor' to funds
+
+# DONE: updated equities, funds and etfs retrieval functions
+
 
 def get_equities_list():
     """
@@ -164,10 +172,12 @@ def get_recent_data(equity, as_json=False, order='ascending'):
                              'recent':
                                  [value.equity_as_json() for value in result]
                              }
-                    return json.dumps(json_)
+
+                    return json.dumps(json_, sort_keys=False)
                 elif as_json is False:
                     df = pd.DataFrame.from_records([value.equity_to_dict() for value in result])
                     df.set_index('Date', inplace=True)
+
                     return df
             else:
                 raise RuntimeError("ERR#004: data retrieval error while scraping.")
@@ -364,7 +374,7 @@ def get_historical_data(equity, start, end, as_json=False, order='ascending'):
                     raise RuntimeError("ERR#004: data retrieval error while scraping.")
 
             if as_json is True:
-                return json.dumps(final)
+                return json.dumps(final, sort_keys=False)
             elif as_json is False:
                 return pd.concat(final)
         else:
@@ -592,10 +602,12 @@ def get_fund_recent_data(fund, as_json=False, order='ascending'):
                              'recent':
                                  [value.fund_as_json() for value in result]
                              }
-                    return json.dumps(json_)
+
+                    return json.dumps(json_, sort_keys=False)
                 elif as_json is False:
                     df = pd.DataFrame.from_records([value.fund_to_dict() for value in result])
                     df.set_index('Date', inplace=True)
+
                     return df
 
             else:
@@ -768,7 +780,7 @@ def get_fund_historical_data(fund, start, end, as_json=False, order='ascending')
                     raise RuntimeError("ERR#004: data retrieval error while scraping.")
 
             if as_json is True:
-                return json.dumps(final)
+                return json.dumps(final, sort_keys=False)
             elif as_json is False:
                 return pd.concat(final)
         else:
@@ -1032,10 +1044,12 @@ def get_etf_recent_data(etf, as_json=False, order='ascending'):
                              'recent':
                                  [value.etf_as_json() for value in result]
                              }
-                    return json.dumps(json_)
+
+                    return json.dumps(json_, sort_keys=False)
                 elif as_json is False:
                     df = pd.DataFrame.from_records([value.etf_to_dict() for value in result])
                     df.set_index('Date', inplace=True)
+
                     return df
 
             else:
@@ -1201,14 +1215,14 @@ def get_etf_historical_data(etf, start, end, as_json=False, order='ascending'):
                         final.append(json_)
                     elif as_json is False:
                         df = pd.DataFrame.from_records([value.etf_to_dict() for value in result])
-                        df.set_index('date', inplace=True)
+                        df.set_index('Date', inplace=True)
 
                         final.append(df)
                 else:
                     raise RuntimeError("ERR#004: data retrieval error while scraping.")
 
             if as_json is True:
-                return json.dumps(final)
+                return json.dumps(final, sort_keys=False)
             elif as_json is False:
                 return pd.concat(final)
         else:

investpy/equities.py

@@ -49,7 +49,9 @@ def get_equity_names():
         raise ConnectionError("ERR#015: error " + req.status_code + ", try again later.")
 
     root_ = fromstring(req.text)
-    path_ = root_.xpath(".//table[@id='cross_rate_markets_stocks_1']/tbody/tr")
+    path_ = root_.xpath(".//table[@id='cross_rate_markets_stocks_1']"
+                        "/tbody"
+                        "/tr")
 
     results = list()
 
@@ -60,7 +62,11 @@ def get_equity_names():
             for element_ in elements_.xpath('.//a'):
                 tag_ = element_.get('href').replace('/equities/', '')
                 full_name_ = element_.get('title').replace(' (CFD)', '')
-                isin_ = get_isin_code(tag_)
+
+                try:
+                    isin_ = get_isin_code(tag_)
+                except (ConnectionError, IndexError):
+                    isin_ = None
 
                 data = {
                     "name": element_.text,
@@ -105,22 +111,30 @@ def get_isin_code(info):
     req = requests.get(url, headers=head, timeout=5)
 
     if req.status_code != 200:
-        # raise ConnectionError("ERR#015: error " + req.status_code + ", try again later.")
-        return None
+        raise ConnectionError("ERR#015: error " + req.status_code + ", try again later.")
 
     root_ = fromstring(req.text)
-    path_ = root_.xpath("/html/body/div[5]/section/div[4]/div[1]/div[2]/div[3]/span[2]")
+    path_ = root_.xpath(".//div[contains(@class, 'overViewBox')]"
+                        "/div[@id='quotes_summary_current_data']"
+                        "/div[@class='right']"
+                        "/div")
 
-    code = None
-
-    if path_:
+    for p in path_:
         try:
-            code = path_[0].text_content().rstrip()
-            time.sleep(.5)
+            if p.xpath("span[not(@class)]")[0].text_content().__contains__('ISIN'):
+                try:
+                    code = p.xpath("span[@class='elp']")[0].text_content().rstrip()
+                    time.sleep(.5)
+
+                    return code
+                except IndexError:
+                    raise IndexError("ERR#017: isin code unavailable or not found.")
+            else:
+                continue
         except IndexError:
             raise IndexError("ERR#017: isin code unavailable or not found.")
 
-    return code
+    return None
 
 
 def list_equities():

investpy/etfs.py

@@ -5,6 +5,8 @@
 
 __author__ = "Alvaro Bartolome <[email protected]>"
 
+import time
+
 import pandas as pd
 import requests
 import json
@@ -40,7 +42,9 @@ def get_etf_names():
         raise ConnectionError("ERR#015: error " + req.status_code + ", try again later.")
 
     root_ = fromstring(req.text)
-    path_ = root_.xpath(".//table[@id='etfs']/tbody/tr")
+    path_ = root_.xpath(".//table[@id='etfs']"
+                        "/tbody"
+                        "/tr")
 
     results = list()
 
@@ -99,11 +103,11 @@ def list_etfs():
 
     if etfs is None:
         raise IOError("ERR#009: etf list not found or unable to retrieve.")
-
-    return etfs['name'].tolist()
+    else:
+        return etfs['name'].tolist()
 
 
-def dict_etfs(columns=['id', 'name', 'symbol', 'tag'], as_json=False):
+def dict_etfs(columns=None, as_json=False):
     """
     This function retrieves all the available etfs and returns a dictionary with the specified columns.
     Available columns are: 'id', 'name', 'symbol' and 'tag'
@@ -114,8 +118,11 @@ def dict_etfs(columns=['id', 'name', 'symbol', 'tag'], as_json=False):
         :returns a dictionary that contains all the available etf values specified in the columns
     """
 
-    if not isinstance(columns, list):
-        raise ValueError("ERR#020: specified columns argument is not a list, it can just be list type.")
+    if columns is None:
+        columns = ['id', 'name', 'symbol', 'tag']
+    else:
+        if not isinstance(columns, list):
+            raise ValueError("ERR#020: specified columns argument is not a list, it can just be list type.")
 
     if not isinstance(as_json, bool):
         raise ValueError("ERR#002: as_json argument can just be True or False, bool type.")

investpy/funds.py

@@ -40,7 +40,9 @@ def get_fund_names():
         raise ConnectionError("ERR#015: error " + req.status_code + ", try again later.")
 
     root_ = fromstring(req.text)
-    path_ = root_.xpath(".//table[@id='etfs']/tbody/tr")
+    path_ = root_.xpath(".//table[@id='etfs']"
+                        "/tbody"
+                        "/tr")
 
     results = list()
 
@@ -49,19 +51,19 @@ def get_fund_names():
             id_ = elements_.get('id').replace('pair_', '')
             symbol = elements_.xpath(".//td[contains(@class, 'symbol')]")[0].get('title')
 
-            nested = elements_.xpath(".//a")[0]
-            info = nested.get('href').replace('/funds/', '')
+            nested = elements_.xpath(".//a")[0].get('title').rstrip()
+            info = elements_.xpath(".//a")[0].get('href').replace('/funds/', '')
 
             if symbol:
                 data = {
-                    "name": nested.text,
+                    "name": nested,
                     "symbol": symbol,
                     "tag": info,
                     "id": id_
                 }
             else:
                 data = {
-                    "name": nested.text,
+                    "name": nested,
                     "symbol": "undefined",
                     "tag": info,
                     "id": id_
@@ -107,7 +109,7 @@ def fund_information_to_json(df):
         'Category': str(df['Category'][0])
     }
 
-    result = json.dumps(json_)
+    result = json.dumps(json_, sort_keys=False)
 
     return result
 
@@ -132,5 +134,9 @@ def list_funds():
 
     if funds is None:
         raise IOError("ERR#005: fund list not found or unable to retrieve.")
+    else:
+        return funds['name'].tolist()
+
 
-    return funds['name'].tolist()
+if __name__ == '__main__':
+    get_fund_names()

investpy/resources/es/equities.csv

@@ -164,7 +164,6 @@ PharmaMar S.A.,959215,ES0169501030,PharmaMar,pharma-mar-sau
 Gigas Hosting SA,960710,ES0105093001,Gigas Hosting SA,gigas-hosting-sa
 Neol Biosolutions SA,960711,ES0105081006,Neol Biosolutions SA,neol-biosolutions-sa
 Agile Content SA,961658,ES0105102000,Agile Content SA,agile-content-sa
-Think Smart SA,961659,ES0105097002,Think Smart SA,think-smart-sa
 Zambal Spain Socimi SA,961773,ES0105080008,Zambal Spain Socimi SA,zambal-spain-socimi-sa
 Oryzon Genomics SA,962024,ES0167733015,Oryzon Genomics,oryzon-genomics-sa
 Tecnoquark Trust SA,962349,ES0105076006,Tecnoquark Trust SA,tecnoquark-trust-sa
@@ -234,3 +233,4 @@ Euripo Properties Socimi,1122284,ES0105387007,Euripo Properties,euripo-propertie
 Proeduca Altus SA,1123486,ES0105400008,Proeduca Altus,proeduca-altus
 Meridia Real Estate III Socimi SA,1123955,ES0105313003,Meridia RE III,meridia-re-iii
 Iffe Futura SA,1123956,ES0171613005,Iffe Futura,iffe-futura
+Compania Espanola de Viviendas en Alquiler SA,1130889,ES0132955008,Viviendas en Alquiler,viviendas-en-alquiler

investpy/resources/es/etfs.csv

@@ -1,7 +1,7 @@
 id,name,symbol,tag
 37633,Lyxor Ibex 35 Doble Inverso Diario,2INVE,lyxor-ibex-35-x2-inverso
 37631,Lyxor Ibex 35 Doble Apalancado Diario C-EUR,IBEXA,lyxor-ibex-35-doble-apalancado
-37632,Accion IBEX 35 Cotizado Armonizado FI,BBVAI,bbva-accion-ibex-35
 47649,BBVA Accion DJ Eurostoxx 50,BBVAE,bbva-accion-dj-eurostoxx-50
-38897,Lyxor Ibex 35 Inverso Diario,INVEX,lyxor-ibex-35-invers
 38898,Lyxor Ibex35 (DR) D-EUR,LYXIB,lyxor-ibex-35
+37632,Accion IBEX 35 Cotizado Armonizado FI,BBVAI,bbva-accion-ibex-35
+38897,Lyxor Ibex 35 Inverso Diario,INVEX,lyxor-ibex-35-invers