Skip to content

Commit

Permalink
fix: dp bugfix
Browse files Browse the repository at this point in the history
  • Loading branch information
linyuan0213 committed Aug 24, 2024
1 parent 574863e commit e53c461
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 5 deletions.
13 changes: 9 additions & 4 deletions app/helper/drissionpage_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,15 @@ def get_page_html(self,
page = ChromiumPage(self.co, timeout=180)
# 等待 page 加载完成
page.wait(1)
page.add_init_js(JS_SCRIPT)
page.set.window.max()
page.set.load_mode.none()
page.get(url, retry=3)
try:
page.add_init_js(JS_SCRIPT)
page.set.window.max()
page.set.load_mode.none()
page.get(url, retry=3)
except Exception as e:
logger.debug(f"DrissionPage Error: {e}")
page.quit()
return ''
if cookies:
if isinstance(cookies, str):
cookies = cookies.strip()
Expand Down
3 changes: 2 additions & 1 deletion app/indexer/client/builtin.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ def list(self, index_id, page=0, keyword=None):
return result_array

@staticmethod
def __spider_search(indexer, keyword=None, page=None, mtype=None, timeout=60):
def __spider_search(indexer, keyword=None, page=None, mtype=None, timeout=90):
"""
根据关键字搜索单个站点
:param: indexer: 站点配置
Expand All @@ -273,6 +273,7 @@ def __spider_search(indexer, keyword=None, page=None, mtype=None, timeout=60):
sleep_count += 1
time.sleep(1)
if sleep_count > timeout:
spider.stop_spider()
break
# 是否发生错误
result_flag = spider.is_error
Expand Down

0 comments on commit e53c461

Please sign in to comment.