From 194ed949461c89529b440fbe2e2bf7b3671816d9 Mon Sep 17 00:00:00 2001 From: blacknon Date: Thu, 7 Sep 2023 23:23:10 +0900 Subject: [PATCH 1/3] update. --- pydork/engine_common.py | 2 +- setup.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pydork/engine_common.py b/pydork/engine_common.py index e3712cc..6c02ae1 100644 --- a/pydork/engine_common.py +++ b/pydork/engine_common.py @@ -579,7 +579,7 @@ def get_result(self, url: str, method='GET', data=None): self.driver.execute_script( "window.scrollTo(0,document.body.scrollHeight)" ) - time.sleep(3) + time.sleep(0.5) # 優先度2: Splash経由でのアクセス(Seleniumが有効になってない場合はこちら) elif self.USE_SPLASH: diff --git a/setup.py b/setup.py index b43ae43..f095968 100755 --- a/setup.py +++ b/setup.py @@ -78,12 +78,13 @@ def get_completefile_install_location(shell): data_files = [] data_files.append((loc['bash'], files['bash'])) data_files.append((loc['zsh'], files['zsh'])) + return data_files name = 'pydork' -version = '1.1.6' -release = '1.1.6' +version = '1.1.7' +release = '1.1.7' if __name__ == "__main__": setuptools.setup( @@ -126,6 +127,7 @@ def get_completefile_install_location(shell): 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'License :: OSI Approved :: MIT License', ], data_files=get_data_files(), From 42299387279a0e0a1fa410cc73abf572a66bbbbb Mon Sep 17 00:00:00 2001 From: blacknon Date: Thu, 7 Sep 2023 23:24:23 +0900 Subject: [PATCH 2/3] =?UTF-8?q?update.=20selenium=E3=81=AE=E3=83=96?= =?UTF-8?q?=E3=83=A9=E3=82=A6=E3=82=B6=E3=82=B9=E3=82=AF=E3=83=AD=E3=83=BC?= =?UTF-8?q?=E3=83=AB=E3=81=AE=E5=87=A6=E7=90=86=E3=82=92=E6=B6=88=E3=81=97?= =?UTF-8?q?=E5=BF=98=E3=82=8C=E3=81=A6=E3=81=9F=E3=81=AE=E3=81=A7=E3=82=B3?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=A2=E3=82=A6=E3=83=88.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pydork/engine_common.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pydork/engine_common.py b/pydork/engine_common.py index 6c02ae1..a72b6a5 100644 --- a/pydork/engine_common.py +++ b/pydork/engine_common.py @@ -575,11 +575,12 @@ def get_result(self, url: str, method='GET', data=None): if self.USE_SELENIUM: result = self.request_selenium(url, method=method, data=data) - for i in range(0, 10): - self.driver.execute_script( - "window.scrollTo(0,document.body.scrollHeight)" - ) - time.sleep(0.5) + # NOTE: seleniumでのブラウザスクロール. googleでの処理で不要になったため、ただ遅くなるだけで不便なので一旦コメントアウト. + # for i in range(0, 10): + # self.driver.execute_script( + # "window.scrollTo(0,document.body.scrollHeight)" + # ) + # time.sleep(0.5) # 優先度2: Splash経由でのアクセス(Seleniumが有効になってない場合はこちら) elif self.USE_SPLASH: From 9531e642bcdfa2614ed2f269989900f8a2fddd65 Mon Sep 17 00:00:00 2001 From: blacknon Date: Mon, 6 May 2024 16:13:18 +0900 Subject: [PATCH 3/3] update. --- pydork/engine_common.py | 22 +++++++++++++++++++++- pydork/engine_google.py | 12 ++++++------ setup.py | 1 - 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pydork/engine_common.py b/pydork/engine_common.py index a72b6a5..cab540e 100644 --- a/pydork/engine_common.py +++ b/pydork/engine_common.py @@ -13,7 +13,6 @@ import requests import os import pickle -import time # selenium driver auto install packages import chromedriver_autoinstaller @@ -706,14 +705,35 @@ def get_text_links(self, soup: BeautifulSoup): list: textの検索結果([xxx,xxx,xxx...]) """ # linkのurlを取得する + self.MESSAGE.print_text( + self.SOUP_SELECT_URL, # type: ignore + header=self.MESSAGE.HEADER + ': ' + \ + Color.GREEN + '[get_text_link.SOUP_SELECT_URL]' + Color.END, + separator=" :", + mode="debug", + ) elements = soup.select(self.SOUP_SELECT_URL) elinks = [e['href'] for e in elements] # linkのtitleを取得する + self.MESSAGE.print_text( + self.SOUP_SELECT_TITLE, # type: ignore + header=self.MESSAGE.HEADER + ': ' + \ + Color.GREEN + '[get_text_link.SOUP_SELECT_TITLE]' + Color.END, + separator=" :", + mode="debug", + ) elements = soup.select(self.SOUP_SELECT_TITLE) etitles = [e.text for e in elements] # linkのtextを取得する + self.MESSAGE.print_text( + self.SOUP_SELECT_TEXT, # type: ignore + header=self.MESSAGE.HEADER + ': ' + \ + Color.GREEN + '[get_text_link.SOUP_SELECT_TEXT]' + Color.END, + separator=" :", + mode="debug", + ) elements = soup.select(self.SOUP_SELECT_TEXT) etext = [e.text for e in elements] diff --git a/pydork/engine_google.py b/pydork/engine_google.py index a4d12b4..a731c0d 100644 --- a/pydork/engine_google.py +++ b/pydork/engine_google.py @@ -202,16 +202,16 @@ def get_links(self, url: str, html: str, type: str): # Selenium経由、かつFirefoxを使っている場合 if self.USE_SELENIUM: - self.SOUP_SELECT_URL = '.yuRUbf > div > a' - self.SOUP_SELECT_TITLE = '.yuRUbf > div > a > .LC20lb' - self.SOUP_SELECT_TEXT = '.lEBKkf' + self.SOUP_SELECT_URL = '.yuRUbf > div > span > a' + self.SOUP_SELECT_TITLE = '.yuRUbf > div > span > a > h3' + self.SOUP_SELECT_TEXT = '.yXK7lf' self.SOUP_SELECT_NEXT_URL = '.AaVjTc > tbody > tr > td > a' # Splash経由で通信している場合 elif self.USE_SPLASH: - self.SOUP_SELECT_URL = '.yuRUbf > a' - self.SOUP_SELECT_TITLE = '.yuRUbf > a > .LC20lb' - self.SOUP_SELECT_TEXT = '.lEBKkf' + self.SOUP_SELECT_URL = '.yuRUbf > div > span > a.href' + self.SOUP_SELECT_TITLE = '.yuRUbf > div > span > a > h3' + self.SOUP_SELECT_TEXT = '.yXK7lf' self.SOUP_SELECT_NEXT_URL = '.AaVjTc > tbody > tr > td > a' # TODO: SEARCH_NEXT_URLを書き換える diff --git a/setup.py b/setup.py index f095968..6224768 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,6 @@ # 補完ファイルインストール用関数 def get_data_files(): - # 補完ファイルのインストール先を取得する関数 def get_completefile_install_location(shell): # pathのprefixを定義