diff --git a/README.md b/README.md index d8acaabc5ac..98ba074315a 100755 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@

All-in-one Browser Automation Framework:
Web Crawling / Testing / Scraping / Stealth

+

seleniumbase%2FSeleniumBase | Trendshift

+

PyPI version SeleniumBase PyPI downloads SeleniumBase Docs SeleniumBase GitHub Actions YouTube Channel

diff --git a/examples/cdp_mode/raw_cdp_quotes.py b/examples/cdp_mode/raw_cdp_quotes.py new file mode 100644 index 00000000000..2c24c0a12c2 --- /dev/null +++ b/examples/cdp_mode/raw_cdp_quotes.py @@ -0,0 +1,24 @@ +import json +from seleniumbase import sb_cdp + +sb = sb_cdp.Chrome() +sb.goto("https://quotes.toscrape.com/") +all_quotes = [] +while True: + quotes = sb.find_elements("div.quote") + for quote in quotes: + all_quotes.append( + { + "author": quote.query_selector("span small").text, + "text": quote.query_selector("span.text").text, + } + ) + if sb.is_element_visible("li.next"): + sb.click("li.next") + else: + break +file_path = "downloaded_files/quotes.json" +with open(file_path, "w", encoding="utf-8") as outfile: + json.dump(all_quotes, outfile) +print(f'Quotes saved to "{file_path}".') +sb.quit() diff --git a/examples/cdp_mode/raw_cdp_yelp.py b/examples/cdp_mode/raw_cdp_yelp.py new file mode 100644 index 00000000000..77cfab1eb17 --- /dev/null +++ b/examples/cdp_mode/raw_cdp_yelp.py @@ -0,0 +1,43 @@ +import json +from seleniumbase import sb_cdp + +sb = sb_cdp.Chrome() +sb.goto("https://www.yelp.com/") +sb.sleep(3) +restaurants_menu = 'button[aria-label*="Restaurants"]' +cafes = 'span:contains("Cafes")' +sb.wait_for_element(restaurants_menu) +sb.sleep(1) +sb.hover_and_click(restaurants_menu, cafes) +sb.sleep(3) +all_cafes = [] +counter = 0 +max_pages = 3 +while True: + dtci = '[data-traffic-crawl-id="SearchResult%s"]' + cafes = sb.find_elements('[data-testid="serp-ia-card"]') + for cafe in cafes: + name = cafe.query_selector(dtci % "BizName") + rating = cafe.query_selector(dtci % "BizRating") + snippet = cafe.query_selector(dtci % "ReviewSnippet") + if name and rating and snippet: + all_cafes.append( + { + "name": name.text, + "rating": rating.text, + "snippet": snippet.text, + } + ) + counter += 1 + if counter >= max_pages: + break + if sb.is_element_visible('span:contains("Next Page")'): + sb.click('span:contains("Next Page")') + sb.sleep(1) + else: + break +file_path = "downloaded_files/cafes.json" +with open(file_path, "w", encoding="utf-8") as outfile: + json.dump(all_cafes, outfile) +print(f'Cafes saved to "{file_path}".') +sb.quit() diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt index 5ab22ff43f3..861d06f2fb2 100644 --- a/mkdocs_build/requirements.txt +++ b/mkdocs_build/requirements.txt @@ -3,7 +3,7 @@ regex>=2026.7.19 pymdown-extensions>=10.21.3 -pipdeptree>=4.2.0 +pipdeptree>=4.2.1 python-dateutil>=2.8.2 click>=8.4.2 Markdown==3.10.3 @@ -15,7 +15,7 @@ Babel==2.18.0 paginate==0.5.7 mkdocs==1.6.1 mkdocs-material==9.6.23 -mkdocs-exclude-search==0.6.6 +mkdocs-exclude-search==0.6.7 mkdocs-simple-hooks==0.1.5 mkdocs-get-deps==0.2.2 mkdocs-material-extensions==1.3.1 diff --git a/requirements.txt b/requirements.txt index 2e0918d6085..59f9d2dd1dc 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,22 +1,17 @@ -pip>=26.0.1;python_version<"3.10" -pip>=26.2.1;python_version>="3.10" +pip>=26.2.1 packaging>=26.3 -setuptools~=70.2;python_version<"3.10" -setuptools>=84.0.0;python_version>="3.10" -wheel>=0.47.0 +setuptools>=84.0.0 +wheel>=0.48.0 attrs>=26.1.0 certifi>=2026.7.22 exceptiongroup>=1.3.1 -websockets~=15.0.1;python_version<"3.10" websockets~=16.1.1;python_version=="3.10" -websockets>=17.0.1;python_version>="3.11" -filelock~=3.19.1;python_version<"3.10" -filelock>=3.32.2;python_version>="3.10" +websockets>=16.1.1;python_version>="3.11" +filelock>=3.32.3 fasteners>=0.20 mycdp>=1.4.0 pynose>=1.5.5 -platformdirs~=4.4.0;python_version<"3.10" -platformdirs>=4.11.2;python_version>="3.10" +platformdirs>=4.11.3 typing-extensions>=4.16.0 sbvirtualdisplay>=1.4.0 MarkupSafe>=3.0.3 @@ -27,62 +22,50 @@ parse-type>=0.6.6 colorama>=0.4.6 psutil>=7.2.2 pyyaml>=6.0.3 -pygments>=2.20.0 +pygments>=2.20.1 pyreadline3>=3.5.4;platform_system=="Windows" tabcompleter>=1.4.1 pdbp>=1.8.2 -idna>=3.18 -charset-normalizer>=3.4.9,<4 -urllib3>=1.26.20,<2;python_version<"3.10" -urllib3>=2.7.0,<3;python_version>="3.10" -requests~=2.32.5;python_version<"3.10" -requests~=2.34.2;python_version>="3.10" +idna>=3.19 +charset-normalizer>=3.5.1,<4 +urllib3>=2.7.0,<3 +requests~=2.34.2 sniffio==1.3.1 h11==0.16.0 outcome==1.3.0.post0 -trio>=0.31.0,<1;python_version<"3.10" -trio>=0.33.0,<1;python_version>="3.10" +trio>=0.34.0,<1 trio-websocket~=0.12.2 -wsproto==1.2.0;python_version<"3.10" -wsproto~=1.3.2;python_version>="3.10" +wsproto~=1.3.2 websocket-client~=1.9.0 -selenium==4.32.0;python_version<"3.10" -selenium==4.46.0;python_version>="3.10" -cssselect==1.3.0;python_version<"3.10" -cssselect>=1.5.0,<2;python_version>="3.10" +selenium==4.47.0 +cssselect>=1.5.0,<2 sortedcontainers==2.4.0 -execnet==2.1.1;python_version<"3.10" -execnet==2.1.2;python_version>="3.10" -iniconfig==2.1.0;python_version<"3.10" -iniconfig==2.3.0;python_version>="3.10" +execnet==2.1.2 +iniconfig==2.3.0 pluggy==1.6.0 pytest==8.4.2;python_version<"3.11" pytest==9.1.1;python_version>="3.11" pytest-html==4.0.2 pytest-metadata==3.1.1 pytest-ordering==0.6 -pytest-rerunfailures==16.0.1;python_version<"3.10" pytest-rerunfailures==16.3;python_version=="3.10" -pytest-rerunfailures==16.4;python_version>="3.11" +pytest-rerunfailures==16.6;python_version>="3.11" pytest-xdist==3.8.0 parameterized==0.9.0 behave==1.2.6 -soupsieve~=2.8.4;python_version<"3.10" -soupsieve~=2.9.2;python_version>="3.10" +soupsieve~=2.9.2 beautifulsoup4~=4.15.0 pyotp~=2.10.0 python-xlib==0.33;platform_system=="Linux" PyAutoGUI>=0.9.54;platform_system=="Linux" -markdown-it-py==3.0.0;python_version<"3.10" -markdown-it-py==4.2.0;python_version>="3.10" +markdown-it-py==4.2.0 mdurl==0.1.2 rich>=15.0.0,<16 # --- Testing Requirements --- # # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.) -coverage>=7.10.7;python_version<"3.10" -coverage>=7.15.4;python_version>="3.10" +coverage>=7.15.4 pytest-cov>=7.1.0 flake8==7.3.0 mccabe==0.7.0 diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 8997f2184ad..3d75d1477ff 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.51.12" +__version__ = "4.52.0" diff --git a/seleniumbase/console_scripts/sb_install.py b/seleniumbase/console_scripts/sb_install.py index 30e297340d2..b9cefc4c4e0 100644 --- a/seleniumbase/console_scripts/sb_install.py +++ b/seleniumbase/console_scripts/sb_install.py @@ -659,8 +659,8 @@ def main(override=None, intel_for_uc=None, force_uc=None): else: not_latest = c5 + "(" + c4 + "Legacy Version" + c5 + ")" + cr p_version = p_version + " " + not_latest - msg = c2 + "chromedriver to download" + cr - log_d("\n*** %s = %s" % (msg, p_version)) + msg = c2 + "chromedriver" + cr + log_d("\n*** Getting %s %s" % (msg, p_version)) else: raise Exception("Could not find chromedriver to download!\n") if not get_latest: @@ -704,9 +704,9 @@ def main(override=None, intel_for_uc=None, force_uc=None): use_version = found_version if not use_version: use_version = get_cft_latest_version_from_milestone(major_version) - msg = c2 + "Chrome for Testing to download" + cr + msg = c2 + "Chrome for Testing" + cr p_version = c3 + use_version + cr - log_d("\n*** %s = %s" % (msg, p_version)) + log_d("\n*** Getting %s %s" % (msg, p_version)) if IS_MAC: if IS_ARM_MAC: platform_code = "mac-arm64" @@ -813,9 +813,9 @@ def main(override=None, intel_for_uc=None, force_uc=None): use_version = found_version if not use_version: use_version = get_cft_latest_version_from_milestone(major_version) - msg = c2 + "Chrome-Headless-Shell to download" + cr + msg = c2 + "Chrome-Headless-Shell" + cr p_version = c3 + use_version + cr - log_d("\n*** %s = %s" % (msg, p_version)) + log_d("\n*** Getting %s %s" % (msg, p_version)) if IS_MAC: if IS_ARM_MAC: platform_code = "mac-arm64" @@ -892,9 +892,9 @@ def main(override=None, intel_for_uc=None, force_uc=None): if not found_geckodriver: url_request = requests_get(download_url) if found_geckodriver or url_request.ok: - msg = c2 + "geckodriver to download" + cr + msg = c2 + "geckodriver" + cr p_version = c3 + use_version + cr - log_d("\n*** %s = %s" % (msg, p_version)) + log_d("\n*** Getting %s %s" % (msg, p_version)) else: raise Exception( "\nCould not find the specified geckodriver " @@ -999,9 +999,9 @@ def main(override=None, intel_for_uc=None, force_uc=None): raise Exception( "Could not find version [%s] of EdgeDriver!" % use_version ) - msg = c2 + "edgedriver to download" + cr + msg = c2 + "edgedriver" + cr p_version = c3 + use_version + cr - log_d("\n*** %s = %s" % (msg, p_version)) + log_d("\n*** Getting %s %s" % (msg, p_version)) elif name == "iedriver": full_version = "4.14.0" use_version = full_version @@ -1029,9 +1029,9 @@ def main(override=None, intel_for_uc=None, force_uc=None): url_request = requests_get_with_retry(headless_ie_url) if url_request.ok: headless_ie_exists = True - msg = c2 + "HeadlessIEDriver to download" + cr + msg = c2 + "HeadlessIEDriver" + cr p_version = c3 + headless_ie_version + cr - log_d("\n*** %s = %s" % (msg, p_version)) + log_d("\n*** Getting %s %s" % (msg, p_version)) else: invalid_run_command() @@ -1095,7 +1095,7 @@ def main(override=None, intel_for_uc=None, force_uc=None): os.remove(new_file) if not driver_file or not driver_path or not filename: raise Exception("headless_ie_selenium.exe missing from Zip file!") - log_d("Extracting %s from %s ..." % (filename, headless_ie_file_name)) + log_d("Extracting %s from %s:" % (filename, headless_ie_file_name)) zip_ref.extractall(downloads_folder) zip_ref.close() os.remove(zip_file_path) @@ -1120,13 +1120,12 @@ def main(override=None, intel_for_uc=None, force_uc=None): os.rmdir(os.path.join(downloads_folder, h_ie_fn)) driver_path = os.path.join(downloads_folder, filename) log_d( - "The file [%s] was saved to:\n%s%s%s\n" + "['%s'] was saved to:\n%s%s%s\n" % (filename, c3, driver_path, cr) ) - log_d("Making [%s %s] executable ..." % (driver_file, use_version)) make_executable(driver_path) log_d( - "%s[%s %s] is now ready for use!%s" + "%s[%s %s] is ready for use!%s" % (c1, driver_file, use_version, cr) ) @@ -1180,7 +1179,7 @@ def main(override=None, intel_for_uc=None, force_uc=None): os.remove(new_file) # Technically the old file now if driver_contents: contents = driver_contents - log_d("Extracting %s from %s ..." % (contents, file_name)) + log_d("Extracting %s from %s:" % (contents, file_name)) if name == "uc_driver": f_name = "uc_driver" new_file = os.path.join(downloads_folder, f_name) @@ -1232,11 +1231,10 @@ def main(override=None, intel_for_uc=None, force_uc=None): d_folder = os.sep.join(pr_file.split(os.sep)[:-1]) + os.sep d_file = pr_file.split(os.sep)[-1] d_ff = c3 + d_folder + cr + "\n" + c3 + d_file + cr - log_d("The file [%s] was saved to:\n%s\n" % (f_name, d_ff)) - log_d("Making [%s %s] executable ..." % (f_name, use_version)) + log_d("['%s'] was saved to:\n%s\n" % (f_name, d_ff)) make_executable(new_file) log_d( - "%s[%s %s] is now ready for use!%s" % + "%s[%s %s] is ready for use!%s" % (c1, f_name, use_version, cr) ) if copy_to_path and os.path.exists(LOCAL_PATH): @@ -1301,7 +1299,7 @@ def main(override=None, intel_for_uc=None, force_uc=None): if str_name == "IEDriverServer.exe": raise Exception("IEDriverServer missing from Zip file!") raise Exception("msedgedriver missing from Zip file!") - log_d("Extracting %s from %s ..." % (contents, file_name)) + log_d("Extracting %s from %s:" % (contents, file_name)) zip_ref.extractall(downloads_folder) zip_ref.close() os.remove(zip_file_path) @@ -1323,13 +1321,12 @@ def main(override=None, intel_for_uc=None, force_uc=None): pr_sep = c3 + os.sep + cr pr_driver_file = c3 + driver_file + cr log_d( - "The file [%s] was saved to:\n%s%s\n%s\n" + "['%s'] was saved to:\n%s%s\n%s\n" % (driver_file, pr_driver_base, pr_sep, pr_driver_file) ) - log_d("Making [%s %s] executable ..." % (driver_file, use_version)) make_executable(driver_path) log_d( - "%s[%s %s] is now ready for use!%s" + "%s[%s %s] is ready for use!%s" % (c1, driver_file, use_version, cr) ) if copy_to_path and os.path.exists(LOCAL_PATH): @@ -1462,7 +1459,7 @@ def main(override=None, intel_for_uc=None, force_uc=None): if "Driver" in new_file or "driver" in new_file: if os.path.exists(new_file): os.remove(new_file) # Technically the old file now - log_d("Extracting %s from %s ..." % (contents, file_name)) + log_d("Extracting %s from %s:" % (contents, file_name)) if sys.version_info < (3, 12): tar.extractall(downloads_folder) else: @@ -1473,11 +1470,10 @@ def main(override=None, intel_for_uc=None, force_uc=None): for f_name in contents: new_file = os.path.join(downloads_folder, str(f_name)) pr_file = c3 + new_file + cr - log_d("The file [%s] was saved to:\n%s\n" % (f_name, pr_file)) - log_d("Making [%s %s] executable ..." % (f_name, use_version)) + log_d("['%s'] was saved to:\n%s\n" % (f_name, pr_file)) make_executable(new_file) log_d( - "%s[%s %s] is now ready for use!%s" + "%s[%s %s] is ready for use!%s" % (c1, f_name, use_version, cr) ) if copy_to_path and os.path.exists(LOCAL_PATH): @@ -1494,9 +1490,8 @@ def main(override=None, intel_for_uc=None, force_uc=None): # Not a .zip file or a .tar.gz file. Just a direct download. if "Driver" in file_name or "driver" in file_name: log_d("%sDownload Complete!%s\n" % (c1, cr)) - log_d("Making [%s] executable ..." % file_name) make_executable(file_path) - log_d("%s[%s] is now ready for use!%s" % (c1, file_name, cr)) + log_d("%s[%s] is ready for use!%s" % (c1, file_name, cr)) log_d("Location of [%s]:\n%s\n" % (file_name, file_path)) diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index 713809a0cc8..d71aa2b1786 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -116,7 +116,9 @@ def __add_sync_methods(self, element): element.send_file = ( lambda *file_paths: self.__send_file(element, *file_paths) ) - element.send_keys = lambda text: self.__send_keys(element, text) + element.send_keys = lambda text, **kwargs: self.__send_keys( + element, text, **kwargs + ) element.set_text = lambda value: self.__set_text(element, value) element.set_value = lambda value: self.__set_value(element, value) element.type = lambda text: self.__type(element, text) @@ -721,9 +723,11 @@ def __send_file(self, element, *file_paths): self.loop.run_until_complete(element.send_file_async(*file_paths)) ) - def __send_keys(self, element, text): + def __send_keys(self, element, text, **kwargs): return ( - self.loop.run_until_complete(element.send_keys_async(text)) + self.loop.run_until_complete( + element.send_keys_async(text, **kwargs) + ) ) def __set_text(self, element, value): @@ -1270,26 +1274,7 @@ def press_keys(self, selector, text, timeout=None): self.__slow_mode_pause_if_set() element = self.select(selector, timeout=timeout) element.scroll_into_view() - submit = False - if text.endswith("\n") or text.endswith("\r"): - submit = True - text = text[:-1] - elif ( - element.tag_name == "textarea" - and "\n" in text - and "\r" not in text - ): - text = text.replace("\n", "\r") - for key in text: - try: - element.send_keys(key) - except AttributeError: - element = self.select(selector, timeout=0.1) - element.send_keys(key) - time.sleep(float(0.0422 + (random.random() / 112.0))) - if submit: - element.send_keys("\r\n") - time.sleep(0.044) + element.send_keys(text, add_delay=True) self.__slow_mode_pause_if_set() self.loop.run_until_complete(self.page.sleep(0.025)) @@ -1367,18 +1352,18 @@ def set_value(self, selector, text, timeout=None): self.loop.run_until_complete(self.page.sleep(0.025)) def submit(self, selector): - submit_script = ( - """elm = document.querySelector('%s'); - const event = new KeyboardEvent("keydown", { - key: "Enter", - keyCode: 13, - code: "Enter", - which: 13, - bubbles: true - }); - elm.dispatchEvent(event);""" % selector - ) - self.loop.run_until_complete(self.page.evaluate(submit_script)) + element = self.find_element(selector, timeout=1) + try: + element.press_keys("\n") + except Exception: + submit_script = ( + """ + const el = document.querySelector('%s'); + const form = el.form ?? el.closest('form'); + form.requestSubmit(); + """ % selector + ) + self.loop.run_until_complete(self.page.evaluate(submit_script)) def evaluate(self, expression): """Run a JavaScript expression and return the result.""" diff --git a/seleniumbase/undetected/cdp_driver/browser.py b/seleniumbase/undetected/cdp_driver/browser.py index 2c72cb65856..384b2a626d4 100644 --- a/seleniumbase/undetected/cdp_driver/browser.py +++ b/seleniumbase/undetected/cdp_driver/browser.py @@ -22,7 +22,6 @@ from seleniumbase import config as sb_config from seleniumbase.fixtures import constants from seleniumbase.fixtures import shared_utils -from typing import List, Optional, Set, Tuple, Union import mycdp as cdp from . import cdp_util as util from . import tab @@ -116,7 +115,7 @@ async def create( incognito: bool = False, guest: bool = False, browser_executable_path: PathLike = None, - browser_args: List[str] = None, + browser_args: list[str] = None, sandbox: bool = True, host: str = None, port: int = None, @@ -160,7 +159,7 @@ def __init__(self, config: Config, **kwargs): ) ) self.config = config - self.targets: List = [] + self.targets: list = [] self.info = None self._target = None self._process = None @@ -182,7 +181,7 @@ def main_tab(self) -> tab.Tab: )[0] @property - def tabs(self) -> List[tab.Tab]: + def tabs(self) -> list[tab.Tab]: """Returns the current targets which are of type "page".""" tabs = filter(lambda item: item.type_ == "page", self.targets) return list(tabs) @@ -199,7 +198,7 @@ def stopped(self): return False return True - async def wait(self, time: Union[float, int] = 1) -> Browser: + async def wait(self, time: float | int = 1) -> Browser: """Wait for