diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index ad6f032fc52..a8a47f2651c 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -133,12 +133,10 @@ from seleniumbase import SB with SB(uc=True, test=True, locale="en", ad_block=True) as sb: sb.activate_cdp_mode() - sb.goto("https://www.pokemon.com/us") + sb.goto("https://www.pokemon.com/us/pokedex") sb.sleep(1.5) sb.click_if_visible("button#onetrust-accept-btn-handler") - sb.sleep(1.2) - sb.click("a span.icon_pokeball") - sb.sleep(2.5) + sb.sleep(1.5) sb.click('b:contains("Show Advanced Search")') sb.sleep(2.5) sb.click('span[data-type="type"][data-value="electric"]') diff --git a/examples/cdp_mode/raw_pokemon.py b/examples/cdp_mode/raw_pokemon.py index aa44ebe5eb4..a084bc5e928 100644 --- a/examples/cdp_mode/raw_pokemon.py +++ b/examples/cdp_mode/raw_pokemon.py @@ -2,12 +2,10 @@ with SB(uc=True, test=True, locale="en", ad_block=True) as sb: sb.activate_cdp_mode() - sb.goto("https://www.pokemon.com/us") + sb.goto("https://www.pokemon.com/us/pokedex") sb.sleep(1.5) sb.click_if_visible("button#onetrust-accept-btn-handler") - sb.sleep(1.2) - sb.click("a span.icon_pokeball") - sb.sleep(2.5) + sb.sleep(1.5) sb.click('b:contains("Show Advanced Search")') sb.sleep(2.5) sb.click('span[data-type="type"][data-value="electric"]') diff --git a/examples/cdp_mode/raw_totalwine.py b/examples/cdp_mode/raw_totalwine.py index 1f3454e9c07..9166dbdc4d2 100644 --- a/examples/cdp_mode/raw_totalwine.py +++ b/examples/cdp_mode/raw_totalwine.py @@ -7,26 +7,27 @@ sb.sleep(1.8) search_box = 'input[data-at="header-search-text"]' search = "The Land by Psagot Cabernet" - if not sb.is_element_present(search_box): - sb.evaluate("window.location.reload();") - sb.sleep(1.8) sb.click_if_visible("#onetrust-close-btn-container button") - sb.sleep(0.5) + sb.sleep(0.6) sb.click_if_visible('button[aria-label="Close modal"]') - sb.sleep(1.2) + sb.sleep(0.6) sb.click(search_box) - sb.sleep(1.2) + sb.sleep(0.6) + sb.click_if_visible('button[aria-label="Close modal"]') + sb.sleep(0.6) sb.press_keys(search_box, search) sb.sleep(0.6) sb.click_if_visible('button[aria-label="Close modal"]') + sb.sleep(0.6) sb.click('button[data-at="header-search-button"]') sb.sleep(1.8) sb.click_if_visible('button[aria-label="Close modal"]') + sb.sleep(0.6) sb.click('img[data-at="product-search-productimage"]') sb.sleep(2.2) print('*** Total Wine Search for "%s":' % search) print(sb.get_text('h1[data-at="product-name-title"]')) - print(sb.get_text('span[data-at="product-mixCaseprice-text"]')) + print(sb.get_text("#priceContainer div")) print("Product Highlights:") print(sb.get_text('p[class*="productInformationReview"]')) print("Product Details:") diff --git a/examples/presenter/uc_presentation_4.py b/examples/presenter/uc_presentation_4.py index fa2a7476f52..454dd188c79 100644 --- a/examples/presenter/uc_presentation_4.py +++ b/examples/presenter/uc_presentation_4.py @@ -462,12 +462,10 @@ def test_presentation_4(self): with SB(uc=True, test=True, locale="en", ad_block=True) as sb: sb.activate_cdp_mode() - sb.goto("https://www.pokemon.com/us") + sb.goto("https://www.pokemon.com/us/pokedex") sb.sleep(1.5) sb.click_if_visible("button#onetrust-accept-btn-handler") - sb.sleep(1.2) - sb.click("a span.icon_pokeball") - sb.sleep(2.5) + sb.sleep(1.5) sb.click('b:contains("Show Advanced Search")') sb.sleep(2.5) sb.click('span[data-type="type"][data-value="electric"]') diff --git a/requirements.txt b/requirements.txt index 74d65780168..061a2e0d7cc 100755 --- a/requirements.txt +++ b/requirements.txt @@ -56,7 +56,7 @@ iniconfig==2.1.0;python_version<"3.10" iniconfig==2.3.0;python_version>="3.10" pluggy==1.6.0 pytest==8.4.2;python_version<"3.11" -pytest==9.1.0;python_version>="3.11" +pytest==9.1.1;python_version>="3.11" pytest-html==4.0.2 pytest-metadata==3.1.1 pytest-ordering==0.6 @@ -79,7 +79,7 @@ rich>=15.0.0,<16 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.) coverage>=7.10.7;python_version<"3.10" -coverage>=7.14.1;python_version>="3.10" +coverage>=7.14.3;python_version>="3.10" pytest-cov>=7.1.0 flake8==7.3.0 mccabe==0.7.0 diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 6aa784b08f0..3765f875930 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.50.1" +__version__ = "4.50.2" diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index eb6991e045d..45bf1c9fc5f 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -2163,6 +2163,10 @@ def click_with_offset(self, selector, x, y, center=False, scroll=True): self.__slow_mode_pause_if_set() self.loop.run_until_complete(self.page.wait(0.2)) + def stop(self): + """Same as quit()""" + self.quit() + def quit(self): """Quit the browser in the Pure CDP Mode Sync format.""" driver = self.driver @@ -3544,8 +3548,9 @@ class Chrome(CDPMethods): def __init__(self, url=None, **kwargs): if not url: url = "about:blank" - driver = cdp_util.start_sync(**kwargs) loop = asyncio.new_event_loop() + kwargs["loop"] = loop + driver = cdp_util.start_sync(**kwargs) page = loop.run_until_complete(driver.get(url)) wait_timeout = 30.0 if hasattr(sb_config, "_cdp_proxy") and sb_config._cdp_proxy: diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index 4b0d435dfb9..eb2b10ea858 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -16841,13 +16841,13 @@ def __process_dashboard(self, has_exception, init=False): '' '' '' - 'Failed: %s' + 'Failed: %s ' '' - 'Skipped: %s' + 'Skipped: %s ' '' - 'Passed: %s' + 'Passed: %s ' '' - 'Untested: %s' + 'Untested: %s ' '' 'Total: %s' "" diff --git a/seleniumbase/undetected/cdp_driver/browser.py b/seleniumbase/undetected/cdp_driver/browser.py index 0c24a3d73f1..a2d88eaa7ee 100644 --- a/seleniumbase/undetected/cdp_driver/browser.py +++ b/seleniumbase/undetected/cdp_driver/browser.py @@ -50,23 +50,29 @@ def deconstruct_browser(): _.config.user_data_dir, ignore_errors=False ) if not os.path.exists(_.config.user_data_dir): + logger.debug( + "Temp profile %s was removed." + % _.config.user_data_dir + ) break else: time.sleep(0.12) except FileNotFoundError: + logger.debug( + "Temp profile %s was removed." % _.config.user_data_dir + ) break except (PermissionError, OSError) as e: if attempt == max_attempts - 1: logger.debug( - "Problem removing data dir %s\n" + "Problem removing data dir %s.\n" "Consider checking whether it's there " - "and remove it by hand\nerror: %s" + "and remove it by hand.\nError: %s" % (_.config.user_data_dir, e) ) break time.sleep(0.12) continue - logging.debug("Temp profile %s was removed." % _.config.user_data_dir) class Browser: @@ -940,10 +946,12 @@ def stop(self, deconstruct=False): logger.debug("Closed the connection using asyncio.run()") except Exception: pass + procs = [] for _ in range(3): try: if connection_id not in sb_config._closed_connection_ids: self._process.terminate() + procs.append(psutil.Process(self._process.pid)) logger.debug( "Terminated browser with pid %d successfully." % self._process.pid @@ -955,20 +963,33 @@ def stop(self, deconstruct=False): except (Exception,): try: self._process.kill() + procs.append(psutil.Process(self._process.pid)) logger.debug( "Killed browser with pid %d successfully." % self._process.pid ) + if connection_id: + sb_config._closed_connection_ids.append(connection_id) + close_success = True break except (Exception,): try: - if hasattr(self, "browser_process_pid"): + if hasattr(self, "_process_pid") and self._process_pid: os.kill(self._process_pid, 15) + try: + procs.append(psutil.Process(self._process_pid)) + except Exception: + pass logger.debug( "Killed browser with pid %d " "using signal 15 successfully." - % self._process.pid + % self._process_pid ) + if connection_id: + sb_config._closed_connection_ids.append( + connection_id + ) + close_success = True break except (TypeError,): logger.info("TypeError", exc_info=True) @@ -976,16 +997,33 @@ def stop(self, deconstruct=False): except (PermissionError,): logger.info( "Browser already stopped, " - "or no permission to kill. Skip." + "or no permission to kill." ) pass except (ProcessLookupError,): logger.info("ProcessLookupError") - pass + break except (Exception,): raise - self._process = None - self._process_pid = None + self._process = None + self._process_pid = None + if procs: + with suppress(Exception): + gone, alive = psutil.wait_procs(procs, timeout=0.8) + for p in gone: + logger.debug("Process has been terminated: %d." % p.pid) + for p in alive: + logger.debug("Process is still alive: %d." % p.pid) + if self.config.user_data_dir and not self.config.uses_custom_data_dir: + for _ in range(3): + try: + time.sleep(0.005) + if os.path.exists(self.config.user_data_dir): + time.sleep(0.005) + shutil.rmtree(self.config.user_data_dir) + break + except Exception: + time.sleep(0.12) if ( hasattr(sb_config, "_xvfb_users") and isinstance(sb_config._xvfb_users, int) @@ -1025,18 +1063,33 @@ def stop(self, deconstruct=False): def silence_pipe_destruction_errors(unraisable): exc_type = unraisable.exc_type exc_value = unraisable.exc_value + exc_value_str = str(exc_value) if exc_value else "" if ( exc_type is ValueError - and "I/O operation on closed pipe" in str(exc_value) + and "I/O operation on closed pipe" in exc_value_str + ): + return + if ( + exc_type is RuntimeError + and "Event loop is closed" in exc_value_str ): return default_unraisablehook(unraisable) sys.unraisablehook = silence_pipe_destruction_errors # Automatically restore Python's default behavior at program exit - atexit.register( - lambda: setattr(sys, "unraisablehook", default_unraisablehook) - ) + # (Looks like this isn't needed, but I'm saving it for reference) + # atexit.register( + # lambda: setattr( + # sys, "unraisablehook", default_unraisablehook + # ) + # ) + + # Custom user_data_dirs should be saved while temp ones are removed + if os.path.exists("%s" % self.config.user_data_dir): + logger.debug("%s still exists." % self.config.user_data_dir) + else: + logger.debug("%s was removed." % self.config.user_data_dir) def quit(self): self.stop() diff --git a/seleniumbase/undetected/cdp_driver/cdp_util.py b/seleniumbase/undetected/cdp_driver/cdp_util.py index 5309f79cd2d..508160e834d 100644 --- a/seleniumbase/undetected/cdp_driver/cdp_util.py +++ b/seleniumbase/undetected/cdp_driver/cdp_util.py @@ -764,14 +764,8 @@ async def start_async(*args, **kwargs) -> Browser: def start_sync(*args, **kwargs) -> Browser: - loop = None - if ( - "loop" in kwargs - and kwargs["loop"] - and hasattr(kwargs["loop"], "create_task") - ): - loop = kwargs["loop"] - else: + loop = kwargs.pop("loop", None) + if not (loop and hasattr(loop, "create_task")): loop = asyncio.new_event_loop() return loop.run_until_complete(start(*args, **kwargs)) diff --git a/seleniumbase/undetected/cdp_driver/config.py b/seleniumbase/undetected/cdp_driver/config.py index e27fe9270e7..986849026bf 100644 --- a/seleniumbase/undetected/cdp_driver/config.py +++ b/seleniumbase/undetected/cdp_driver/config.py @@ -25,7 +25,7 @@ ] logger = logging.getLogger(__name__) -is_posix = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2")) +is_posix = sys.platform.startswith(("darwin", "cygwin", "linux")) PathLike = Union[str, pathlib.Path] AUTO = None diff --git a/seleniumbase/undetected/cdp_driver/element.py b/seleniumbase/undetected/cdp_driver/element.py index 04461c4da4a..2a748d615d7 100644 --- a/seleniumbase/undetected/cdp_driver/element.py +++ b/seleniumbase/undetected/cdp_driver/element.py @@ -372,12 +372,17 @@ async def click_async(self): arguments = [cdp.runtime.CallArgument( object_id=self._remote_object.object_id )] - script = 'sessionStorage.getItem("pxsid") !== null;' + # The next part may be getting detected. Comment-out for now. + '''script1 = 'sessionStorage.getItem("pxsid") !== null;' + script2 = 'sessionStorage.getItem("PIM-SESSION-ID") !== null;' using_px = True + using_pim = True with suppress(Exception): - using_px = await self.tab.evaluate(script) - if not using_px: - await self.flash_async(0.25) + using_px = await self.tab.evaluate(script1) + with suppress(Exception): + using_pim = await self.tab.evaluate(script2) + if not using_px and not using_pim: + await self.flash_async(0.25)''' await self._tab.send( cdp.runtime.call_function_on( "(el) => el.click()", @@ -506,12 +511,17 @@ async def mouse_click_async( logger.warning("Could not calculate box model for %s", self) return logger.debug("Clicking on location: %.2f, %.2f" % center) - script = 'sessionStorage.getItem("pxsid") !== null;' + # The next part may be getting detected. Comment-out for now. + '''script1 = 'sessionStorage.getItem("pxsid") !== null;' + script2 = 'sessionStorage.getItem("PIM-SESSION-ID") !== null;' using_px = True + using_pim = True + with suppress(Exception): + using_px = await self.tab.evaluate(script1) with suppress(Exception): - using_px = await self.tab.evaluate(script) - if not using_px: - asyncio.create_task(self.flash_async(0.25)) + using_pim = await self.tab.evaluate(script2) + if not using_px and not using_pim: + asyncio.create_task(self.flash_async(0.25))''' asyncio.create_task( self._tab.send( cdp.input_.dispatch_mouse_event( @@ -570,11 +580,15 @@ async def mouse_click_with_offset_async( logger.debug("Clicking on location: %.2f, %.2f" % center_pos) else: logger.debug("Clicking on location: %.2f, %.2f" % (x_pos, y_pos)) - script = 'sessionStorage.getItem("pxsid") !== null;' + script1 = 'sessionStorage.getItem("pxsid") !== null;' + script2 = 'sessionStorage.getItem("PIM-SESSION-ID") !== null;' using_px = True + using_pim = True + with suppress(Exception): + using_px = await self.tab.evaluate(script1) with suppress(Exception): - using_px = await self.tab.evaluate(script) - if not using_px: + using_pim = await self.tab.evaluate(script2) + if not using_px and not using_pim: asyncio.create_task( self.flash_async( x_offset=x_offset - (width / 2), diff --git a/setup.py b/setup.py index 47aa6aaebe3..0bb09fdbd42 100755 --- a/setup.py +++ b/setup.py @@ -220,7 +220,7 @@ 'iniconfig==2.3.0;python_version>="3.10"', 'pluggy==1.6.0', 'pytest==8.4.2;python_version<"3.11"', - 'pytest==9.1.0;python_version>="3.11"', + 'pytest==9.1.1;python_version>="3.11"', 'pytest-html==4.0.2', # Newer ones had issues 'pytest-metadata==3.1.1', 'pytest-ordering==0.6', @@ -252,7 +252,7 @@ # Usage: coverage run -m pytest; coverage html; coverage report "coverage": [ 'coverage>=7.10.7;python_version<"3.10"', - 'coverage>=7.14.1;python_version>="3.10"', + 'coverage>=7.14.3;python_version>="3.10"', 'pytest-cov>=7.1.0', ], # pip install -e .[flake8]