From 1ce0d237ff22d83749128164abafa275f828697f Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Thu, 16 Jul 2026 11:06:55 +0200 Subject: [PATCH 1/3] feat: add multi-tab support to WebAgent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track all browser pages in PlaywrightAgentOs via a `_pages` list and a `context.on("page")` listener that captures every new tab as it opens. New tabs automatically become the active tab when `auto_follow_new_tab=True` (the default); set it to `False` to keep the current tab active and switch manually. Three new tools let the LLM manage tabs autonomously: - `list_tabs` — returns index, title, and URL for every open tab - `switch_tab` — makes the tab at the given index active - `close_tab` — closes the tab at the given index (guards the last tab) `WebAgent` exposes `auto_follow_new_tab` as a constructor parameter and includes the three new tools in its default tool set. The system prompt is updated to reflect multi-tab capability. Co-Authored-By: Claude Sonnet 4.6 --- src/askui/prompts/act_prompts.py | 9 +- src/askui/tools/playwright/agent_os.py | 129 ++++++++++++++++++++++++- src/askui/tools/playwright/tools.py | 86 +++++++++++++++++ src/askui/web_agent.py | 20 +++- 4 files changed, 236 insertions(+), 8 deletions(-) diff --git a/src/askui/prompts/act_prompts.py b/src/askui/prompts/act_prompts.py index e2b7af1c..64ce8d08 100644 --- a/src/askui/prompts/act_prompts.py +++ b/src/askui/prompts/act_prompts.py @@ -193,8 +193,11 @@ WEB_BROWSER_CAPABILITIES = """You are an autonomous AI agent that can interact with web interfaces through computer vision and browser control. -* You are utilizing a webbrowser in full-screen mode. So you are only - seeing the content of the currently opened webpage (tab). +* You are utilizing a webbrowser in full-screen mode. You can see and + interact with one tab at a time — the currently active tab. +* You can manage multiple browser tabs using the list_tabs, switch_tab, + and close_tab tools. When a link opens a new tab, use list_tabs to find + it and switch_tab to move to it if needed. * Your primary goal is to execute tasks efficiently and reliably. * Operate independently and make informed decisions without requiring user input. @@ -228,7 +231,7 @@ * Test Device: Yes, with full permissions""" WEB_AGENT_DEVICE_INFORMATION = """* Environment: Web browser in full-screen mode -* Visibility: Only current webpage content (single tab) +* Visibility: Active tab content (use list_tabs / switch_tab to manage multiple tabs) * Interaction: Mouse, keyboard, and browser-specific controls""" # Report Format diff --git a/src/askui/tools/playwright/agent_os.py b/src/askui/tools/playwright/agent_os.py index a56640d6..efcbe4c0 100644 --- a/src/askui/tools/playwright/agent_os.py +++ b/src/askui/tools/playwright/agent_os.py @@ -73,7 +73,7 @@ class PlaywrightAgentOs(ComputerAgentOS): This implementation uses Playwright's Python SDK to control browser automation and simulate user interactions. It provides mouse control, keyboard input, - and screen capture functionality through a browser context. + screen capture, and multi-tab management functionality through a browser context. Args: reporter (Reporter, optional): Reporter used for reporting. Defaults to @@ -96,6 +96,11 @@ class PlaywrightAgentOs(ComputerAgentOS): When ``None``, downloads are left in Playwright's temporary location (and deleted when the browser closes). The directory is created if it does not exist. Defaults to `None`. + auto_follow_new_tab (bool, optional): When `True`, any new tab opened by the + browser (e.g. via ``target="_blank"`` links or ``window.open()``) + automatically becomes the active tab. When `False`, new tabs are tracked + but the active tab does not change; use `switch_tab()` to move to them + manually. Defaults to `True`. """ _REPORTER_ROLE_NAME: str = "PlaywrightAgentOS" @@ -110,6 +115,7 @@ def __init__( install_browser: bool = True, install_dependencies: bool = False, download_dir: str | Path | None = None, + auto_follow_new_tab: bool = True, ) -> None: self._browser_type = browser_type self._headless = headless @@ -118,12 +124,14 @@ def __init__( self._install_browser = install_browser self._install_dependencies = install_dependencies self._download_dir = Path(download_dir) if download_dir is not None else None + self._auto_follow_new_tab = auto_follow_new_tab # Playwright objects self._playwright: Playwright | None = None self._browser: Browser | None = None self._context: BrowserContext | None = None self._page: Page | None = None + self._pages: list[Page] = [] self._reporter: Reporter = reporter # Event listening state @@ -224,8 +232,11 @@ def connect(self) -> None: no_viewport=True, ) + self._pages = [] self._page = self._context.new_page() + self._pages.append(self._page) self._page.on("download", self._on_download) + self._context.on("page", self._on_new_page) # Navigate to a blank page to ensure we have a working page self._page.goto("data:text/html,

Starting...

") self._reporter.add_message( @@ -233,6 +244,27 @@ def connect(self) -> None: "Connected to playwright browser", ) + def _on_new_page(self, page: Page) -> None: + """Track a newly opened browser tab and optionally make it active. + + Registered as a ``page`` event handler on the `BrowserContext`. Called + whenever a new tab is opened (e.g. via ``target="_blank"`` links or + ``window.open()``). When `auto_follow_new_tab` is `True`, the new page + immediately becomes the active page; otherwise it is only tracked. + + Args: + page (Page): The newly opened Playwright page. + """ + page.on("download", self._on_download) + self._pages.append(page) + if self._auto_follow_new_tab: + self._page = page + page.bring_to_front() + self._reporter.add_message( + self._REPORTER_ROLE_NAME, + f"New tab opened: '{page.url}'", + ) + def _on_download(self, download: Download) -> None: """Register a started download for deterministic saving. @@ -380,9 +412,9 @@ def disconnect(self) -> None: # the copy and leaves a truncated file on disk. self._deliver_and_flush_downloads() - if self._page: - self._page.close() - self._page = None + # Clear page tracking; context.close() handles actual page teardown. + self._pages.clear() + self._page = None if self._context: self._context.close() @@ -705,6 +737,95 @@ def retrieve_active_display(self) -> Display: ), ) + # --- Tab management --- + + def list_tabs(self) -> list[dict[str, int | str]]: + """Return metadata for every currently open browser tab. + + Returns: + list[dict[str, int | str]]: One entry per open tab, each containing: + + - ``index`` (int): Zero-based tab index used by `switch_tab` and + `close_tab`. + - ``title`` (str): Page title (empty string if unavailable). + - ``url`` (str): Current URL of the page. + """ + tabs: list[dict[str, int | str]] = [] + for i, page in enumerate(self._pages): + try: + title: str = page.title() + except Exception: # noqa: BLE001 + title = "" + tabs.append({"index": i, "title": title, "url": page.url}) + return tabs + + def switch_tab(self, index: int) -> None: + """Switch the active browser tab to the tab at ``index``. + + Args: + index (int): Zero-based index of the tab to activate (see + `list_tabs` for available indices). + + Raises: + RuntimeError: If no browser session is active. + IndexError: If ``index`` is out of range. + """ + if not self._pages: + error_msg = "No open tabs. Call connect() first." + raise RuntimeError(error_msg) + if index < 0 or index >= len(self._pages): + error_msg = ( + f"Tab index {index} is out of range. " + f"Available indices: 0-{len(self._pages) - 1}." + ) + raise IndexError(error_msg) + self._page = self._pages[index] + self._page.bring_to_front() + self._reporter.add_message( + self._REPORTER_ROLE_NAME, + f"switch_tab(index={index}) -> '{self._page.url}'", + ) + + def close_tab(self, index: int) -> None: + """Close the browser tab at ``index``. + + When the closed tab was the active one, the agent automatically + switches to the nearest remaining tab (the tab at ``index - 1``, or + the new last tab when ``index`` was the last one). + + Args: + index (int): Zero-based index of the tab to close (see `list_tabs` + for available indices). + + Raises: + RuntimeError: If no browser session is active or if only one tab + remains (closing it would leave the browser with no pages). + IndexError: If ``index`` is out of range. + """ + if not self._pages: + error_msg = "No open tabs. Call connect() first." + raise RuntimeError(error_msg) + if len(self._pages) <= 1: + error_msg = "Cannot close the last remaining tab." + raise RuntimeError(error_msg) + if index < 0 or index >= len(self._pages): + error_msg = ( + f"Tab index {index} is out of range. " + f"Available indices: 0-{len(self._pages) - 1}." + ) + raise IndexError(error_msg) + page_to_close = self._pages.pop(index) + was_active = self._page is page_to_close + page_to_close.close() + if was_active: + new_index = min(index, len(self._pages) - 1) + self._page = self._pages[new_index] + self._page.bring_to_front() + self._reporter.add_message( + self._REPORTER_ROLE_NAME, + f"close_tab(index={index})", + ) + def _convert_key(self, key: PcKey | ModifierKey) -> str: """ Convert our key format to Playwright's key format. diff --git a/src/askui/tools/playwright/tools.py b/src/askui/tools/playwright/tools.py index b7d52141..d56b155b 100644 --- a/src/askui/tools/playwright/tools.py +++ b/src/askui/tools/playwright/tools.py @@ -544,3 +544,89 @@ def __init__(self, agent_os: PlaywrightAgentOs | None = None) -> None: def __call__(self) -> str: url = self.agent_os.get_page_url() return f"Current page URL: {url}" + + +class PlaywrightListTabsTool(PlaywrightBaseTool): + """Lists all open browser tabs with their index, title, and URL.""" + + def __init__(self, agent_os: PlaywrightAgentOs | None = None) -> None: + super().__init__( + name="list_tabs", + description=( + "List all currently open browser tabs. " + "Returns the index, title, and URL of each tab. " + "Use the index with switch_tab or close_tab." + ), + agent_os=agent_os, + ) + + @override + def __call__(self) -> str: + tabs = self.agent_os.list_tabs() + if not tabs: + return "No open tabs." + lines = [f"[{t['index']}] {t['title']} — {t['url']}" for t in tabs] + return "Open tabs:\n" + "\n".join(lines) + + +class PlaywrightSwitchTabTool(PlaywrightBaseTool): + """Switches the active browser tab to a tab at a given index.""" + + def __init__(self, agent_os: PlaywrightAgentOs | None = None) -> None: + super().__init__( + name="switch_tab", + description=( + "Switch to a different browser tab by its index. " + "Use list_tabs to see all open tabs and their indices." + ), + input_schema={ + "type": "object", + "properties": { + "index": { + "type": "integer", + "description": ( + "Zero-based index of the tab to switch to (from list_tabs)." + ), + }, + }, + "required": ["index"], + }, + agent_os=agent_os, + ) + + @override + def __call__(self, index: int) -> str: + self.agent_os.switch_tab(index) + return f"Switched to tab {index}." + + +class PlaywrightCloseTabTool(PlaywrightBaseTool): + """Closes a browser tab by its index.""" + + def __init__(self, agent_os: PlaywrightAgentOs | None = None) -> None: + super().__init__( + name="close_tab", + description=( + "Close a browser tab by its index. " + "Use list_tabs to see available tabs. " + "The last remaining tab cannot be closed." + ), + input_schema={ + "type": "object", + "properties": { + "index": { + "type": "integer", + "description": ( + "Zero-based index of the tab to close (from list_tabs)." + ), + }, + }, + "required": ["index"], + }, + agent_os=agent_os, + ) + + @override + def __call__(self, index: int) -> str: + self.agent_os.close_tab(index) + return f"Tab {index} was closed." diff --git a/src/askui/web_agent.py b/src/askui/web_agent.py index cf2b4981..4df9c97a 100644 --- a/src/askui/web_agent.py +++ b/src/askui/web_agent.py @@ -21,6 +21,7 @@ from askui.tools.playwright.agent_os_facade import PlaywrightAgentOsFacade from askui.tools.playwright.tools import ( PlaywrightBackTool, + PlaywrightCloseTabTool, PlaywrightForwardTool, PlaywrightGetPageTitleTool, PlaywrightGetPageUrlTool, @@ -28,12 +29,14 @@ PlaywrightKeyboardPressedTool, PlaywrightKeyboardReleaseTool, PlaywrightKeyboardTapTool, + PlaywrightListTabsTool, PlaywrightMouseClickTool, PlaywrightMouseHoldDownTool, PlaywrightMouseMoveTool, PlaywrightMouseReleaseTool, PlaywrightMouseScrollTool, PlaywrightScreenshotTool, + PlaywrightSwitchTabTool, PlaywrightTypeTool, ) @@ -69,6 +72,12 @@ class WebAgent(Agent): (auto-renamed on filename collision). When `None`, downloads are left in Playwright's temporary location and removed when the browser closes. Defaults to `None`. + auto_follow_new_tab (bool, optional): When `True`, any new tab opened by the + browser (e.g. via ``target="_blank"`` links or ``window.open()``) + automatically becomes the active tab so subsequent actions target it. + When `False`, new tabs are tracked but the active tab does not change; + use `agent.os.switch_tab()` or the `switch_tab` tool to move to them + manually. Defaults to `True`. Example: ```python @@ -88,6 +97,7 @@ class WebAgent(Agent): "truncation_strategy", "secrets", "download_dir", + "auto_follow_new_tab", } ) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) @@ -101,9 +111,14 @@ def __init__( truncation_strategy: TruncationStrategy | None = None, secrets: list[Secret] | None = None, download_dir: str | Path | None = None, + auto_follow_new_tab: bool = True, ) -> None: reporter = CompositeReporter(reporters=reporters) - self.os = PlaywrightAgentOs(reporter, download_dir=download_dir) + self.os = PlaywrightAgentOs( + reporter, + download_dir=download_dir, + auto_follow_new_tab=auto_follow_new_tab, + ) super().__init__( reporter=reporter, retry=retry, @@ -158,6 +173,9 @@ def get_default_tools() -> list[Tool]: PlaywrightForwardTool(), PlaywrightGetPageTitleTool(), PlaywrightGetPageUrlTool(), + PlaywrightListTabsTool(), + PlaywrightSwitchTabTool(), + PlaywrightCloseTabTool(), ExceptionTool(), ] From 458dcca492c7f71d6ff1d679c0869b5381e27047 Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Thu, 16 Jul 2026 11:35:55 +0200 Subject: [PATCH 2/3] fix: correct auto-follow-new-tab for Playwright sync API Two bugs prevented auto-follow from working: 1. bring_to_front() deadlock: the _on_new_page callback runs in Playwright's background asyncio thread. Calling sync Playwright methods from that context deadlocks the greenlet dispatcher. Removed bring_to_front() from the handler; only the thread-safe self._page = page assignment is kept. 2. Race condition: the background-thread event fires after the main thread has already moved on to the next call (e.g. screenshot()), so self._page isn't updated in time. Added _sync_pages(), which is called at the start of every screenshot() from the main thread. It polls context.pages (safe from main thread), picks up any pages the event handler missed, and calls bring_to_front() safely there. Co-Authored-By: Claude Sonnet 4.6 --- src/askui/tools/playwright/agent_os.py | 57 +++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/src/askui/tools/playwright/agent_os.py b/src/askui/tools/playwright/agent_os.py index efcbe4c0..b60adbeb 100644 --- a/src/askui/tools/playwright/agent_os.py +++ b/src/askui/tools/playwright/agent_os.py @@ -245,12 +245,19 @@ def connect(self) -> None: ) def _on_new_page(self, page: Page) -> None: - """Track a newly opened browser tab and optionally make it active. + """Track a newly opened browser tab. Registered as a ``page`` event handler on the `BrowserContext`. Called - whenever a new tab is opened (e.g. via ``target="_blank"`` links or - ``window.open()``). When `auto_follow_new_tab` is `True`, the new page - immediately becomes the active page; otherwise it is only tracked. + from Playwright's background asyncio thread whenever a new tab is opened + (e.g. via ``target="_blank"`` links or ``window.open()``). + + Only thread-safe operations are performed here: appending to the tracked + list and reassigning ``self._page``. Sync Playwright methods such as + ``bring_to_front()`` must NOT be called here — they require the main + greenlet context and deadlock when invoked from the background thread. + Auto-follow (including ``bring_to_front()``) is completed on the main + thread by `_sync_pages`, which is called at the start of every + `screenshot`. Args: page (Page): The newly opened Playwright page. @@ -259,12 +266,47 @@ def _on_new_page(self, page: Page) -> None: self._pages.append(page) if self._auto_follow_new_tab: self._page = page - page.bring_to_front() self._reporter.add_message( self._REPORTER_ROLE_NAME, f"New tab opened: '{page.url}'", ) + def _sync_pages(self) -> None: + """Sync tracked pages with the live browser context, main-thread safe. + + Detects pages that appeared after the last ``_on_new_page`` callback + (e.g. due to the race between the background event thread and the main + thread), registers their download listeners, appends them to + ``_pages``, and — when ``auto_follow_new_tab`` is ``True`` — makes the + newest one active and calls ``bring_to_front()`` so the browser UI + reflects the switch. + + Also prunes any pages that have been closed since the last sync. + Safe to call from the main thread at the start of every ``screenshot``. + """ + if self._context is None: + return + + known_ids = {id(p) for p in self._pages} + new_pages: list[Page] = [] + for page in self._context.pages: + if id(page) not in known_ids: + page.on("download", self._on_download) + self._pages.append(page) + known_ids.add(id(page)) + new_pages.append(page) + + if new_pages and self._auto_follow_new_tab: + self._page = new_pages[-1] + + # Prune closed pages + self._pages = [p for p in self._pages if not p.is_closed()] + if self._page is not None and self._page.is_closed(): + self._page = self._pages[-1] if self._pages else None + + if self._page is not None and (new_pages and self._auto_follow_new_tab): + self._page.bring_to_front() + def _on_download(self, download: Download) -> None: """Register a started download for deterministic saving. @@ -453,6 +495,11 @@ def screenshot(self, report: bool = True, unscaled: bool = False) -> Image.Image error_msg = "No active page. Call connect() first." raise RuntimeError(error_msg) + # Sync page tracking on the main thread before capturing. This catches + # new tabs opened since the last call (the background-thread event + # handler may have lost the race) and calls bring_to_front() safely. + self._sync_pages() + screenshot_bytes = self._page.screenshot(scale="css") screenshot = Image.open(io.BytesIO(screenshot_bytes)) # Taking the screenshot pumps the event loop, so any download that From 5ad260ab65dbdfa108064629da96304900485540 Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Thu, 16 Jul 2026 11:39:50 +0200 Subject: [PATCH 3/3] fix: reliably flush new-tab events before screenshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fix removed bring_to_front() from the event handler but left a timing race: the "page" event only fires when the Playwright event loop is pumped, which happens during page.screenshot(). By that point the CDP screenshot command has already been sent to the OLD page, so auto-follow had no effect on what the agent saw. Fix: pump the event loop (wait_for_timeout(0)) at the start of screenshot() BEFORE _sync_pages() is called. This forces the dispatcher to process all queued events — including new-page events — so that by the time the screenshot is taken, self._page already points to the new tab. Also introduce a _needs_bring_to_front flag. _on_new_page sets it instead of calling bring_to_front() directly; _sync_pages() clears it and calls bring_to_front() from the main greenlet after the pump. This avoids silent error-deferral: any exception raised by a sync Playwright call inside an EventGreenlet is stored in connection._error and re-raised at the next API call, which could corrupt an unrelated operation. Co-Authored-By: Claude Sonnet 4.6 --- src/askui/tools/playwright/agent_os.py | 52 +++++++++++++++++--------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/src/askui/tools/playwright/agent_os.py b/src/askui/tools/playwright/agent_os.py index b60adbeb..7503cfb1 100644 --- a/src/askui/tools/playwright/agent_os.py +++ b/src/askui/tools/playwright/agent_os.py @@ -134,6 +134,10 @@ def __init__( self._pages: list[Page] = [] self._reporter: Reporter = reporter + # Set to True by _on_new_page when a new tab is followed; cleared by + # _sync_pages() after calling bring_to_front() on the main thread. + self._needs_bring_to_front: bool = False + # Event listening state self._listening = False self._event_queue: list[InputEvent] = [] @@ -266,6 +270,12 @@ def _on_new_page(self, page: Page) -> None: self._pages.append(page) if self._auto_follow_new_tab: self._page = page + # Signal _sync_pages() to call bring_to_front() on the main thread. + # bring_to_front() is deliberately NOT called here: even though + # Playwright's EventGreenlet supports sync calls, any exception it + # raises would be silently deferred and re-raised at the next API + # call, which could fail an unrelated operation such as screenshot(). + self._needs_bring_to_front = True self._reporter.add_message( self._REPORTER_ROLE_NAME, f"New tab opened: '{page.url}'", @@ -274,37 +284,41 @@ def _on_new_page(self, page: Page) -> None: def _sync_pages(self) -> None: """Sync tracked pages with the live browser context, main-thread safe. - Detects pages that appeared after the last ``_on_new_page`` callback - (e.g. due to the race between the background event thread and the main - thread), registers their download listeners, appends them to - ``_pages``, and — when ``auto_follow_new_tab`` is ``True`` — makes the - newest one active and calls ``bring_to_front()`` so the browser UI - reflects the switch. - - Also prunes any pages that have been closed since the last sync. - Safe to call from the main thread at the start of every ``screenshot``. + Called at the start of every ``screenshot`` (after ``_pump_event_loop`` + has flushed all pending page events). Does three things: + + 1. Picks up any pages that ``_on_new_page`` could not track because of + the race between the dispatcher firing the event and ``screenshot`` + being called — after the pump the event has already fired, so this + acts as a safety net rather than the primary detection path. + 2. Prunes pages that have been closed since the last call. + 3. Calls ``bring_to_front()`` on the main thread when + ``_needs_bring_to_front`` is set. The flag is raised by + ``_on_new_page`` instead of calling ``bring_to_front()`` there + directly, because a deferred exception from inside an EventGreenlet + would be re-raised at the next API call and silently break + unrelated operations. """ if self._context is None: return known_ids = {id(p) for p in self._pages} - new_pages: list[Page] = [] for page in self._context.pages: if id(page) not in known_ids: page.on("download", self._on_download) self._pages.append(page) known_ids.add(id(page)) - new_pages.append(page) - - if new_pages and self._auto_follow_new_tab: - self._page = new_pages[-1] + if self._auto_follow_new_tab: + self._page = page + self._needs_bring_to_front = True # Prune closed pages self._pages = [p for p in self._pages if not p.is_closed()] if self._page is not None and self._page.is_closed(): self._page = self._pages[-1] if self._pages else None - if self._page is not None and (new_pages and self._auto_follow_new_tab): + if self._needs_bring_to_front and self._page is not None: + self._needs_bring_to_front = False self._page.bring_to_front() def _on_download(self, download: Download) -> None: @@ -495,9 +509,11 @@ def screenshot(self, report: bool = True, unscaled: bool = False) -> Image.Image error_msg = "No active page. Call connect() first." raise RuntimeError(error_msg) - # Sync page tracking on the main thread before capturing. This catches - # new tabs opened since the last call (the background-thread event - # handler may have lost the race) and calls bring_to_front() safely. + # Pump the event loop before syncing so any pending "page" events + # (new tabs opened by the previous action) are processed first. + # Without this, the new-page event fires during page.screenshot() + # after the CDP command has already been sent to the old page. + self._pump_event_loop(0) self._sync_pages() screenshot_bytes = self._page.screenshot(scale="css")