Browser automation

┌────────────────────┐
                 │  ComputerUseAgent  │
                 │  ──────────────────│
                 │  goal              │
                 │  llm               │
                 │  browser           │  (BrowserSession protocol)
                 │  max_iterations    │
                 └──────────┬─────────┘
                            │ screenshot + goal
                            ▼
                  ┌─────────────────┐
                  │   vision LLM    │
                  └─────────┬───────┘
                            │ ACTION: click 100,200
                            ▼
                  ┌─────────────────┐
                  │  parse_action() │ → ComputerAction
                  └─────────┬───────┘
                            │ click(100, 200)
                            ▼
                  ┌─────────────────┐
                  │ BrowserSession  │  (Playwright in prod, mock in tests)
                  └─────────────────┘

from shipit_agent.computer_use import ComputerUseAgent, MockBrowserSession

class StubLLM:
    def __init__(self, replies): self._r, self._i = replies, 0
    def complete(self, *, messages, **_):
        text, self._i = self._r[self._i], self._i + 1
        return text

llm = StubLLM(["I'll start at apple.com.\nACTION: navigate https://apple.com",
    "Navigate to iPhone.\nACTION: click 240,80",
    "Open the 15 Pro page.\nACTION: click 600,400",
    "Found it: $999.\nACTION: done iPhone 15 Pro starts at $999.",])

agent = ComputerUseAgent(
    llm=llm,
    browser=MockBrowserSession(),
    goal="Find iPhone 15 Pro starting price on apple.com",
    max_iterations=8,
)

result = agent.run()
print(result.status)        # "done"
print(result.final_text)    # "iPhone 15 Pro starts at $999."
print(result.iterations)    # 4

pip install playwright
playwright install chromium

from shipit_agent.computer_use import (
    ComputerUseAgent, PlaywrightBrowserSession,
)

with PlaywrightBrowserSession.launch(
    headless=True,                # set False to watch it run
    viewport_size=(1280, 720),
    start_url="about:blank",
) as browser:
    agent = ComputerUseAgent(
        llm=opus_llm,             # Claude with computer-use beta access
        browser=browser,
        goal="Find the cheapest direct SFO → JFK flight on May 20.",
        max_iterations=15,
    )
    result = agent.run()
    print(result.final_text)

ACTION: click X,Y           ← left-click at pixel (x, y)
ACTION: type "text"         ← type into the focused element
ACTION: key Enter           ← press a special key
ACTION: scroll DX DY        ← scroll by (dx, dy) pixels
ACTION: scroll 600          ← single int = vertical scroll
ACTION: navigate URL        ← open a URL
ACTION: screenshot          ← re-read current state
ACTION: done <answer>       ← end the run with a final answer

{
  "type": "tool_use",
  "name": "computer",
  "input": {
    "action": "left_click",
    "coordinate": [320, 180],
    "rationale": "Clicking the search icon."
  }
}

result = agent.run()  # goal: "find the lowest price for X on these 3 sites"
# Action history shows every navigation + screenshot + click;
# result.final_text has the comparison.

goal = (
    "Fill the contact form with name='Alice Cooper', "
    "email='alice@example.com', message='Interested in Pro plan.'. "
    "Pause before submitting."
)
result = agent.run()
# Run completes when the agent reaches the 'submit' button.
# The human reviews the screenshot, then clicks for real.

goal = (
    "Sign up for an account with email='test+{ts}@example.com' "
    "and verify the dashboard loads with the welcome banner. "
    "Report PASS or FAIL."
)
result = agent.run()
assert "PASS" in result.final_text

class FlakyBrowser(MockBrowserSession):
    def click(self, x, y):
        if self._first:
            self._first = False
            raise RuntimeError("Element not visible at (x, y)")
        super().click(x, y)

from shipit_agent.verifier import VerifierNetwork

verifier = VerifierNetwork(llm=haiku_llm, goal="Look up prices, no purchases")
# Wrap any tool the agent has access to — the verifier vetoes destructive
# actions (e.g. clicking 'Buy now' when the goal is just to look)

class VNCBrowserSession:
    def __init__(self, host: str): self._host = host
    def screenshot(self) -> str:    # base64 PNG
        ...
    def click(self, x, y): ...
    def type_text(self, text): ...
    def key(self, k): ...
    def scroll(self, dx, dy): ...
    def navigate(self, url): ...
    def close(self): ...

agent = ComputerUseAgent(llm=llm, browser=VNCBrowserSession("...:5900"), goal="...")

ComputerUseAgent(
    llm=...,
    browser=...,
    goal="...",
    max_iterations=10,                 # safety cap on think-act cycles
    viewport_size=(1280, 720),         # for LLM context
    action_emit_mode="auto",           # "auto" / "anthropic" / "text"
    screenshot_format="png",           # "png" / "jpeg"
)

ComputerUseAgent(
    *, llm,
    browser: BrowserSession,
    goal: str,
    max_iterations: int = 10,
    viewport_size: tuple[int, int] | None = None,
    action_emit_mode: str = "auto",
    screenshot_format: str = "png",
)

PlaywrightBrowserSession.launch(
    *, headless=True,
    viewport_size=(1280, 720),
    start_url="about:blank",
) -> PlaywrightBrowserSession  # also context manager

from shipit_agent import Agent, VerifierNetwork
from shipit_agent.computer_use import (
    BrowserAgentTool, PlaywrightBrowserSession,
)

# 1. Build the browser tool — it owns the sub-agent's LLM and browser factory
browser_tool = BrowserAgentTool(
    llm=opus_llm,                                    # vision-capable
    browser_factory=lambda: PlaywrightBrowserSession.launch(headless=True),
    max_iterations=12,
)

# 2. Optional: add a verifier so destructive actions get gated
verifier = VerifierNetwork(llm=haiku_llm, goal="Research only — no purchases.")

# 3. Hand the browser_use tool to your main planning agent
agent = Agent(
    llm=opus_llm,
    tools=[browser_tool, WebSearchTool(), PDFTool()],
    verifier=verifier,
)

# 4. Run a high-level goal — the main agent decides when to call browser_use
result = agent.run(
    "Find the cheapest direct SFO-JFK flight on May 20 "
    "and summarise the booking page."
)

{
  "type": "object",
  "properties": {
    "goal": {
      "type": "string",
      "description": "What the browser sub-agent should accomplish, in plain English."
    }
  },
  "required": ["goal"]
}

Browser sub-agent finished (status=done, iterations=4).
Goal: Find the cheapest direct SFO-JFK flight on May 20.

Answer: Cheapest direct is JetBlue B6 1234 at $187, departs 11:30am.

Last actions:
  · iter 0: navigate
  · iter 1: type
  · iter 2: click
  · iter 3: done

browser_tool = BrowserAgentTool(
    llm=vision_llm,
    browser_factory=lambda: PlaywrightBrowserSession.launch(headless=False),
    share_browser=True,                              # ← reuse session
)

# Each tool call now operates on the same browser, preserving state
agent.run("Log in to the dashboard with credentials from the vault.")
agent.run("Open the billing page.")
agent.run("Download the latest invoice as a PDF.")

# Always close at the end
browser_tool.close()

from shipit_agent.computer_use import BrowserAgentTool, MockBrowserSession

tool = BrowserAgentTool(
    llm=fake_llm,
    browser_factory=lambda: MockBrowserSession(),
)
result = tool.run(goal="testing — no real browser needed")