feat: Browserbase Web Reader

docs/docs/examples/data_connectors/WebPageDemo.ipynb

@@ -130,6 +130,58 @@
  "display(Markdown(f\"<b>{response}</b>\"))"
  ]
  },
+ {
+ "cell_type": "markdown",
+ "id": "005d14cd",
+ "metadata": {},
+ "source": [
+ "# Using Browserbase Reader 🅱️\n",
+ "\n",
+ "[Browserbase](https://browserbase.com) is a serverless platform for running headless browsers, it offers advanced debugging, session recordings, stealth mode, integrated proxies and captcha solving.\n",
+ "\n",
+ "## Installation and Setup\n",
+ "\n",
+ "- Get an API key from [browserbase.com](https://browserbase.com) and set it in environment variables (`BROWSERBASE_API_KEY`).\n",
+ "- Install the [Browserbase SDK](http://github.com/browserbase/python-sdk):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c74e6425",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "% pip install browserbase"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c23d02bc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from llama_index.readers.web import BrowserbaseWebReader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7e71d347",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "reader = BrowserbaseWebReader()\n",
+ "docs = reader.load_data(\n",
+ " urls=[\n",
+ " \"https://example.com\",\n",
+ " ],\n",
+ " # Text mode\n",
+ " text_content=False,\n",
+ ")"
+ ]
+ },
  {
  "cell_type": "markdown",
  "id": "15f46387",

llama-index-integrations/readers/llama-index-readers-web/llama_index/readers/web/__init__.py

@@ -5,6 +5,7 @@
 from llama_index.readers.web.beautiful_soup_web.base import (
  BeautifulSoupWebReader,
 )
+from llama_index.readers.web.browserbase.base import BrowserbaseWebReader
 from llama_index.readers.web.firecrawl_web.base import FireCrawlWebReader
 from llama_index.readers.web.knowledge_base.base import (
  KnowledgeBaseWebReader,
@@ -42,6 +43,7 @@
 __all__ = [
  "AsyncWebPageReader",
  "BeautifulSoupWebReader",
+ "BrowserbaseWebReader",
  "FireCrawlWebReader",
  "KnowledgeBaseWebReader",
  "MainContentExtractorReader",

llama-index-integrations/readers/llama-index-readers-web/llama_index/readers/web/browserbase_web/BUILD

@@ -0,0 +1,5 @@
+python_sources()
+
+python_requirements(
+ name="reqs",
+)

llama-index-integrations/readers/llama-index-readers-web/llama_index/readers/web/browserbase_web/README.md

@@ -0,0 +1,47 @@
+# Browserbase Web Reader
+
+[Browserbase](https://browserbase.com) is a serverless platform for running headless browsers, it offers advanced debugging, session recordings, stealth mode, integrated proxies and captcha solving.
+
+## Installation and Setup
+
+- Get an API key from [browserbase.com](https://browserbase.com) and set it in environment variables (`BROWSERBASE_API_KEY`).
+- Install the [Browserbase SDK](http://github.com/browserbase/python-sdk):
+
+```
+pip install browserbase
+```
+
+## Usage
+
+### Loading documents
+
+You can load webpages into LlamaIndex using `BrowserbaseWebReader`. Optionally, you can set `text_content` parameter to convert the pages to text-only representation.
+
+```python
+from llama_index.readers.web import BrowserbaseWebReader
+
+
+reader = BrowserbaseWebReader()
+docs = reader.load_data(
+ urls=[
+ "https://example.com",
+ ],
+ # Text mode
+ text_content=False,
+)
+```
+
+### Loading images
+
+You can also load screenshots of webpages (as bytes) for multi-modal models.
+
+```python
+from browserbase import Browserbase
+from base64 import b64encode
+
+browser = Browserbase()
+screenshot = browser.screenshot("https://browserbase.com")
+
+# Optional. Convert to base64
+img_encoded = b64encode(screenshot).decode()
+```

llama-index-integrations/readers/llama-index-readers-web/llama_index/readers/web/browserbase_web/base.py

@@ -0,0 +1,48 @@
+import logging
+from typing import Optional, Iterator, Sequence
+from llama_index.core.readers.base import BaseReader
+from llama_index.core.schema import Document
+
+
+logger = logging.getLogger(__name__)
+
+
+class BrowserbaseWebReader(BaseReader):
+ """BrowserbaseWebReader.
+
+ Load pre-rendered web pages using a headless browser hosted on Browserbase.
+ Depends on `browserbase` package.
+ Get your API key from https://browserbase.com
+ """
+
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ ) -> None:
+ try:
+ from browserbase import Browserbase
+ except ImportError:
+ raise ImportError(
+ "`browserbase` package not found, please run `pip install browserbase`"
+ )
+
+ self.browserbase = Browserbase(api_key=api_key)
+
+ def lazy_load_data(
+ self, urls: Sequence[str], text_content: bool = False
+ ) -> Iterator[Document]:
+ """Load pages from URLs."""
+ pages = self.browserbase.load_urls(urls, text_content)
+
+ for i, page in enumerate(pages):
+ yield Document(
+ text=page,
+ metadata={
+ "url": urls[i],
+ },
+ )
+
+
+if __name__ == "__main__":
+ reader = BrowserbaseWebReader()
+ logger.info(reader.load_data(urls=["https://example.com"]))

llama-index-integrations/readers/llama-index-readers-web/llama_index/readers/web/browserbase_web/requirements.txt

@@ -0,0 +1 @@
+browserbase