02_Document_Splitting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ccc8565c",
   "metadata": {},
   "source": [
    "# Document Splitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "361fa867",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "from langchain.text_splitter import (\n",
    "    RecursiveCharacterTextSplitter,\n",
    "    CharacterTextSplitter,\n",
    "    TokenTextSplitter,\n",
    "    MarkdownHeaderTextSplitter\n",
    ")\n",
    "from langchain.document_loaders import PyPDFLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8f16f2f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "api_file = open(\"../api_key.txt\")\n",
    "api_key = api_file.read()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c77a815e",
   "metadata": {},
   "source": [
    "## Text Splitter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9a070d8",
   "metadata": {},
   "source": [
    "### Comparison between Recursive and Character Text Splitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "db181dcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "chunk_size = 26\n",
    "chunk_overlap = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cae3967c",
   "metadata": {},
   "outputs": [],
   "source": [
    "recursive_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=chunk_size,\n",
    "    chunk_overlap=chunk_overlap\n",
    ")\n",
    "\n",
    "character_splitter = CharacterTextSplitter(\n",
    "    chunk_size=chunk_size,\n",
    "    chunk_overlap=chunk_overlap\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "290eb685",
   "metadata": {},
   "outputs": [],
   "source": [
    "text1 = \"abcdefghijklmnopqrstuvwxyz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e496fc58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abcdefghijklmnopqrstuvwxyz']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recursive_splitter.split_text(text1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ad6c2277",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abcdefghijklmnopqrstuvwxyz']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "character_splitter.split_text(text1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bf7bc11d",
   "metadata": {},
   "outputs": [],
   "source": [
    "text2 = 'abcdefghijklmnopqrstuvwxyzabcdefg'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b6d75f4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abcdefghijklmnopqrstuvwxyz', 'wxyzabcdefg']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recursive_splitter.split_text(text2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "36c29a62",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abcdefghijklmnopqrstuvwxyzabcdefg']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "character_splitter.split_text(text2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6f26b28c",
   "metadata": {},
   "outputs": [],
   "source": [
    "text3 = \"a b c d e f g h i j k l m n o p q r s t u v w x y z\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "806c1db0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a b c d e f g h i j k l m', 'l m n o p q r s t u v w x', 'w x y z']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recursive_splitter.split_text(text3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "23dbfcdd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a b c d e f g h i j k l m n o p q r s t u v w x y z']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "character_splitter.split_text(text3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a57545fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a b c d e f g h i j k l m', 'l m n o p q r s t u v w x', 'w x y z']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "character_splitter = CharacterTextSplitter(\n",
    "    chunk_size=chunk_size,\n",
    "    chunk_overlap=chunk_overlap,\n",
    "    separator = ' '\n",
    ")\n",
    "\n",
    "character_splitter.split_text(text3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa8f295c",
   "metadata": {},
   "source": [
    "### Recursive Splitting Details\n",
    "\n",
    "RecursiveCharacterTextSplitter is recommended for generic text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5b9a2ddc",
   "metadata": {},
   "outputs": [],
   "source": [
    "some_text = \"\"\"When writing documents, writers will use document structure to group content. \\\n",
    "This can convey to the reader, which idea's are related. For example, closely related ideas \\\n",
    "are in sentances. Similar ideas are in paragraphs. Paragraphs form a document. \\n\\n  \\\n",
    "Paragraphs are often delimited with a carriage return or two carriage returns. \\\n",
    "Carriage returns are the \"backslash n\" you see embedded in this string. \\\n",
    "Sentences have a period at the end, but also, have a space.\\\n",
    "and words are separated by space.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c4bbf3e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "496"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(some_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9ab3bc21",
   "metadata": {},
   "outputs": [],
   "source": [
    "chunk_size = 450\n",
    "chunk_overlap = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "416d4741",
   "metadata": {},
   "outputs": [],
   "source": [
    "recursive_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=chunk_size,\n",
    "    chunk_overlap=chunk_overlap,\n",
    "    separators = [\"\\n\\n\", \"\\n\", \" \", \"\"]\n",
    ")\n",
    "\n",
    "character_splitter = CharacterTextSplitter(\n",
    "    chunk_size=chunk_size,\n",
    "    chunk_overlap=chunk_overlap,\n",
    "    separator = ' '\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "07e6083b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"When writing documents, writers will use document structure to group content. This can convey to the reader, which idea's are related. For example, closely related ideas are in sentances. Similar ideas are in paragraphs. Paragraphs form a document.\",\n",
       " 'Paragraphs are often delimited with a carriage return or two carriage returns. Carriage returns are the \"backslash n\" you see embedded in this string. Sentences have a period at the end, but also, have a space.and words are separated by space.']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recursive_splitter.split_text(some_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b01f5988",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['When writing documents, writers will use document structure to group content. This can convey to the reader, which idea\\'s are related. For example, closely related ideas are in sentances. Similar ideas are in paragraphs. Paragraphs form a document. \\n\\n Paragraphs are often delimited with a carriage return or two carriage returns. Carriage returns are the \"backslash n\" you see embedded in this string. Sentences have a period at the end, but also,',\n",
       " 'have a space.and words are separated by space.']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "character_splitter.split_text(some_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af596993",
   "metadata": {},
   "source": [
    "**Let's reduce the chunk size a bit and add a period to our separators:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cd078467",
   "metadata": {},
   "outputs": [],
   "source": [
    "recursive_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=150,\n",
    "    chunk_overlap=0,\n",
    "    separators = [\"\\n\\n\", \"\\n\", \"\\. \", \" \", \"\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8fc078b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"When writing documents, writers will use document structure to group content. This can convey to the reader, which idea's are related. For example,\",\n",
       " 'closely related ideas are in sentances. Similar ideas are in paragraphs. Paragraphs form a document.',\n",
       " 'Paragraphs are often delimited with a carriage return or two carriage returns. Carriage returns are the \"backslash n\" you see embedded in this',\n",
       " 'string. Sentences have a period at the end, but also, have a space.and words are separated by space.']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recursive_splitter.split_text(some_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f4e14098",
   "metadata": {},
   "outputs": [],
   "source": [
    "recursive_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=150,\n",
    "    chunk_overlap=0,\n",
    "    separators = [\"\\n\\n\", \"\\n\", \"(?<=\\. )\", \" \", \"\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "63c55188",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"When writing documents, writers will use document structure to group content. This can convey to the reader, which idea's are related. For example,\",\n",
       " 'closely related ideas are in sentances. Similar ideas are in paragraphs. Paragraphs form a document.',\n",
       " 'Paragraphs are often delimited with a carriage return or two carriage returns. Carriage returns are the \"backslash n\" you see embedded in this',\n",
       " 'string. Sentences have a period at the end, but also, have a space.and words are separated by space.']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recursive_splitter.split_text(some_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "246ce1e0",
   "metadata": {},
   "source": [
    "## PDF Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2add8421",
   "metadata": {},
   "outputs": [],
   "source": [
    "loader = PyPDFLoader(\"documents/MachineLearning-Lecture01.pdf\")\n",
    "pages = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0f3928bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = CharacterTextSplitter(\n",
    "    separator=\"\\n\",\n",
    "    chunk_size=1000,\n",
    "    chunk_overlap=150,\n",
    "    length_function=len\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d8ba21b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = text_splitter.split_documents(pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7ea988f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "77"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "41927f6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pages)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c9b9b90",
   "metadata": {},
   "source": [
    "## Token Splitting\n",
    "\n",
    "We can also split on token count explicity, if we want.\n",
    "\n",
    "This can be useful because LLMs often have context windows designated in tokens.\n",
    "\n",
    "Tokens are often ~4 characters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8aac0c9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "text1 = \"foo bar bazzyfoo\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "74bace5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = TokenTextSplitter(chunk_size=1, chunk_overlap=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "7bd9f49f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', ' bar', ' b', 'az', 'zy', 'foo']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_splitter.split_text(text1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "bb0a65c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = TokenTextSplitter(chunk_size=10, chunk_overlap=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "d1228123",
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = text_splitter.split_documents(pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "9b1bef88",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content='MachineLearning-Lecture01  \\n', metadata={'source': 'documents/MachineLearning-Lecture01.pdf', 'page': 0})"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "a3364e00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'source': 'documents/MachineLearning-Lecture01.pdf', 'page': 0}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pages[0].metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "175eb779",
   "metadata": {},
   "source": [
    "## Context Aware Splitting\n",
    "\n",
    "Chunking aims to keep text with common context together.\n",
    "\n",
    "A text splitting often uses sentences or other delimiters to keep related text together but many documents (such as Markdown) have structure (headers) that can be explicitly used in splitting.\n",
    "\n",
    "We can use MarkdownHeaderTextSplitter to preserve header metadata in our chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "94caa31f",
   "metadata": {},
   "outputs": [],
   "source": [
    "markdown_document = \"\"\"# Title\\n\\n \\\n",
    "## Chapter 1\\n\\n \\\n",
    "Hi this is Jim\\n\\n Hi this is Joe\\n\\n \\\n",
    "### Section \\n\\n \\\n",
    "Hi this is Lance \\n\\n \n",
    "## Chapter 2\\n\\n \\\n",
    "Hi this is Molly\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "4b3b3590",
   "metadata": {},
   "outputs": [],
   "source": [
    "headers_to_split_on = [\n",
    "    (\"#\", \"Header 1\"),\n",
    "    (\"##\", \"Header 2\"),\n",
    "    (\"###\", \"Header 3\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "8b06f2a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=headers_to_split_on)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d49f2667",
   "metadata": {},
   "outputs": [],
   "source": [
    "markdown_header_splits = markdown_splitter.split_text(markdown_document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "f12399af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content='Hi this is Jim  \\nHi this is Joe', metadata={'Header 1': 'Title', 'Header 2': 'Chapter 1'})"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "markdown_header_splits[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "5016306d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content='Hi this is Lance', metadata={'Header 1': 'Title', 'Header 2': 'Chapter 1', 'Header 3': 'Section'})"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "markdown_header_splits[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "d30160ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content='Hi this is Molly', metadata={'Header 1': 'Title', 'Header 2': 'Chapter 2'})"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "markdown_header_splits[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "067000d4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}